diff --git a/.gitattributes b/.gitattributes index 143d21cf121..e14b7936245 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,3 +13,8 @@ *.ttf binary *.woff binary *.woff2 binary +# Machine Learning Models +*.tflite binary +*.onnx binary +*.pb binary +*.pt binary diff --git a/.ls-lint.yml b/.ls-lint.yml index 08570eac95c..bc58a27a99d 100644 --- a/.ls-lint.yml +++ b/.ls-lint.yml @@ -71,6 +71,12 @@ ls: .ts: camelCase .tsx: camelCase .test.tsx: camelCase + apps/webapp/src/script/repositories/media/backgroundEffects: + .dir: camelCase + .ts: camelCase + .tsx: camelCase + .test.tsx: camelCase + .types.ts: camelCase apps/webapp/src/script/repositories/assets: .dir: camelCase .ts: camelCase diff --git a/apps/webapp/assets/images/backgrounds/office-1.png b/apps/webapp/assets/images/backgrounds/office-1.png new file mode 100644 index 00000000000..96a4bb0af3f Binary files /dev/null and b/apps/webapp/assets/images/backgrounds/office-1.png differ diff --git a/apps/webapp/assets/images/backgrounds/office-2.png b/apps/webapp/assets/images/backgrounds/office-2.png new file mode 100644 index 00000000000..d7fb5b4dce4 Binary files /dev/null and b/apps/webapp/assets/images/backgrounds/office-2.png differ diff --git a/apps/webapp/assets/images/backgrounds/wire-1.png b/apps/webapp/assets/images/backgrounds/wire-1.png new file mode 100644 index 00000000000..c9abaa9ed91 Binary files /dev/null and b/apps/webapp/assets/images/backgrounds/wire-1.png differ diff --git a/apps/webapp/assets/mediapipe-models/selfie_multiclass_256x256.tflite b/apps/webapp/assets/mediapipe-models/selfie_multiclass_256x256.tflite new file mode 100644 index 00000000000..584ed7ab19d Binary files /dev/null and b/apps/webapp/assets/mediapipe-models/selfie_multiclass_256x256.tflite differ diff --git a/apps/webapp/assets/mediapipe-models/selfie_segmenter.tflite b/apps/webapp/assets/mediapipe-models/selfie_segmenter.tflite deleted file mode 100644 index 59db56edb7e..00000000000 Binary files a/apps/webapp/assets/mediapipe-models/selfie_segmenter.tflite and /dev/null differ diff --git a/apps/webapp/assets/mediapipe-models/selfie_segmenter_landscape.tflite b/apps/webapp/assets/mediapipe-models/selfie_segmenter_landscape.tflite new file mode 100644 index 00000000000..3a508f8df9c Binary files /dev/null and b/apps/webapp/assets/mediapipe-models/selfie_segmenter_landscape.tflite differ diff --git a/apps/webapp/src/i18n/en-US.json b/apps/webapp/src/i18n/en-US.json index f33099182de..5dbb2e1419b 100644 --- a/apps/webapp/src/i18n/en-US.json +++ b/apps/webapp/src/i18n/en-US.json @@ -2032,8 +2032,28 @@ "verify.headline": "You’ve got mail", "verify.resendCode": "Resend code", "verify.subhead": "Enter the six-digit verification code we sent to{newline}{email}", + "videoCallBackgroundAdd": "Add background...", + "videoCallBackgroundBlurHigh": "High", + "videoCallBackgroundBlurLow": "Low", + "videoCallBackgroundBlurSectionLabel": "Blur", + "videoCallBackgroundEffectsLabel": "Background Settings", + "videoCallBackgroundEnableHighQualityBlur": "Enable high quality blur", + "videoCallBackgroundNoEffect": "No background effect", + "videoCallBackgroundNone": "None", + "videoCallBackgroundOffice1": "Office 1", + "videoCallBackgroundOffice2": "Office 2", + "videoCallBackgroundOffice3": "Office 3", + "videoCallBackgroundOffice4": "Office 4", + "videoCallBackgroundOffice5": "Office 5", + "videoCallBackgroundSettings": "Background Settings", + "videoCallBackgroundUpload": "Upload background", + "videoCallBackgroundVirtual": "Virtual Background", + "videoCallBackgroundVirtualSectionLabel": "Virtual Backgrounds", + "videoCallBackgroundWire1": "Wire 1", + "videoCallBackgroundsLabel": "Backgrounds", "videoCallMenuMoreAddReaction": "Add reaction", "videoCallMenuMoreAudioSettings": "Audio Settings", + "videoCallMenuMoreCameraSettings": "Camera Settings", "videoCallMenuMoreChangeView": "Change view", "videoCallMenuMoreCloseReactions": "Close reactions", "videoCallMenuMoreHideParticipants": "Hide participants", diff --git a/apps/webapp/src/script/auth/util/test/TestUtil.tsx b/apps/webapp/src/script/auth/util/test/TestUtil.tsx index c4a3c567e2b..8de9fb8480e 100644 --- a/apps/webapp/src/script/auth/util/test/TestUtil.tsx +++ b/apps/webapp/src/script/auth/util/test/TestUtil.tsx @@ -57,9 +57,12 @@ import sk from 'I18n/sk-SK.json'; import sl from 'I18n/sl-SI.json'; import tr from 'I18n/tr-TR.json'; import uk from 'I18n/uk-UA.json'; +import {CallingRepository} from 'Repositories/calling/CallingRepository'; import {Participant} from 'Repositories/calling/Participant'; import {Conversation} from 'Repositories/entity/Conversation'; import {User} from 'Repositories/entity/User'; +import {BackgroundEffectsController} from 'Repositories/media/backgroundEffects/effects/backgroundEffectsController'; +import {BackgroundEffectsHandler} from 'Repositories/media/backgroundEffectsHandler'; import {MediaDevicesHandler} from 'Repositories/media/MediaDevicesHandler'; import {setStrings} from 'Util/localizerUtil'; import {createUuid} from 'Util/uuid'; @@ -190,3 +193,16 @@ export const buildMediaDevicesHandler = () => { setOnMediaDevicesRefreshHandler: jest.fn(), } as unknown as MediaDevicesHandler; }; + +export const buildCallingRepository = () => { + const controller: BackgroundEffectsController = { + getQuality: jest.fn(), + getCapabilityInfo: jest.fn(), + } as unknown as BackgroundEffectsController; + const backgroundEffectsHandler = new BackgroundEffectsHandler(controller); + + return { + getBackgroundEffectsHandler: () => backgroundEffectsHandler, + isSuperhighQualityTierAllowed: jest.fn(), + } as unknown as CallingRepository; +}; diff --git a/apps/webapp/src/script/components/ConfigToolbar/ConfigToolbar.tsx b/apps/webapp/src/script/components/ConfigToolbar/ConfigToolbar.tsx index 2b08adb4c07..fb3783108f1 100644 --- a/apps/webapp/src/script/components/ConfigToolbar/ConfigToolbar.tsx +++ b/apps/webapp/src/script/components/ConfigToolbar/ConfigToolbar.tsx @@ -43,6 +43,9 @@ export function ConfigToolbar() { const wrapperRef = useRef(null); const [avsDebuggerEnabled, setAvsDebuggerEnabled] = useState(!!window.wire?.app?.debug?.isEnabledAvsDebugger()); const [avsRustSftEnabled, setAvsRustSftEnabled] = useState(!!window.wire?.app?.debug?.isEnabledAvsRustSFT()); + const [videoBackgroundEffectsFeatureEnabled, setVideoBackgroundEffectsFeatureEnabled] = useState( + !!window.wire?.app?.debug?.isVideoBackgroundEffectsFeatureEnabled(), + ); const [coreCryptoLevel, setCoreCryptoLevel] = useState(CoreCryptoLogLevel.Info); // Toggle config tool on 'cmd/ctrl + shift + 2' @@ -219,6 +222,24 @@ export function ConfigToolbar() { ); }; + const handleBackgroundEffectsFeature = (isChecked: boolean) => { + setVideoBackgroundEffectsFeatureEnabled(!!window.wire?.app?.debug?.enableVideoBackgroundEffectsFeature(isChecked)); + }; + const renderBackgroundEffectsFeatureSelect = () => { + return ( +
+ + handleBackgroundEffectsFeature(isChecked)} + /> +
+ ); + }; + const renderGzipSwitch = () => { return (
@@ -318,6 +339,10 @@ export function ConfigToolbar() {
+
{renderBackgroundEffectsFeatureSelect()}
+ +
+
{renderGzipSwitch()}

diff --git a/apps/webapp/src/script/components/calling/CallingOverlayContainer.tsx b/apps/webapp/src/script/components/calling/CallingOverlayContainer.tsx index c4147e275a5..e2f78aa9b2b 100644 --- a/apps/webapp/src/script/components/calling/CallingOverlayContainer.tsx +++ b/apps/webapp/src/script/components/calling/CallingOverlayContainer.tsx @@ -170,7 +170,7 @@ const CallingContainer = ({ switchCameraInput={switchCameraInput} switchMicrophoneInput={switchMicrophoneInput} switchSpeakerOutput={switchSpeakerOutput} - switchBlurredBackground={status => callingRepository.switchVideoBackgroundBlur(status)} + switchVideoBackgroundEffect={effect => callingRepository.switchVideoBackgroundEffect(effect)} setMaximizedParticipant={setMaximizedParticipant} setActiveCallViewTab={setActiveCallViewTab} toggleMute={toggleMute} diff --git a/apps/webapp/src/script/components/calling/FullscreenVideoCall.tsx b/apps/webapp/src/script/components/calling/FullscreenVideoCall.tsx index 6dc75aacfb3..8ee52db8482 100644 --- a/apps/webapp/src/script/components/calling/FullscreenVideoCall.tsx +++ b/apps/webapp/src/script/components/calling/FullscreenVideoCall.tsx @@ -17,14 +17,13 @@ * */ -import React, {useEffect, useState} from 'react'; +import {ChangeEvent, useEffect, useState} from 'react'; import {DefaultConversationRoleName} from '@wireapp/api-client/lib/conversation/'; import cx from 'classnames'; import {container} from 'tsyringe'; import { - TabIndex, Checkbox, CheckboxLabel, CloseDetachedWindowIcon, @@ -32,11 +31,13 @@ import { IconButtonVariant, OpenDetachedWindowIcon, QUERY, + TabIndex, } from '@wireapp/react-ui-kit'; import {WebAppEvents} from '@wireapp/webapp-events'; import {useAppNotification} from 'Components/AppNotification/AppNotification'; import {useCallAlertState} from 'Components/calling/useCallAlertState'; +import {VideoBackgroundPerformancePanel} from 'Components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel'; import {ConversationClassifiedBar} from 'Components/ClassifiedBar/ClassifiedBar'; import * as Icon from 'Components/Icon'; import {ModalComponent} from 'Components/Modals/ModalComponent'; @@ -47,6 +48,9 @@ import {Participant} from 'Repositories/calling/Participant'; import type {Grid} from 'Repositories/calling/videoGridHandler'; import type {Conversation} from 'Repositories/entity/Conversation'; import {MediaDevicesHandler} from 'Repositories/media/MediaDevicesHandler'; +import {useBackgroundEffectsStore} from 'Repositories/media/useBackgroundEffectsStore'; +import type {BackgroundEffectSelection} from 'Repositories/media/VideoBackgroundEffects'; +import {BUILTIN_BACKGROUNDS} from 'Repositories/media/VideoBackgroundEffects'; import {PropertiesRepository} from 'Repositories/properties/PropertiesRepository'; import {TeamState} from 'Repositories/team/TeamState'; import {useActiveWindowMatchMedia} from 'src/script/hooks/useActiveWindowMatchMedia'; @@ -63,14 +67,15 @@ import {Duration} from './Duration'; import { classifiedBarStyles, headerActionsWrapperStyles, - paginationWrapperStyles, - videoTopBarStyles, minimizeButtonStyles, openDetachedWindowButtonStyles, paginationStyles, + paginationWrapperStyles, + videoTopBarStyles, } from './FullscreenVideoCall.styles'; import {GroupVideoGrid} from './GroupVideoGrid'; import {Pagination} from './Pagination/Pagination'; +import {VideoBackgroundSettings} from './VideoControls/VideoBackgroundSettings/VideoBackgroundSettings'; import {VideoControls} from './VideoControls/VideoControls'; import {useWarningsState} from '../../view_model/WarningsContainer/WarningsState'; @@ -95,7 +100,7 @@ export interface FullscreenVideoCallProps { switchCameraInput: (deviceId: string) => void; switchMicrophoneInput: (deviceId: string) => void; switchSpeakerOutput: (deviceId: string) => void; - switchBlurredBackground: (status: boolean) => void; + switchVideoBackgroundEffect: (effect: BackgroundEffectSelection) => void; teamState?: TeamState; callState?: CallState; toggleCamera: (call: Call) => void; @@ -125,7 +130,7 @@ const FullscreenVideoCall = ({ switchCameraInput, switchMicrophoneInput, switchSpeakerOutput, - switchBlurredBackground, + switchVideoBackgroundEffect, setMaximizedParticipant, setActiveCallViewTab, toggleMute, @@ -189,6 +194,7 @@ const FullscreenVideoCall = ({ const openPopup = () => callingRepository.setViewModeDetached(); const [isParticipantsListOpen, toggleParticipantsList] = useToggleState(false); + const [isBackgroundSidebarOpen, setIsBackgroundSidebarOpen] = useState(false); const callNotification = useAppNotification({ activeWindow: viewMode === CallingViewMode.DETACHED_WINDOW ? detachedWindow! : window, @@ -276,6 +282,17 @@ const FullscreenVideoCall = ({ const isPaginationVisible = !maximizedParticipant && activeCallViewTab === CallViewTab.ALL && totalPages > 1; const isModerator = selfUser && roles[selfUser.id] === DefaultConversationRoleName.WIRE_ADMIN; + const backgroundEffectsHandler = callingRepository.getBackgroundEffectsHandler(); + + const selectedBackgroundEffect = useBackgroundEffectsStore(state => state.preferredEffect); + + const handleBackgroundSidebarSelect = (effect: BackgroundEffectSelection) => { + void switchVideoBackgroundEffect(effect); + }; + + const handleEnableHighQualityBlur = (event: ChangeEvent) => { + callingRepository.allowSuperhighQualityTier(event.target.checked); + }; return (
)} + {isMobile && isBackgroundSidebarOpen && ( + setIsBackgroundSidebarOpen(false)} + highQualityBlurAllowed={callingRepository.isSuperhighQualityTierAllowed()} + /> + )}
{isMobile && isPaginationVisible && ( @@ -445,11 +472,12 @@ const FullscreenVideoCall = ({ toggleIsHandRaised={toggleIsHandRaised} switchMicrophoneInput={switchMicrophoneInput} switchSpeakerOutput={switchSpeakerOutput} - switchBlurredBackground={switchBlurredBackground} + switchVideoBackgroundEffect={switchVideoBackgroundEffect} switchCameraInput={switchCameraInput} setActiveCallViewTab={setActiveCallViewTab} setMaximizedParticipant={setMaximizedParticipant} sendEmoji={sendEmoji} + onOpenBackgroundSettings={() => setIsBackgroundSidebarOpen(true)} /> )} @@ -466,6 +494,16 @@ const FullscreenVideoCall = ({ onClose={toggleParticipantsList} /> )} + {!isMobile && isBackgroundSidebarOpen && ( + setIsBackgroundSidebarOpen(false)} + highQualityBlurAllowed={callingRepository.isSuperhighQualityTierAllowed()} + /> + )} setIsConfirmCloseModalOpen(false)} @@ -490,7 +528,7 @@ const FullscreenVideoCall = ({ wrapperCSS={{marginTop: 16}} data-uie-name="do-not-ask-again-checkbox" id="do-not-ask-again-checkbox" - onChange={(event: React.ChangeEvent) => + onChange={(event: ChangeEvent) => localStorage.setItem( LOCAL_STORAGE_KEY_FOR_SCREEN_SHARING_CONFIRM_MODAL, event.target.checked.toString(), @@ -535,6 +573,7 @@ const FullscreenVideoCall = ({ )} +
); }; diff --git a/apps/webapp/src/script/components/calling/GroupVideoGrid.tsx b/apps/webapp/src/script/components/calling/GroupVideoGrid.tsx index 449d00a62dc..bd8c2864413 100644 --- a/apps/webapp/src/script/components/calling/GroupVideoGrid.tsx +++ b/apps/webapp/src/script/components/calling/GroupVideoGrid.tsx @@ -144,7 +144,7 @@ const GroupVideoGrid = ({ 'hasActiveVideo', 'sharesScreen', 'videoStream', - 'blurredVideoStream', + 'processedVideoStream', ]); const [rowsAndColumns, setRowsAndColumns] = useState( @@ -287,7 +287,7 @@ const GroupVideoGrid = ({ css={{ transform: thumbnail.hasActiveVideo && !thumbnail.sharesScreen ? 'rotateY(180deg)' : 'initial', }} - srcObject={thumbnail.blurredVideoStream?.stream ?? thumbnail.videoStream} + srcObject={thumbnail.processedVideoStream?.stream ?? thumbnail.videoStream} /> {selfIsMuted && !minimized && ( diff --git a/apps/webapp/src/script/components/calling/GroupVideoGridTile.tsx b/apps/webapp/src/script/components/calling/GroupVideoGridTile.tsx index c4b39bb663a..11c5e7d2c2b 100644 --- a/apps/webapp/src/script/components/calling/GroupVideoGridTile.tsx +++ b/apps/webapp/src/script/components/calling/GroupVideoGridTile.tsx @@ -65,7 +65,7 @@ const GroupVideoGridTile = ({ videoState, handRaisedAt, videoStream, - blurredVideoStream, + processedVideoStream, isActivelySpeaking, isAudioEstablished, isSwitchingVideoResolution, @@ -73,7 +73,7 @@ const GroupVideoGridTile = ({ 'isMuted', 'handRaisedAt', 'videoStream', - 'blurredVideoStream', + 'processedVideoStream', 'isActivelySpeaking', 'videoState', 'isAudioEstablished', @@ -142,7 +142,7 @@ const GroupVideoGridTile = ({ see https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide. */ muted - srcObject={blurredVideoStream?.stream ?? videoStream} + srcObject={processedVideoStream?.stream ?? videoStream} className="group-video-grid__element-video" css={groupVideoElementVideo(isMaximized || sharesScreen, participant === selfParticipant && sharesCamera)} /> diff --git a/apps/webapp/src/script/components/calling/VideoControls/VideoBackgroundSettings/VideoBackgroundSettings.styles.ts b/apps/webapp/src/script/components/calling/VideoControls/VideoBackgroundSettings/VideoBackgroundSettings.styles.ts new file mode 100644 index 00000000000..adcc8f1aaec --- /dev/null +++ b/apps/webapp/src/script/components/calling/VideoControls/VideoBackgroundSettings/VideoBackgroundSettings.styles.ts @@ -0,0 +1,143 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {CSSObject} from '@emotion/react'; + +import {media} from '@wireapp/react-ui-kit'; + +export const backgroundSettingsWrapperStyles: CSSObject = { + borderLeft: '1px solid var(--border-color)', + display: 'flex', + flexDirection: 'column', + backgroundColor: 'var(--app-bg-secondary)', + overflowY: 'hidden', + width: 280, + flexShrink: 0, + + [media.mobile]: { + position: 'absolute', + inset: 0, + width: '100%', + height: '100%', + zIndex: 2, + border: '2px solid var(--accent-color)', + borderRadius: 10, + }, +}; + +export const backgroundSettingsHeaderStyles: CSSObject = { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + padding: '12px 16px', + borderBottom: '1px solid var(--border-color)', + flexShrink: 0, + + '& button': { + minWidth: 'auto', + minHeight: 'auto', + }, +}; + +export const backgroundSettingsTitleStyles: CSSObject = { + fontSize: 14, + fontWeight: 'var(--font-weight-semibold)', + color: 'var(--main-color)', +}; + +export const backgroundSettingsScrollableContentStyles: CSSObject = { + overflowY: 'auto', + flex: 1, + padding: '12px 16px 16px', + display: 'flex', + flexDirection: 'column', + gap: 20, +}; + +export const sectionLabelStyles: CSSObject = { + fontSize: 11, + fontWeight: 600, + letterSpacing: '0.06em', + textTransform: 'uppercase', + color: 'var(--gray-70)', + marginBottom: 8, +}; + +/** 2-column grid for blur and virtual background tiles. */ +export const tileGridStyles: CSSObject = { + display: 'grid', + gridTemplateColumns: 'repeat(2, 1fr)', + gap: 8, +}; + +export const tileButtonStyles: CSSObject = { + background: 'none', + border: 'none', + color: 'var(--main-color)', + cursor: 'pointer', + display: 'flex', + flexDirection: 'column', + gap: 5, + padding: 0, + textAlign: 'center', + + '&:focus-visible .bg-tile__preview': { + outline: '2px solid var(--accent-color-focus)', + outlineOffset: 2, + }, + + '&[data-selected="true"] .bg-tile__preview': { + borderColor: 'var(--accent-color)', + boxShadow: '0 0 0 2px var(--accent-color)', + }, + + '&:hover .bg-tile__preview': { + transform: 'translateY(-1px)', + }, + + '&:disabled': { + cursor: 'default', + opacity: 0.5, + }, +}; + +export const tilePreviewStyles: CSSObject = { + position: 'relative', + width: '100%', + height: 70, + borderRadius: 8, + border: '1px solid var(--inactive-call-button-border)', + backgroundColor: 'var(--gray-20)', + backgroundSize: 'cover', + backgroundPosition: 'center', + overflow: 'hidden', + transition: 'transform 0.15s ease, box-shadow 0.15s ease, border-color 0.15s ease', + display: 'flex', + alignItems: 'center', + justifyContent: 'center', +}; + +export const tilePreviewContentStyles: CSSObject = { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + gap: 4, + fontSize: 10, + color: 'black', // no design for dark mode, so keeping it black. +}; diff --git a/apps/webapp/src/script/components/calling/VideoControls/VideoBackgroundSettings/VideoBackgroundSettings.tsx b/apps/webapp/src/script/components/calling/VideoControls/VideoBackgroundSettings/VideoBackgroundSettings.tsx new file mode 100644 index 00000000000..e50e6cfa77e --- /dev/null +++ b/apps/webapp/src/script/components/calling/VideoControls/VideoBackgroundSettings/VideoBackgroundSettings.tsx @@ -0,0 +1,190 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {ChangeEvent, CSSProperties, ReactNode} from 'react'; + +import {BlurHighIcon, BlurLowIcon, Checkbox, CheckboxLabel, CircleIcon} from '@wireapp/react-ui-kit'; + +import {FadingScrollbar} from 'Components/FadingScrollbar'; +import * as Icon from 'Components/Icon'; +import type {BackgroundEffectSelection, BuiltinBackground} from 'Repositories/media/VideoBackgroundEffects'; +import {t} from 'Util/localizerUtil'; + +import { + backgroundSettingsHeaderStyles, + backgroundSettingsScrollableContentStyles, + backgroundSettingsTitleStyles, + backgroundSettingsWrapperStyles, + sectionLabelStyles, + tileButtonStyles, + tileGridStyles, + tilePreviewContentStyles, + tilePreviewStyles, +} from './VideoBackgroundSettings.styles'; + +interface VideoBackgroundSettingsProps { + selectedEffect: BackgroundEffectSelection; + backgrounds: BuiltinBackground[]; + onSelectEffect: (effect: BackgroundEffectSelection) => void; + onEnableHighQualityBlur: (event: ChangeEvent) => void; + onClose: () => void; + highQualityBlurAllowed: boolean; +} + +const isEffectSelected = (selected: BackgroundEffectSelection, candidate: BackgroundEffectSelection): boolean => { + if (selected.type !== candidate.type) { + return false; + } + if (selected.type === 'blur' && candidate.type === 'blur') { + return selected.level === candidate.level; + } + if (selected.type === 'virtual' && candidate.type === 'virtual') { + return selected.backgroundId === candidate.backgroundId; + } + return true; +}; + +interface BackgroundTileProps { + effect: BackgroundEffectSelection; + selectedEffect: BackgroundEffectSelection; + onSelectEffect: (effect: BackgroundEffectSelection) => void; + previewContent?: ReactNode; + previewStyle?: CSSProperties; +} + +const BackgroundTile = ({ + effect, + selectedEffect, + onSelectEffect, + previewContent, + previewStyle, +}: BackgroundTileProps) => { + const selected = isEffectSelected(selectedEffect, effect); + return ( + + ); +}; + +export const VideoBackgroundSettings = ({ + selectedEffect, + backgrounds, + onSelectEffect, + highQualityBlurAllowed, + onEnableHighQualityBlur, + onClose, +}: VideoBackgroundSettingsProps) => { + const handleEnableHighQualityBlur = (event: ChangeEvent) => { + onEnableHighQualityBlur(event); + }; + + return ( +
+
+ {t('videoCallBackgroundEffectsLabel')} + +
+ + + {/* No background effect — full-width tile */} + + + {t('videoCallBackgroundNoEffect')} +
+ } + /> + + {/* Blur section */} +
+
{t('videoCallBackgroundBlurSectionLabel')}
+
+ + + {t('videoCallBackgroundBlurLow')} +
+ } + /> + + + {t('videoCallBackgroundBlurHigh')} +
+ } + /> + + + +
+ ) => handleEnableHighQualityBlur(event)} + > + + {t('videoCallBackgroundEnableHighQualityBlur')} + + +
+ + {/* Virtual backgrounds section */} +
+
{t('videoCallBackgroundVirtualSectionLabel')}
+
+ {backgrounds.map(background => ( + + ))} +
+
+ + + ); +}; diff --git a/apps/webapp/src/script/components/calling/VideoControls/VideoControls.styles.ts b/apps/webapp/src/script/components/calling/VideoControls/VideoControls.styles.ts index fe561fa96f4..9ef6ab06740 100644 --- a/apps/webapp/src/script/components/calling/VideoControls/VideoControls.styles.ts +++ b/apps/webapp/src/script/components/calling/VideoControls/VideoControls.styles.ts @@ -93,3 +93,149 @@ export const videoControlInActiveStyles = css` background-color: var(--accent-color-highlight); } `; + +export const videoOptionsMenuStyles: CSSObject = { + position: 'absolute', + bottom: 44, + left: '50%', + transform: 'translateX(-50%)', + width: 320, + maxHeight: '70vh', + borderRadius: 16, + backgroundColor: 'var(--app-bg-secondary)', + border: '1px solid var(--message-actions-border)', + boxShadow: '0 12px 30px rgba(0, 0, 0, 0.18)', + display: 'flex', + flexDirection: 'column', + gap: 12, + overflowY: 'auto', + zIndex: 10, + padding: 0, + paddingTop: 12, +}; + +export const videoOptionsSelectMenuStyles: CSSObject = { + backgroundColor: 'transparent', + boxShadow: 'none', + borderRadius: 0, + marginTop: 0, +}; + +export const videoOptionsSelectGroupHeadingStyles: CSSObject = { + // fontSize: 11, + // fontWeight: 600, + // letterSpacing: '0.06em', + // textTransform: 'uppercase', + // color: 'var(--gray-70)', +}; + +export const videoOptionsRowButtonStyles = css` + align-items: center; + background: none; + border: none; + + color: inherit; + cursor: pointer; /* gleiche Einrückung wie Select Option */ + + display: flex; + font: inherit; + + justify-content: space-between; + padding: 8px 12px 8px 32px; + + text-align: left; + width: 100%; + + &:hover { + background-color: var(--gray-10); + } + + &:focus-visible { + outline: 2px solid var(--accent-color-focus); + outline-offset: -2px; + } +`; + +export const videoOptionsRowIconStyles: CSSObject = { + width: 12, + height: 12, + fill: 'currentColor', +}; + +export const videoOptionsBackButtonStyles = css` + align-items: center; + background: none; + border: none; + color: var(--main-color); + cursor: pointer; + display: inline-flex; + font-size: 12px; + font-weight: 600; + gap: 6px; + padding: 0; + + &:focus-visible { + outline: 2px solid var(--accent-color-focus); + outline-offset: 2px; + } +`; + +export const videoOptionsSheetStyles: CSSObject = { + position: 'fixed', + left: 0, + right: 0, + bottom: 0, + padding: '16px 16px 20px', + backgroundColor: 'var(--app-bg-secondary)', + borderTopLeftRadius: 16, + borderTopRightRadius: 16, + boxShadow: '0 -12px 30px rgba(0, 0, 0, 0.2)', + zIndex: 1001, + maxHeight: '70vh', + overflowY: 'auto', +}; + +export const videoOptionsSheetHeaderStyles: CSSObject = { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + marginBottom: 12, +}; + +export const videoOptionsSheetTitleStyles: CSSObject = { + fontSize: 12, + fontWeight: 600, + letterSpacing: '0.04em', + textTransform: 'uppercase', + color: 'var(--gray-70)', +}; + +export const videoOptionsBackdropStyles: CSSObject = { + position: 'fixed', + inset: 0, + backgroundColor: 'rgba(0, 0, 0, 0.35)', + zIndex: 1000, +}; + +export const videoOptionInlineMenuStyles: CSSObject = { + width: '100%', + minWidth: '0', + boxSizing: 'border-box', +}; + +export const videoOptionLabelStyles = { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + width: '100%', +}; + +export const videoOptionLabelTextStyles = {}; + +export const videoOptionLabelIconStyles = { + color: 'inherit', +}; + +export const videoOptionsInlineWrapperStyles = { + marginBottom: 0, +}; diff --git a/apps/webapp/src/script/components/calling/VideoControls/VideoControls.tsx b/apps/webapp/src/script/components/calling/VideoControls/VideoControls.tsx index 673945d7bf4..65cd9d73978 100644 --- a/apps/webapp/src/script/components/calling/VideoControls/VideoControls.tsx +++ b/apps/webapp/src/script/components/calling/VideoControls/VideoControls.tsx @@ -17,13 +17,24 @@ * */ -import React, {useCallback, useEffect, useMemo, useState} from 'react'; +import React, {useCallback, useEffect, useMemo, useRef, useState} from 'react'; import classNames from 'classnames'; import {container} from 'tsyringe'; import {CALL_TYPE} from '@wireapp/avs'; -import {EmojiIcon, GridIcon, MoreIcon, QUERY, RaiseHandIcon, TabIndex} from '@wireapp/react-ui-kit'; +import { + BlurHighIcon, + BlurLowIcon, + CircleIcon, + EmojiIcon, + GridIcon, + ImageIcon, + MoreIcon, + QUERY, + RaiseHandIcon, + TabIndex, +} from '@wireapp/react-ui-kit'; import {WebAppEvents} from '@wireapp/webapp-events'; import * as Icon from 'Components/Icon'; @@ -34,7 +45,10 @@ import {CallingViewMode, CallState} from 'Repositories/calling/CallState'; import {Participant} from 'Repositories/calling/Participant'; import {Conversation} from 'Repositories/entity/Conversation'; import {ElectronDesktopCapturerSource, MediaDevicesHandler} from 'Repositories/media/MediaDevicesHandler'; +import {useBackgroundEffectsStore} from 'Repositories/media/useBackgroundEffectsStore'; import {useMediaDevicesStore} from 'Repositories/media/useMediaDevicesStore'; +import {BackgroundEffectSelection, DEFAULT_BUILTIN_BACKGROUND_ID} from 'Repositories/media/VideoBackgroundEffects'; +import {DEFAULT_BACKGROUND_EFFECT} from 'Repositories/media/VideoBackgroundEffects'; import {PropertiesRepository} from 'Repositories/properties/PropertiesRepository'; import {PROPERTIES_TYPE} from 'Repositories/properties/PropertiesType'; import {TeamState} from 'Repositories/team/TeamState'; @@ -55,13 +69,43 @@ import { videoControlDisabledStyles, videoControlInActiveStyles, videoControlsWrapperStyles, + videoOptionsBackdropStyles, + videoOptionsMenuStyles, + videoOptionsRowIconStyles, + videoOptionsSheetStyles, } from './VideoControls.styles'; import {VideoControlsSelect} from './VideoControlsSelect/VideoControlsSelect'; -enum BlurredBackgroundStatus { - OFF = 'bluroff', - ON = 'bluron', -} +type BackgroundOptionValue = 'none' | 'blur-high' | 'blur-low' | 'virtual' | 'settings'; + +const BACKGROUND_OPTION_VALUES = new Set(['none', 'blur-high', 'blur-low', 'virtual', 'settings']); + +const mapValueToEffect = (value: BackgroundOptionValue): BackgroundEffectSelection => { + switch (value) { + case 'none': + return {type: 'none'}; + case 'blur-high': + return {type: 'blur', level: 'high'}; + case 'blur-low': + return {type: 'blur', level: 'low'}; + case 'virtual': + return {type: 'virtual', backgroundId: DEFAULT_BUILTIN_BACKGROUND_ID}; + default: + return {type: 'none'}; + } +}; + +const mapEffectToValue = (effect: BackgroundEffectSelection): BackgroundOptionValue => { + switch (effect.type) { + case 'blur': + return effect.level === 'high' ? 'blur-high' : 'blur-low'; + case 'virtual': + case 'custom': + return 'virtual'; + default: + return 'none'; + } +}; /** * Maps video input devices to select options. @@ -118,11 +162,12 @@ interface VideoControlsProps { toggleIsHandRaised: (isHandRaised: boolean) => void; switchMicrophoneInput: (deviceId: string) => void; switchSpeakerOutput: (deviceId: string) => void; - switchBlurredBackground: (status: boolean) => void; + switchVideoBackgroundEffect: (effect: BackgroundEffectSelection) => void; switchCameraInput: (deviceId: string) => void; setActiveCallViewTab: (tab: CallViewTab) => void; setMaximizedParticipant: (call: Call, participant: Participant | null) => void; sendEmoji: (emoji: string, call: Call) => void; + onOpenBackgroundSettings?: () => void; } export const VideoControls = ({ @@ -142,11 +187,12 @@ export const VideoControls = ({ toggleIsHandRaised, switchMicrophoneInput, switchSpeakerOutput, - switchBlurredBackground, + switchVideoBackgroundEffect, switchCameraInput, setActiveCallViewTab, setMaximizedParticipant, sendEmoji, + onOpenBackgroundSettings, teamState = container.resolve(TeamState), callState = container.resolve(CallState), }: VideoControlsProps) => { @@ -157,7 +203,6 @@ export const VideoControls = ({ handRaisedAt: selfHandRaisedAt, } = useKoSubscribableChildren(selfParticipant, ['sharesScreen', 'sharesCamera', 'handRaisedAt']); const { - ENABLE_BLUR_BACKGROUND: isBlurredBackgroundEnabled, ENABLE_PRESS_SPACE_TO_UNMUTE: isPressSpaceToUnmuteEnable, ENABLE_IN_CALL_REACTIONS: isInCallReactionsEnable, ENABLE_IN_CALL_HAND_RAISE: isInCallHandRaiseEnable, @@ -167,14 +212,16 @@ export const VideoControls = ({ const {is1to1: is1to1Conversation} = useKoSubscribableChildren(conversation, ['is1to1']); - const {blurredVideoStream} = useKoSubscribableChildren(selfParticipant, ['blurredVideoStream']); - const hasBlurredBackground = !!blurredVideoStream; + const selectedBackgroundEffect = + useBackgroundEffectsStore(state => state.preferredEffect) ?? DEFAULT_BACKGROUND_EFFECT; + const isVideoBackgroundEffectsFeatureEnabled = useBackgroundEffectsStore(state => state.isFeatureEnabled); const {participants} = useKoSubscribableChildren(call, ['participants']); const [showEmojisBar, setShowEmojisBar] = useState(false); const {viewMode, detachedWindow} = useKoSubscribableChildren(callState, ['viewMode', 'detachedWindow']); + const activeWindow = viewMode === CallingViewMode.DETACHED_WINDOW && detachedWindow ? detachedWindow : window; const {isVideoCallingEnabled} = useKoSubscribableChildren(teamState, ['isVideoCallingEnabled']); @@ -200,6 +247,8 @@ export const VideoControls = ({ const [audioOptionsOpen, setAudioOptionsOpen] = useState(false); const [videoOptionsOpen, setVideoOptionsOpen] = useState(false); const [isCallViewOpen, setIsCallViewOpen] = useState(false); + const videoOptionsMenuRef = useRef(null); + const videoOptionsSheetRef = useRef(null); const showToggleVideo = isVideoCallingEnabled && @@ -299,48 +348,132 @@ export const VideoControls = ({ switchSpeakerOutput(speaker.id); }; - const blurredBackgroundOptions = { - label: t('videoCallbackgroundBlurHeadline'), - options: [ + const cameraOptions = useMemo(() => { + const cameraDevices = mapVideoInputDevices(videoInputDevices); + return [ { - // Blurring is not possible if webgl context is not available - isDisabled: !document.createElement('canvas').getContext('webgl2'), - label: t('videoCallbackgroundBlur'), - value: BlurredBackgroundStatus.ON, - dataUieName: 'blur', - id: BlurredBackgroundStatus.ON, + label: t('videoCallvideoInputCamera'), + options: cameraDevices, }, + ]; + }, [videoInputDevices]); + + const selectedCameraOption = + cameraOptions[0].options.find(({id}) => id === currentCameraDevice) ?? cameraOptions[0].options[0]; + const selectedCameraOptions = [selectedCameraOption]; + + /** + * Handles camera device selection from the dropdown. + * + * Finds the selected camera device by value and switches the active camera input. + * + * @param selectedOption - Selected option value (device ID). + */ + const updateCameraOptions = (selectedOption: string) => { + const camera = cameraOptions[0].options.find(({value}) => value === selectedOption) ?? selectedCameraOption; + switchCameraInput(camera.id); + }; + + const currentBlurOption = useMemo( + () => + selectedBackgroundEffect.type === 'blur' && selectedBackgroundEffect.level === 'low' + ? {label: t('videoCallBackgroundBlurLow'), value: 'blur-low', icon: } + : { + label: t('videoCallBackgroundBlurHigh'), + value: 'blur-high', + icon: , + }, + [selectedBackgroundEffect], + ); + + const backgroundOptions = useMemo( + () => [ { - label: t('videoCallbackgroundNotBlurred'), - value: BlurredBackgroundStatus.OFF, - dataUieName: 'no-blur', - id: BlurredBackgroundStatus.OFF, + label: t('videoCallBackgroundEffectsLabel'), + options: [ + {label: t('videoCallBackgroundNone'), value: 'none', icon: }, + currentBlurOption, + {label: t('videoCallBackgroundVirtual'), value: 'virtual', icon: }, + { + label: t('videoCallBackgroundSettings'), + value: 'settings', + icon: , + }, + ], }, ], - }; + [currentBlurOption], + ); - const videoOptions = [ - { - label: t('videoCallvideoInputCamera'), - options: mapVideoInputDevices(videoInputDevices), + /** Merged options: camera group + (if enabled) background group. */ + const options = useMemo( + () => (isVideoBackgroundEffectsFeatureEnabled ? [...cameraOptions, ...backgroundOptions] : cameraOptions), + [cameraOptions, backgroundOptions, isVideoBackgroundEffectsFeatureEnabled], + ); + + const handleBackgroundSelect = useCallback( + (effect: BackgroundEffectSelection) => { + void switchVideoBackgroundEffect(effect); + if (isMobile) { + setVideoOptionsOpen(false); + } }, - ...(isBlurredBackgroundEnabled ? [blurredBackgroundOptions] : []), - ]; + [isMobile, switchVideoBackgroundEffect], + ); + + const handleVideoSelectChange = useCallback( + (selectedOption: any) => { + const value = selectedOption?.value as string | undefined; + if (value === 'settings') { + setVideoOptionsOpen(false); + onOpenBackgroundSettings?.(); + return; + } + if (value && BACKGROUND_OPTION_VALUES.has(value)) { + setVideoOptionsOpen(false); + const effect = mapValueToEffect(value as BackgroundOptionValue); + handleBackgroundSelect(effect); + return; + } + if (value) { + updateCameraOptions(value); + } + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [onOpenBackgroundSettings, handleBackgroundSelect], + ); + + const selectedBackgroundValue = mapEffectToValue(selectedBackgroundEffect); + + const isVideoOptionSelected = useCallback( + (option: any) => option.value === selectedCameraOption.value || option.value === selectedBackgroundValue, + [selectedCameraOption, selectedBackgroundValue], + ); - const selectedVideoOptions = [currentCameraDevice, hasBlurredBackground] - .flatMap(device => videoOptions.flatMap(options => options.options.filter(item => item.id === device)) ?? []) - .concat(hasBlurredBackground ? blurredBackgroundOptions.options[0] : blurredBackgroundOptions.options[1]); - - const updateVideoOptions = (selectedOption: string | BlurredBackgroundStatus) => { - const camera = videoOptions[0].options.find(({value}) => value === selectedOption) ?? selectedVideoOptions[0]; - if (selectedOption === BlurredBackgroundStatus.ON) { - switchBlurredBackground(true); - } else if (selectedOption === BlurredBackgroundStatus.OFF) { - switchBlurredBackground(false); - } else { - switchCameraInput(camera.id); + useEffect(() => { + if (!videoOptionsOpen) { + return undefined; } - }; + const handlePointerDown = (event: MouseEvent) => { + const target = event.target as Node; + if (videoOptionsMenuRef.current?.contains(target) || videoOptionsSheetRef.current?.contains(target)) { + return; + } + setVideoOptionsOpen(false); + }; + const handleKeyDown = (event: KeyboardEvent) => { + if (isEscapeKey(event)) { + setVideoOptionsOpen(false); + } + }; + + activeWindow.document.addEventListener('pointerdown', handlePointerDown); + activeWindow.document.addEventListener('keydown', handleKeyDown); + return () => { + activeWindow.document.removeEventListener('pointerdown', handlePointerDown); + activeWindow.document.removeEventListener('keydown', handleKeyDown); + }; + }, [activeWindow, videoOptionsOpen]); const handleEmojiClick = (selectedEmoji: string) => sendEmoji(selectedEmoji, call); @@ -417,7 +550,7 @@ export const VideoControls = ({ const isInCallHandRaiseControlVisible = isInCallHandRaiseEnable && !is1to1Conversation; - const emojisBarTargetWindow = viewMode === CallingViewMode.DETACHED_WINDOW ? detachedWindow! : window; + const emojisBarTargetWindow = activeWindow; return (
    @@ -445,26 +578,36 @@ export const VideoControls = ({ /> )} {isMobile && videoOptionsOpen && ( - { - updateVideoOptions(String(selectedOption?.value)); - setVideoOptionsOpen(false); - }} - onKeyDown={event => - handleKeyDown({ - event, - callback: () => toggleCamera(call), - keys: [KEY.ENTER, KEY.SPACE], - }) - } - id="select-camera" - dataUieName="select-camera" - options={videoOptions} - onMenuClose={() => setVideoOptionsOpen(false)} - menuIsOpen={videoOptionsOpen} - menuCSS={{width: '100vw', minWidth: 'initial'}} - /> + <> +
    setVideoOptionsOpen(false)} + onKeyDown={event => isEscapeKey(event) && setVideoOptionsOpen(false)} + role="button" + tabIndex={0} + /> +
    + + handleKeyDown({ + event, + callback: () => toggleCamera(call), + keys: [KEY.ENTER, KEY.SPACE], + }) + } + overlayMenu={false} + showHeader + onClose={() => setVideoOptionsOpen(false)} + isOptionSelected={isVideoOptionSelected} + /> +
    + )} {!isDesktop && isCallViewOpen && ( {!isMobile && ( - + {videoOptionsOpen && ( +
    updateVideoOptions(String(selectedOption?.value))} - onKeyDown={event => isEscapeKey(event) && setVideoOptionsOpen(false)} + value={selectedCameraOptions} id="select-camera" dataUieName="select-camera" - options={videoOptions} - menuIsOpen - wrapperCSS={{marginBottom: 0}} + options={options} + menuIsOpen={videoOptionsOpen} + onChange={handleVideoSelectChange} + onKeyDown={event => isEscapeKey(event) && setVideoOptionsOpen(false)} + overlayMenu={false} + isOptionSelected={isVideoOptionSelected} /> - - - ) : ( - +
    )} - +
    )} )} diff --git a/apps/webapp/src/script/components/calling/VideoControls/VideoControlsSelect/VideoControlsSelect.tsx b/apps/webapp/src/script/components/calling/VideoControls/VideoControlsSelect/VideoControlsSelect.tsx index 82e79bc704c..c56fccacaf9 100644 --- a/apps/webapp/src/script/components/calling/VideoControls/VideoControlsSelect/VideoControlsSelect.tsx +++ b/apps/webapp/src/script/components/calling/VideoControls/VideoControlsSelect/VideoControlsSelect.tsx @@ -22,9 +22,26 @@ import React from 'react'; import {Select} from '@wireapp/react-ui-kit'; import {selectGroupStyles} from 'Components/calling/VideoControls/VideoControlsSelect/VideoControlsSelect.styles'; +import * as Icon from 'Components/Icon'; +import {t} from 'Util/localizerUtil'; -type VideoControlsSelectProps = Pick< - React.ComponentProps>, +import { + videoOptionInlineMenuStyles, + videoOptionLabelIconStyles, + videoOptionLabelStyles, + videoOptionLabelTextStyles, + videoOptionsInlineWrapperStyles, + videoOptionsSelectGroupHeadingStyles, + videoOptionsSelectMenuStyles, + videoOptionsSheetHeaderStyles, + videoOptionsSheetTitleStyles, +} from '../VideoControls.styles'; + +type SelectProps = React.ComponentProps>; +type SelectOption = SelectProps['options'] extends Array ? T : never; + +type BaseSelectProps = Pick< + SelectProps, | 'value' | 'id' | 'dataUieName' @@ -32,11 +49,39 @@ type VideoControlsSelectProps = Pick< | 'onChange' | 'onMenuClose' | 'menuIsOpen' + | 'menuPlacement' | 'onKeyDown' | 'wrapperCSS' | 'menuCSS' + | 'isOptionSelected' + | 'overlayMenu' >; +export type VideoControlsSelectProps = BaseSelectProps & { + showHeader?: boolean; + onClose?: () => void; +}; + +type VideoOptionLabelProps = { + option: SelectOption & { + icon?: React.ReactNode; + label: React.ReactNode; + }; +}; + +const VideoOptionLabel = ({option}: VideoOptionLabelProps) => { + if (!option.icon) { + return <>{option.label}; + } + + return ( +
    + {option.label} + {option.icon} +
    + ); +}; + export const VideoControlsSelect = ({ value, id, @@ -46,30 +91,62 @@ export const VideoControlsSelect = ({ onKeyDown, onMenuClose, menuIsOpen, + menuPlacement, wrapperCSS, menuCSS, + overlayMenu, + showHeader, + onClose, + isOptionSelected, }: VideoControlsSelectProps) => { + const isInlineMenu = overlayMenu === false; + const menuCssWithInlineMenu = isInlineMenu + ? { + ...videoOptionsSelectMenuStyles, + ...videoOptionInlineMenuStyles, + } + : menuCSS; + return ( - : undefined + } + /> + ); }; diff --git a/apps/webapp/src/script/components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel.styles.ts b/apps/webapp/src/script/components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel.styles.ts new file mode 100644 index 00000000000..8fa762b866a --- /dev/null +++ b/apps/webapp/src/script/components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel.styles.ts @@ -0,0 +1,124 @@ +/* + * Wire + * Copyright (C) 2026 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {css} from '@emotion/react'; + +export const performancePanelContainerStyles = css({ + position: 'fixed', + top: 16, + left: 16, + zIndex: 9999, +}); + +export const performancePanelStyles = css({ + background: '#fff', + borderRadius: 12, + padding: 16, + minWidth: 220, + boxShadow: '0 4px 12px rgba(0,0,0,0.15)', +}); + +export const performancePanelButtonStyles = css({ + marginBottom: 8, +}); + +export const performancePanelSelectStyles = css({ + width: '100%', + marginBottom: 12, +}); + +export const buttonBaseStyles = css({ + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + + height: 40, + padding: '0 16px', + + borderRadius: 9999, // pill shape + border: 'none', + + fontSize: 14, + fontWeight: 500, + + cursor: 'pointer', + transition: 'all 0.15s ease', + + boxShadow: '0 2px 6px rgba(0,0,0,0.15)', +}); + +export const buttonNeutralStyles = css({ + background: '#fff', + color: '#000', + + '&:hover': { + background: '#f2f2f2', + }, +}); + +export const buttonPrimaryStyles = css({ + background: '#111', + color: '#fff', + + '&:hover': { + background: '#000', + }, +}); + +export const buttonDangerStyles = css({ + background: '#e53935', + color: '#fff', + + '&:hover': { + background: '#c62828', + }, +}); + +export const buttonIconOnlyStyles = css({ + width: 40, + padding: 0, +}); + +export const buttonRowStyles = css({ + display: 'flex', + gap: 8, // Abstand zwischen Buttons + marginTop: 12, +}); + +export const metricsListStyles = css({ + marginTop: 12, + padding: 12, + background: '#f7f7f7', + borderRadius: 8, + fontSize: 12, +}); + +export const metricsRowStyles = css({ + display: 'flex', + justifyContent: 'space-between', + marginBottom: 4, +}); + +export const metricsLabelStyles = css({ + color: '#666', +}); + +export const metricsValueStyles = css({ + fontWeight: 500, +}); diff --git a/apps/webapp/src/script/components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel.tsx b/apps/webapp/src/script/components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel.tsx new file mode 100644 index 00000000000..606f6ab822f --- /dev/null +++ b/apps/webapp/src/script/components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel.tsx @@ -0,0 +1,232 @@ +/* + * Wire + * Copyright (C) 2026 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {ChangeEvent, ReactNode, useCallback, useEffect, useMemo, useState} from 'react'; + +import { + buttonBaseStyles, + buttonNeutralStyles, + buttonRowStyles, + metricsLabelStyles, + metricsListStyles, + metricsRowStyles, + metricsValueStyles, + performancePanelContainerStyles, + performancePanelSelectStyles, + performancePanelStyles, +} from 'Components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel.styles'; +import {QualityMode} from 'Repositories/media/backgroundEffects'; +import {CapabilityInfo} from 'Repositories/media/backgroundEffects/backgroundEffectsWorkerTypes'; +import type {BackgroundEffectsHandler} from 'Repositories/media/backgroundEffectsHandler'; +import {useBackgroundEffectsStore} from 'Repositories/media/useBackgroundEffectsStore'; + +type PerformancePanelProps = { + backgroundEffectsHandler: BackgroundEffectsHandler; +}; + +const QUALITY_OPTIONS: readonly QualityMode[] = ['auto', 'superhigh', 'high', 'medium', 'low', 'bypass']; + +const formatMs = (value?: number | null): string => { + return typeof value === 'number' ? `${value.toFixed(1)} ms` : '-'; +}; + +const formatPercent = (value?: number | null): string => { + return typeof value === 'number' ? `${value.toFixed(1)} %` : '-'; +}; + +const formatValue = (value?: string | number | null): string => { + return value === null || value === undefined || value === '' ? '-' : String(value); +}; + +type MetricRowProps = { + label: string; + value: ReactNode; +}; + +const MetricRow = ({label, value}: MetricRowProps) => ( +
    + {label} + {value} +
    +); + +const POLLING_INTERVAL = 500; + +export const VideoBackgroundPerformancePanel = ({backgroundEffectsHandler}: PerformancePanelProps) => { + const isFeatureEnabled = useBackgroundEffectsStore(state => state.isFeatureEnabled); + const renderMetrics = useBackgroundEffectsStore(state => state.metrics); + const model = useBackgroundEffectsStore(state => state.model); + + const [selectedQuality, setSelectedQuality] = useState(() => backgroundEffectsHandler.getQuality()); + const [isPanelOpen, setIsPanelOpen] = useState(false); + const [capabilityInfo, setCapabilityInfo] = useState(null); + + useEffect(() => { + setCapabilityInfo(backgroundEffectsHandler.getCapabilityInfo()); + }, [backgroundEffectsHandler]); + + // Quality polling (fallback for non-reactive quality) + useEffect(() => { + const interval = setInterval(() => { + const current = backgroundEffectsHandler.getQuality(); + setSelectedQuality(prev => (prev !== current ? current : prev)); + }, POLLING_INTERVAL); + + return () => clearInterval(interval); + }, [backgroundEffectsHandler]); + + // Auto close if disabled + useEffect(() => { + if (!isFeatureEnabled && isPanelOpen) { + setIsPanelOpen(false); + } + }, [isFeatureEnabled, isPanelOpen]); + + const handleOpenPanel = useCallback(() => { + setIsPanelOpen(true); + }, []); + + const handleClosePanel = useCallback(() => { + setIsPanelOpen(false); + }, []); + + const handleQualityChange = useCallback( + (event: ChangeEvent) => { + const nextQuality = event.target.value as QualityMode; + setSelectedQuality(nextQuality); + backgroundEffectsHandler.applyQuality(nextQuality); + }, + [backgroundEffectsHandler], + ); + + const handleApplyQuality = useCallback(() => { + backgroundEffectsHandler.applyQuality(selectedQuality); + }, [backgroundEffectsHandler, selectedQuality]); + + const handleResetQuality = useCallback(() => { + const defaultQuality: QualityMode = 'auto'; + setSelectedQuality(defaultQuality); + backgroundEffectsHandler.applyQuality(defaultQuality); + }, [backgroundEffectsHandler]); + + const metricRows = useMemo(() => { + if (!renderMetrics) { + return []; + } + + return [ + {label: 'Quality', value: formatValue(renderMetrics.tier)}, + {label: 'Total', value: formatMs(renderMetrics.avgTotalMs)}, + {label: 'Segmentation', value: formatMs(renderMetrics.avgSegmentationMs)}, + {label: 'GPU', value: formatMs(renderMetrics.avgGpuMs)}, + {label: 'Budget', value: formatMs(renderMetrics.budget)}, + {label: 'ML delegate type', value: formatValue(renderMetrics.ml)}, + {label: 'Utilization', value: formatPercent(renderMetrics.utilShare)}, + {label: 'ML', value: formatPercent(renderMetrics.mlShare)}, + {label: 'WebGL', value: formatPercent(renderMetrics.webglShare)}, + { + label: 'Delegate', + value: formatValue(renderMetrics.segmentationDelegate), + }, + {label: 'Dropped', value: formatValue(renderMetrics.droppedFrames)}, + ]; + }, [renderMetrics]); + + const capabilityRows = useMemo(() => { + if (!capabilityInfo) { + return []; + } + + return [ + {label: 'WebGL2', value: capabilityInfo.webgl2 ? '✔' : '✖'}, + {label: 'Worker', value: capabilityInfo.worker ? '✔' : '✖'}, + { + label: 'OffscreenCanvas', + value: capabilityInfo.offscreenCanvas ? '✔' : '✖', + }, + { + label: 'VideoFrameCallback', + value: capabilityInfo.requestVideoFrameCallback ? '✔' : '✖', + }, + ]; + }, [capabilityInfo]); + + if (!isFeatureEnabled) { + return null; + } + + return ( +
    + + + {isPanelOpen && ( +
    +

    Performance Panel

    + + + +
    + + + + + +
    + +
    + + + {metricRows.map(row => ( + + ))} + + {capabilityRows.map(row => ( + + ))} +
    +
    + )} +
    + ); +}; diff --git a/apps/webapp/src/script/components/calling/fullscreenVideoCall.test.tsx b/apps/webapp/src/script/components/calling/fullscreenVideoCall.test.tsx index db36ca18d58..b4bed0d053d 100644 --- a/apps/webapp/src/script/components/calling/fullscreenVideoCall.test.tsx +++ b/apps/webapp/src/script/components/calling/fullscreenVideoCall.test.tsx @@ -29,7 +29,7 @@ import {User} from 'Repositories/entity/User'; import {PropertiesRepository} from 'Repositories/properties/PropertiesRepository'; import {PropertiesService} from 'Repositories/properties/PropertiesService'; import {SelfService} from 'Repositories/self/SelfService'; -import {buildMediaDevicesHandler, withTheme} from 'src/script/auth/util/test/TestUtil'; +import {buildCallingRepository, buildMediaDevicesHandler, withTheme} from 'src/script/auth/util/test/TestUtil'; import {FullscreenVideoCall, FullscreenVideoCallProps} from './FullscreenVideoCall'; @@ -62,6 +62,7 @@ describe('fullscreenVideoCall', () => { isMuted: false, propertiesRepository: new PropertiesRepository({} as PropertiesService, {} as SelfService), mediaDevicesHandler: buildMediaDevicesHandler(), + callingRepository: buildCallingRepository(), videoGrid: {grid: [], thumbnail: null} as Grid, }; return props as FullscreenVideoCallProps; diff --git a/apps/webapp/src/script/main/app.ts b/apps/webapp/src/script/main/app.ts index a40ea1c403e..45e507a301a 100644 --- a/apps/webapp/src/script/main/app.ts +++ b/apps/webapp/src/script/main/app.ts @@ -69,6 +69,8 @@ import {GiphyService} from 'Repositories/extension/GiphyService'; import {IntegrationRepository} from 'Repositories/integration/IntegrationRepository'; import {IntegrationService} from 'Repositories/integration/IntegrationService'; import {LifeCycleRepository} from 'Repositories/LifeCycleRepository/LifeCycleRepository'; +import {BackgroundEffectsController} from 'Repositories/media/backgroundEffects/effects/backgroundEffectsController'; +import {BackgroundEffectsHandler} from 'Repositories/media/backgroundEffectsHandler'; import {MediaConstraintsHandler} from 'Repositories/media/MediaConstraintsHandler'; import {MediaDevicesHandler} from 'Repositories/media/MediaDevicesHandler'; import {MediaStreamHandler} from 'Repositories/media/MediaStreamHandler'; @@ -212,6 +214,7 @@ export class App { const mediaStreamHandler = new MediaStreamHandler(mediaConstraintsHandler); const mediaDevicesHandler = new MediaDevicesHandler(); + const backgroundEffectsHandler = new BackgroundEffectsHandler(new BackgroundEffectsController()); container.registerInstance(MediaDevicesHandler, mediaDevicesHandler); container.registerInstance(MediaStreamHandler, mediaStreamHandler); @@ -274,6 +277,7 @@ export class App { mediaStreamHandler, mediaDevicesHandler, serverTimeHandler, + backgroundEffectsHandler, ); repositories.self = new SelfRepository(selfService, repositories.user, repositories.team, repositories.client); diff --git a/apps/webapp/src/script/repositories/calling/CallingRepository.test.ts b/apps/webapp/src/script/repositories/calling/CallingRepository.test.ts index e32197992c3..65b6a50d3ff 100644 --- a/apps/webapp/src/script/repositories/calling/CallingRepository.test.ts +++ b/apps/webapp/src/script/repositories/calling/CallingRepository.test.ts @@ -633,6 +633,7 @@ describe('CallingRepository ISO', () => { { toServerTimestamp: jest.fn().mockImplementation(() => Date.now()), } as any, // ServerTimeHandler + {} as any, // BackgroundEffectsHandler {} as any, // APIClient { findConversation: jest.fn().mockImplementation(() => conversation), @@ -756,6 +757,7 @@ describe.skip('E2E audio call', () => { serverTimeHandler as any, {} as any, {} as any, + {} as any, ); const user = new User('user-1'); let remoteWuser: number; @@ -898,6 +900,7 @@ describe('NotificationHandlingState', () => { serverTimeHandler as any, mediaDevicesHandler, {} as any, + {} as any, ); const user = new User('user-1'); let wCall: Wcall; @@ -958,6 +961,7 @@ describe('init AVS state', () => { serverTimeHandler as any, mediaDevicesHandler, {} as any, + {} as any, ); const user = new User('user-1'); beforeEach(() => { diff --git a/apps/webapp/src/script/repositories/calling/CallingRepository.ts b/apps/webapp/src/script/repositories/calling/CallingRepository.ts index c822d47e5d2..6504ca13934 100644 --- a/apps/webapp/src/script/repositories/calling/CallingRepository.ts +++ b/apps/webapp/src/script/repositories/calling/CallingRepository.ts @@ -70,9 +70,11 @@ import {CallingEvent} from 'Repositories/event/CallingEvent'; import {EventRepository} from 'Repositories/event/EventRepository'; import {EventSource} from 'Repositories/event/EventSource'; import {NOTIFICATION_HANDLING_STATE} from 'Repositories/event/NotificationHandlingState'; +import {BackgroundEffectsHandler} from 'Repositories/media/backgroundEffectsHandler'; import type {MediaDevicesHandler} from 'Repositories/media/MediaDevicesHandler'; import type {MediaStreamHandler} from 'Repositories/media/MediaStreamHandler'; import {MediaType} from 'Repositories/media/MediaType'; +import type {BackgroundEffectSelection, BackgroundSource} from 'Repositories/media/VideoBackgroundEffects'; import {TeamState} from 'Repositories/team/TeamState'; import {EventName} from 'Repositories/tracking/EventName'; import * as trackingHelpers from 'Repositories/tracking/Helpers'; @@ -152,7 +154,6 @@ export class CallingRepository { private readonly acceptVersionWarning: (conversationId: QualifiedId) => void; private readonly callLog: string[]; private readonly logger: Logger; - private enableBackgroundBlur = false; private avsVersion: number = 0; private incomingCallCallback: (call: Call) => void; private isReady: boolean = false; @@ -191,6 +192,7 @@ export class CallingRepository { private readonly mediaStreamHandler: MediaStreamHandler, private readonly mediaDevicesHandler: MediaDevicesHandler, private readonly serverTimeHandler: ServerTimeHandler, + private readonly backgroundEffectsHandler: BackgroundEffectsHandler, private readonly apiClient = container.resolve(APIClient), private readonly conversationState = container.resolve(ConversationState), private readonly callState = container.resolve(CallState), @@ -318,20 +320,110 @@ export class CallingRepository { } }; - public async switchVideoBackgroundBlur(enable: boolean): Promise { + /** + * Switches the video background effect for the self participant in the active call. + * + * This method: + * 1. Stores the effect as the preferred background effect + * 2. Applies the effect to the participant's video stream + * 3. Replaces the media source with the processed stream if successfully applied + * + * If no active call exists or no video feed is available, only updates the + * participant's effect observable without applying processing. For 'none' effect, + * switches back to the original video feed. + * + * @param effect - Background effect to apply ('none', 'blur', 'virtual', or 'custom'). + * @param customBackground - Optional custom background source for 'custom' effect type. + * @returns Promise that resolves when the effect switch is complete. + */ + public async switchVideoBackgroundEffect( + effect: BackgroundEffectSelection, + customBackground?: BackgroundSource, + ): Promise { + // persisted chosen background effect, don't care we have call or not + this.backgroundEffectsHandler.setPreferredBackgroundEffect(effect, customBackground); + await this.applyCurrentBackgroundEffectOnSelfParticipant(true); + } + + public allowSuperhighQualityTier(event: boolean) { + if (this.isSuperhighQualityTierAllowed()) { + this.backgroundEffectsHandler.enableSuperhighQualityTier(event); + } + } + + public isSuperhighQualityTierAllowed() { + return this.backgroundEffectsHandler.isSuperhighQualityTierAllowed(); + } + + public getBackgroundEffectsHandler(): BackgroundEffectsHandler { + return this.backgroundEffectsHandler; + } + + private async applyCurrentBackgroundEffectOnSelfParticipant( + changeAvsSendingMediaSource = false, + ): Promise { const activeCall = this.callState.joinedCall(); if (!activeCall) { + // There is no call even there is no self-participant! + this.logger.warn('There is no call even there is no self-participant to apply background effects'); return; } + + // Read self-Participant state to change const selfParticipant = activeCall.getSelfParticipant(); - selfParticipant.releaseBlurredVideoStream(); - const videoFeed = selfParticipant.videoStream(); - if (!videoFeed) { + const hasActiveVideo = selfParticipant.hasActiveVideo(); + const sharesScreen = selfParticipant.sharesScreen(); + + if (sharesScreen) { + this.logger.error('The application allows to apply background effects in case of screen shares'); + return; + } + + // let's check if background should be disabled, then let's do it and go back to the original video + if (!this.backgroundEffectsHandler.isBackgroundEffectEnabled()) { + selfParticipant.releaseProcessedVideoStream(); + if (hasActiveVideo && changeAvsSendingMediaSource) { + // So let's switch back to the original video source + this.logger.info('Disable background effects.'); + this.changeMediaSource(selfParticipant.videoStream(), MediaType.VIDEO, false); + } + return selfParticipant.videoStream(); + } + + if (!hasActiveVideo) { + // no Video nothing to change!! + this.logger.warn('No video exists to apply apply background effects'); return; } - this.enableBackgroundBlur = enable; - const newVideoFeed = enable ? ((await selfParticipant.setBlurredBackground(true)) as MediaStream) : videoFeed; - this.changeMediaSource(newVideoFeed, MediaType.VIDEO, false); + + // Hold a reference to the old stream so we can release it AFTER the new one is assigned, + const previousStream = selfParticipant.processedVideoStream(); + + const {applied, media} = await this.backgroundEffectsHandler.applyBackgroundEffect(selfParticipant.videoStream()); + + // The BackgroundEffectsHandler decide not to change the video stream, so we're going on with the original video. + if (!applied) { + previousStream?.release(); + selfParticipant.processedVideoStream(undefined); + if (changeAvsSendingMediaSource) { + this.logger.info('Background effect could not applied! Switch back to original video stream!'); + this.changeMediaSource(selfParticipant.videoStream(), MediaType.VIDEO, false); + } + return selfParticipant.videoStream(); + } + + // Assign new stream first, then release old. + selfParticipant.processedVideoStream(media); + if (previousStream !== media) { + previousStream?.release(); + } + + // in case we also want to instantly change AVS sending source we do it now, + if (changeAvsSendingMediaSource) { + this.changeMediaSource(media.stream, MediaType.VIDEO, false); + this.logger.info('Background effect applied!'); + } + return media.stream; } getStats(conversationId: QualifiedId) { @@ -599,6 +691,8 @@ export class CallingRepository { } private storeCall(call: Call): void { + // @TODO check why this is needed, backgound effect should be global + // call.getSelfParticipant().backgroundEffect(this.preferredBackgroundEffect); this.callState.calls.push(call); } @@ -620,7 +714,9 @@ export class CallingRepository { const mediaStream = await this.getMediaStream({audio, camera}, call.isGroupOrConference); if (call.state() !== CALL_STATE.NONE) { selfParticipant.updateMediaStream(mediaStream, true); - await selfParticipant.setBlurredBackground(this.enableBackgroundBlur); + if (this.backgroundEffectsHandler.isBackgroundEffectEnabled()) { + await this.applyCurrentBackgroundEffectOnSelfParticipant(); + } if (camera) { call.getSelfParticipant().videoState(VIDEO_STATE.STARTED); } @@ -1854,7 +1950,14 @@ export class CallingRepository { public async refreshVideoInput() { const stream = await this.mediaStreamHandler.requestMediaStream(false, true, false, false); this.stopMediaSource(MediaType.VIDEO); - const clonedMediaStream = this.changeMediaSource(stream, MediaType.VIDEO); + let clonedMediaStream = this.changeMediaSource(stream, MediaType.VIDEO); + const activeCall = this.callState.joinedCall(); + if (activeCall && this.backgroundEffectsHandler.isBackgroundEffectEnabled()) { + const processedStream = await this.applyCurrentBackgroundEffectOnSelfParticipant(true); + if (processedStream) { + clonedMediaStream = processedStream; + } + } return clonedMediaStream; } @@ -2537,7 +2640,11 @@ export class CallingRepository { const mediaStream = await this.getMediaStream(missingStreams, call.isGroupOrConference); this.mediaStreamQuery = undefined; selfParticipant.updateMediaStream(mediaStream, true); - await selfParticipant.setBlurredBackground(this.enableBackgroundBlur); + + if (this.backgroundEffectsHandler.isBackgroundEffectEnabled()) { + await this.applyCurrentBackgroundEffectOnSelfParticipant(); + } + return selfParticipant.getMediaStream(); } catch (error: unknown) { this.mediaStreamQuery = undefined; diff --git a/apps/webapp/src/script/repositories/calling/Participant.ts b/apps/webapp/src/script/repositories/calling/Participant.ts index 8a8f2b4d076..ded25de94ac 100644 --- a/apps/webapp/src/script/repositories/calling/Participant.ts +++ b/apps/webapp/src/script/repositories/calling/Participant.ts @@ -24,7 +24,7 @@ import {VIDEO_STATE} from '@wireapp/avs'; import {AvsDebugger} from '@wireapp/avs-debugger'; import {User} from 'Repositories/entity/User'; -import {applyBlur} from 'Repositories/media/VideoBackgroundBlur'; +import {getLogger, Logger} from 'Util/logger'; import {matchQualifiedIds} from 'Util/qualifiedId'; export type UserId = string; @@ -33,8 +33,11 @@ export type ClientId = string; export class Participant { // Video public readonly videoState = observable(VIDEO_STATE.STOPPED); + // The (self-) participant hold everytime the original video and this will never change. public readonly videoStream = observable(); - public readonly blurredVideoStream = observable<{stream: MediaStream; release: () => void} | undefined>(); + // In case of background changes effected we store this resulting media stream here. + // The CallingRepository will decide if this stream is sent. Because background changes are global app state changes! + public readonly processedVideoStream = observable<{stream: MediaStream; release: () => void} | undefined>(); public readonly hasActiveVideo: ko.PureComputed; public readonly hasPausedVideo: ko.PureComputed; public readonly sharesScreen: ko.PureComputed; @@ -44,6 +47,7 @@ export class Participant { public readonly isActivelySpeaking = observable(false); public readonly isSendingVideo: ko.PureComputed; public readonly isAudioEstablished = observable(false); + private readonly logger: Logger; // Audio public readonly audioStream = observable(); @@ -54,6 +58,7 @@ export class Participant { public readonly user: User, public readonly clientId: ClientId, ) { + this.logger = getLogger('Participant'); this.hasActiveVideo = pureComputed(() => { return (this.sharesCamera() || this.sharesScreen()) && !!this.videoStream(); }); @@ -82,19 +87,10 @@ export class Participant { }); } - public releaseBlurredVideoStream(): void { - this.blurredVideoStream()?.release(); - this.blurredVideoStream(undefined); - } - - public async setBlurredBackground(isBlurred: boolean) { - const originalVideoStream = this.videoStream(); - if (isBlurred && originalVideoStream) { - this.blurredVideoStream(await applyBlur(originalVideoStream)); - } else { - this.releaseBlurredVideoStream(); - } - return this.blurredVideoStream()?.stream; + public releaseProcessedVideoStream(): void { + this.logger.info('Stop the current background effect!'); + this.processedVideoStream()?.release(); + this.processedVideoStream(undefined); } readonly doesMatchIds = (userId: QualifiedId, clientId: ClientId): boolean => @@ -106,7 +102,7 @@ export class Participant { } setVideoStream(videoStream: MediaStream, stopTracks: boolean): void { - this.releaseBlurredVideoStream(); + this.releaseProcessedVideoStream(); this.releaseStream(this.videoStream(), stopTracks); this.videoStream(videoStream); } @@ -124,13 +120,13 @@ export class Participant { getMediaStream(): MediaStream { const audioTracks: MediaStreamTrack[] = this.audioStream()?.getTracks() ?? []; const videoTracks: MediaStreamTrack[] = - this.blurredVideoStream()?.stream.getTracks() ?? this.videoStream()?.getTracks() ?? []; + this.processedVideoStream()?.stream.getTracks() ?? this.videoStream()?.getTracks() ?? []; return new MediaStream(audioTracks.concat(videoTracks)); } releaseVideoStream(stopTracks: boolean): void { this.releaseStream(this.videoStream(), stopTracks); - this.releaseBlurredVideoStream(); + this.releaseProcessedVideoStream(); this.videoStream(undefined); } diff --git a/apps/webapp/src/script/repositories/media/VideoBackgroundBlur.ts b/apps/webapp/src/script/repositories/media/VideoBackgroundBlur.ts deleted file mode 100644 index d4c6095f3ce..00000000000 --- a/apps/webapp/src/script/repositories/media/VideoBackgroundBlur.ts +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Wire - * Copyright (C) 2024 Wire Swiss GmbH - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see http://www.gnu.org/licenses/. - * - */ - -import {ImageSegmenter, FilesetResolver} from '@mediapipe/tasks-vision'; - -import {VideoDimensions, blurBackground, initShaderProgram} from './BackgroundBlurrer'; - -enum SEGMENTATION_MODEL { - // Other models are available and could be tested later on (https://github.com/google-ai-edge/mediapipe/blob/master/docs/solutions/models.md#selfie-segmentation), this one is optimized for performance and yeild rather good results for our MVP - PERFORMANCE = './assets/mediapipe-models/selfie_segmenter.tflite', -} -enum FRAMERATE { - LOW = 30, - HIGH = 60, -} - -const QualitySettings = { - segmentationModel: SEGMENTATION_MODEL.PERFORMANCE, - framerate: FRAMERATE.LOW, -}; - -// Calculate the FPS interval -const fpsInterval = 1000 / QualitySettings.framerate; -let then = Date.now(); -let now = then; -let elapsed = 0; - -let rafId: number; - -// Function to predict the webcam feed processed by the ImageSegmenter -function startBlurProcess( - segmenter: ImageSegmenter, - webGlContext: WebGLRenderingContext, - videoEl: HTMLVideoElement, - videoDimensions: VideoDimensions, -) { - now = Date.now(); - elapsed = now - then; - - // If enough time has elapsed, draw the video frame and segment it - if (elapsed > fpsInterval) { - then = now - (elapsed % fpsInterval); - - const startTimeMs = performance.now(); - - try { - segmenter.segmentForVideo(videoEl, startTimeMs, result => - blurBackground(result, videoEl, webGlContext, videoDimensions), - ); - } catch (error: unknown) { - console.error('Failed to segment video', error); - } - } - rafId = window.requestAnimationFrame(() => startBlurProcess(segmenter, webGlContext, videoEl, videoDimensions)); - return () => { - window.cancelAnimationFrame(rafId); - }; -} - -async function createSegmenter(canvas: HTMLCanvasElement): Promise { - const video = await FilesetResolver.forVisionTasks('/min/mediapipe/wasm'); - return ImageSegmenter.createFromOptions(video, { - baseOptions: { - modelAssetPath: QualitySettings.segmentationModel, - delegate: 'GPU', - }, - canvas, - runningMode: 'VIDEO', - outputCategoryMask: false, - outputConfidenceMasks: true, - }); -} - -/** - * Will create a new MediaStream that will both segment each frame and apply a blur effect to the background. - * @param originalStream the stream that contains the video that needs background blur - * @returns a promise that resolves to an object containing the new MediaStream and a release function to stop the blur process - */ -export async function applyBlur(stream: MediaStream): Promise<{stream: MediaStream; release: () => void}> { - // Create a video element to display the webcam feed - const videoEl = document.createElement('video'); - // Create a canvas element that will be to draw the blurred frames - // Store the video dimensions - const videoDimensions = {width: 0, height: 0}; - - videoEl.srcObject = stream; - videoEl.onloadedmetadata = () => { - // Ensure metadata is loaded to get video dimensions - videoDimensions.width = videoEl.videoWidth || 1240; - videoDimensions.height = videoEl.videoHeight || 720; - videoEl.play().catch((error: unknown) => console.error('Error playing the video: ', error)); - }; - - return new Promise(resolve => { - videoEl.onplay = async () => { - const glContext = document.createElement('canvas'); - glContext.height = videoDimensions.height; - glContext.width = videoDimensions.width; - - const gl = initShaderProgram(glContext, videoDimensions); - const segmenter = await createSegmenter(glContext); - - const stopBlurProcess = startBlurProcess(segmenter, gl, videoEl, videoDimensions); - const videoStream = glContext.captureStream(QualitySettings.framerate).getVideoTracks()[0]; - const blurredMediaStream = new MediaStream([videoStream]); - - resolve({ - stream: blurredMediaStream, - release: () => { - stopBlurProcess(); - stopVideo(videoEl); - segmenter.close(); - }, - }); - }; - }); -} - -function stopVideo(videoEl: HTMLVideoElement) { - // Check if the video element is playing and if so, stop it. - if (!videoEl.paused && !videoEl.ended) { - videoEl.pause(); - videoEl.srcObject = null; // Disconnect the media stream - videoEl.load(); // Reset the video element - } -} diff --git a/apps/webapp/src/script/repositories/media/VideoBackgroundEffects.ts b/apps/webapp/src/script/repositories/media/VideoBackgroundEffects.ts new file mode 100644 index 00000000000..0929e876424 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/VideoBackgroundEffects.ts @@ -0,0 +1,277 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Blur intensity level for background blur effects. + */ +export type BlurLevel = 'low' | 'high'; + +/** + * Discriminated union type representing the selected background effect. + * + * Effect types: + * - 'none': No background effect applied + * - 'blur': Blur effect with specified intensity level + * - 'virtual': Virtual background replacement with a builtin background image + * - 'custom': Custom background (user-provided image/video) + */ +export type BackgroundEffectSelection = + | {type: 'none'} + | {type: 'blur'; level: BlurLevel} + | {type: 'virtual'; backgroundId: string} + | {type: 'custom'}; + +/** + * Background source type for virtual background mode. + * + * Supports both HTMLImageElement (loaded images) and ImageBitmap (processed bitmaps). + */ +export type BackgroundSource = HTMLImageElement | ImageBitmap; + +/** + * Default background effect selection (no effect applied). + */ +export const DEFAULT_BACKGROUND_EFFECT: BackgroundEffectSelection = {type: 'none'}; + +/** + * Blur strength values mapped to blur levels. + * + * Values range from 0.0 (no blur) to 1.0 (maximum blur). These are passed + * directly to the background effects controller's blur strength parameter. + */ +export const BLUR_STRENGTHS: Record = { + low: 0.7, + high: 1.0, +}; + +type BuiltinBackgroundLabelKey = + | 'videoCallBackgroundOffice1' + | 'videoCallBackgroundOffice2' + | 'videoCallBackgroundOffice3' + | 'videoCallBackgroundOffice4' + | 'videoCallBackgroundOffice5' + | 'videoCallBackgroundWire1'; + +/** + * Base definition for builtin background images. + * + * Contains metadata for a predefined background option available in the UI. + */ +type BuiltinBackgroundDefinition = { + /** Unique identifier for the background. */ + id: string; + /** Localization key for the background display name. */ + labelKey: BuiltinBackgroundLabelKey; + /** URL path to the background image file. */ + imageUrl: string; + /** Color palette used for gradient fallback if image fails to load. */ + previewColors: string[]; +}; + +/** + * Complete builtin background definition with computed preview gradient. + * + * Extends BuiltinBackgroundDefinition with a CSS gradient string generated + * from the preview colors for use in UI preview tiles. + */ +export type BuiltinBackground = BuiltinBackgroundDefinition & { + /** CSS linear gradient string generated from previewColors. */ + previewGradient: string; +}; + +/** + * Builds a CSS linear gradient string from an array of color values. + * + * Creates a 135-degree diagonal gradient with evenly spaced color stops. + * + * @param colors - Array of CSS color strings (hex, rgb, named colors, etc.). + * @returns CSS linear-gradient() function string. + */ +const buildGradient = (colors: string[]) => `linear-gradient(135deg, ${colors.join(', ')})`; +export const DEFAULT_BUILTIN_BACKGROUND_ID = 'wire-1'; +const BUILTIN_BACKGROUND_DEFINITIONS: BuiltinBackgroundDefinition[] = [ + { + id: DEFAULT_BUILTIN_BACKGROUND_ID, + labelKey: 'videoCallBackgroundWire1', + imageUrl: '/assets/images/backgrounds/wire-1.png', + previewColors: ['#1a1a1a', '#2d2d2d', '#4a4a4a'], + }, + { + id: 'office-1', + labelKey: 'videoCallBackgroundOffice1', + imageUrl: '/assets/images/backgrounds/office-1.png', + previewColors: ['#4a5568', '#718096', '#cbd5e0'], + }, + { + id: 'office-2', + labelKey: 'videoCallBackgroundOffice2', + imageUrl: '/assets/images/backgrounds/office-2.png', + previewColors: ['#2d3748', '#4a5568', '#718096'], + }, +]; + +export const BUILTIN_BACKGROUNDS: BuiltinBackground[] = BUILTIN_BACKGROUND_DEFINITIONS.map(definition => ({ + ...definition, + previewGradient: buildGradient(definition.previewColors), +})); + +/** Maximum number of cached background images before evicting oldest entries. */ +const MAX_BACKGROUND_CACHE_ENTRIES = 8; +/** LRU cache for loaded background images, keyed by background ID. */ +const backgroundImageCache = new Map(); + +/** + * Retrieves a cached background image and updates its position in the LRU cache. + * + * Implements LRU (Least Recently Used) cache behavior by moving the accessed + * entry to the end of the map (most recently used position). + * + * @param backgroundId - Unique identifier for the background to retrieve. + * @returns Cached HTMLImageElement if found, undefined otherwise. + */ +const getCachedImage = (backgroundId: string): HTMLImageElement | undefined => { + const cached = backgroundImageCache.get(backgroundId); + if (!cached) { + return undefined; + } + backgroundImageCache.delete(backgroundId); + backgroundImageCache.set(backgroundId, cached); + return cached; +}; + +/** + * Stores a background image in the cache with LRU eviction policy. + * + * If the cache exceeds MAX_BACKGROUND_CACHE_ENTRIES, the oldest entry + * (first key in iteration order) is evicted to make room. + * + * @param backgroundId - Unique identifier for the background. + * @param image - HTMLImageElement to cache. + */ +const setCachedImage = (backgroundId: string, image: HTMLImageElement): void => { + if (backgroundImageCache.has(backgroundId)) { + backgroundImageCache.delete(backgroundId); + } + backgroundImageCache.set(backgroundId, image); + if (backgroundImageCache.size > MAX_BACKGROUND_CACHE_ENTRIES) { + const oldestKey = backgroundImageCache.keys().next().value; + if (oldestKey) { + backgroundImageCache.delete(oldestKey); + } + } +}; + +/** + * Loads an image from a URL with CORS support. + * + * Creates a new HTMLImageElement, sets crossOrigin to 'anonymous' for CORS, + * and returns a Promise that resolves when the image loads successfully. + * + * @param src - URL path to the image file. + * @returns Promise resolving to the loaded HTMLImageElement. + * @throws Error if the image fails to load. + */ +const loadImage = (src: string): Promise => + new Promise((resolve, reject) => { + const image = new Image(); + image.crossOrigin = 'anonymous'; + image.onload = () => resolve(image); + image.onerror = () => reject(new Error(`Failed to load background image: ${src}`)); + image.src = src; + }); + +/** + * Creates an ImageBitmap from a linear gradient defined by color stops. + * + * Generates a 1920x1080 bitmap with a diagonal linear gradient using the + * provided colors. Uses OffscreenCanvas when available (Web Worker context), + * otherwise falls back to HTMLCanvasElement (main thread). + * + * @param colors - Array of CSS color strings for gradient stops. + * @returns Promise resolving to an ImageBitmap of the gradient. + * @throws Error if 2D context creation fails. + */ +const createGradientBitmap = async (colors: string[]): Promise => { + const width = 1920; + const height = 1080; + const canvas = + typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(width, height) : document.createElement('canvas'); + const isOffscreenCanvas = typeof OffscreenCanvas !== 'undefined' && canvas instanceof OffscreenCanvas; + if (!isOffscreenCanvas) { + (canvas as HTMLCanvasElement).width = width; + (canvas as HTMLCanvasElement).height = height; + } + const ctx = canvas.getContext('2d'); + if (!ctx) { + throw new Error('Failed to create 2D context for background gradient.'); + } + const gradient = ctx.createLinearGradient(0, 0, width, height); + const stops = Math.max(colors.length - 1, 1); + colors.forEach((color, index) => { + gradient.addColorStop(index / stops, color); + }); + ctx.fillStyle = gradient; + ctx.fillRect(0, 0, width, height); + return createImageBitmap(canvas as OffscreenCanvas | HTMLCanvasElement); +}; + +/** + * Retrieves a builtin background definition by ID. + * + * Searches the BUILTIN_BACKGROUNDS array for a background matching the + * provided ID. Returns undefined if no match is found. + * + * @param backgroundId - Unique identifier for the background to find. + * @returns BuiltinBackground object if found, undefined otherwise. + */ +export const getBuiltinBackground = (backgroundId: string): BuiltinBackground | undefined => + BUILTIN_BACKGROUNDS.find(background => background.id === backgroundId); + +/** + * Loads a background source for virtual background mode. + * + * This function: + * 1. Looks up the builtin background definition by ID + * 2. Checks the LRU cache for a previously loaded image + * 3. If cached, returns the cached image + * 4. If not cached, attempts to load the image from the URL + * 5. If image load fails, falls back to a gradient bitmap generated from preview colors + * 6. Caches successful image loads for future use + * + * @param backgroundId - Unique identifier for the builtin background to load. + * @returns Promise resolving to HTMLImageElement (loaded image) or ImageBitmap (gradient fallback). + * @throws Error if the backgroundId is unknown (not found in BUILTIN_BACKGROUNDS). + */ +export const loadBackgroundSource = async (backgroundId: string): Promise => { + const background = getBuiltinBackground(backgroundId); + if (!background) { + throw new Error(`Unknown background id: ${backgroundId}`); + } + const cachedImage = getCachedImage(backgroundId); + if (cachedImage) { + return cachedImage; + } + try { + const image = await loadImage(background.imageUrl); + setCachedImage(backgroundId, image); + return image; + } catch (_error) { + return createGradientBitmap(background.previewColors); + } +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/README.md b/apps/webapp/src/script/repositories/media/backgroundEffects/README.md new file mode 100644 index 00000000000..c8861935b86 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/README.md @@ -0,0 +1,332 @@ +# Background Effects + +## 🚧 Limited Release - Real-World Testing Phase + +**Status:** Production-ready code, limited to **Edge and Internal releases only** for real-world validation. + +### Why Limited Release? + +Several components need real-world testing across diverse hardware before general availability: + +- **Capability Detection & BackgroundEffectsRenderingPipeline Selection** - Validate automatic pipeline selection ( + worker-webgl2, main-webgl2, + canvas2d) across different device configurations +- **Quality Controller** - Test adaptive quality tier system (A/B/C/D) under varying CPU/GPU load scenarios on different + hardware types +- **Performance Characteristics** - Gather data on CPU/GPU utilization patterns across device ranges + +### Known Areas for Future Improvement + +- **Segmentation Quality** - Edge detection, matte refinement, and temporal stability can be enhanced +- **Overlay Quality** - Blending, color matching, and edge handling improvements + +### Next Steps + +Collect performance metrics and user feedback from limited release to validate behavior before expanding to broader +rollout. + +--- + +Production-grade background blur and virtual background pipeline that avoids WebRTC Insertable Streams. It processes the +original camera track, renders to a canvas (via WebGL2 or Canvas2D), and exposes a processed track using +`canvas.captureStream()`. + +## Features + +- **Multi-pipeline architecture**: Worker + OffscreenCanvas + WebGL2 (preferred), main-thread WebGL2, Canvas2D fallback, + and passthrough +- **ML-based segmentation**: Low-res MediaPipe Selfie Segmentation for person/background separation +- **Advanced post-processing**: Joint bilateral smoothing, temporal stabilization, and GPU compositing +- **Adaptive quality control**: Automatic quality tier adjustment (A-D) based on performance metrics +- **Backpressure management**: Prevents unbounded frame queues with single-frame-in-flight design +- **Debug visualization**: Mask overlay, mask-only, edge-only, and class modes for inspection +- **Runtime controls**: Change mode, quality, blur strength, and background sources without restarting + +## Quick Start + +```typescript +import { BackgroundEffectsController } from 'Repositories/media/backgroundEffects'; + +// Create controller +const controller = new BackgroundEffectsController(); + +// Get camera stream +const stream = await navigator.mediaDevices.getUserMedia({ + video: { width: 1280, height: 720 }, +}); +const inputTrack = stream.getVideoTracks()[0]; + +// Start pipeline +const { outputTrack, stop } = await controller.start(inputTrack, { + mode: 'blur', + blurStrength: 0.6, + quality: 'auto', + targetFps: 30, +}); + +// Use output track with WebRTC +const pc = new RTCPeerConnection(); +pc.addTrack(outputTrack, new MediaStream([outputTrack])); + +// Runtime controls +controller.setMode('virtual'); +controller.setBackgroundSource(document.querySelector('#bgImage') as HTMLImageElement); +controller.setDebugMode('maskOverlay'); +controller.setBlurStrength(0.3); +controller.setQuality('A'); + +// Cleanup +stop(); +``` + +## API Reference + +### BackgroundEffectsController + +Main controller class that orchestrates the entire background effects pipeline. + +#### Methods + +**`start(inputTrack: MediaStreamTrack, opts?: StartOptions): Promise<{outputTrack: MediaStreamTrack; stop: () => void}>` +** + +Starts the background effects pipeline. Detects browser capabilities, selects optimal pipeline, initializes components, +and begins frame processing. + +- `inputTrack`: Input video track (e.g., from `getUserMedia`) +- `opts`: Configuration options (all optional with defaults) +- Returns: Promise resolving to output track and stop function + +**`setMode(mode: EffectMode): void`** + +Changes the effect mode at runtime. + +- `mode`: `'blur'` | `'virtual'` | `'passthrough'` + +**`setBlurStrength(value: number): void`** + +Sets blur strength for blur effect mode. Value is clamped to [0, 1]. + +- `value`: Blur strength (0 = no blur, 1 = maximum blur) + +**`setBackgroundSource(source: HTMLImageElement | HTMLVideoElement | ImageBitmap): void`** + +Sets the background source for virtual background mode. + +- `source`: Image element, video element, or ImageBitmap +- For images: Converted to ImageBitmap and transferred once +- For videos: Sampled at ~15fps and converted to ImageBitmap frames + +**`setDebugMode(mode: DebugMode): void`** + +Sets debug visualization mode for inspecting segmentation masks. + +- `mode`: `'off'` | `'maskOverlay'` | `'maskOnly'` | `'edgeOnly'` | `'classOverlay'` | `'classOnly'` + +**`setQuality(mode: QualityMode): void`** + +Sets quality mode. 'auto' enables adaptive quality based on performance metrics. + +- `mode`: `'auto'` | `'A'` | `'B'` | `'C'` | `'D'` + +**`stop(): void`** + +Stops the pipeline and cleans up all resources. Should be called when the pipeline is no longer needed. + +### StartOptions + +Configuration options for `start()` method: + +```typescript +interface StartOptions { + targetFps?: number; // Default: 30 + quality?: QualityMode; // Default: 'auto' + qualityPolicy?: + | 'auto' + | 'conservative' + | 'aggressive' + | ((capabilities) => { + initialTier: 'A' | 'B' | 'C' | 'D'; + segmentationModelByTier?: Partial>; + }); + debugMode?: DebugMode; // Default: 'off' + mode?: EffectMode; // Default: 'blur' + blurStrength?: number; // Default: 0.5 (0-1) + backgroundImage?: HTMLImageElement | ImageBitmap; + backgroundVideo?: HTMLVideoElement; + backgroundColor?: string; + segmentationModelPath?: string; // Optional override for all tiers + segmentationModelByTier?: Partial>; + useWorker?: boolean; // Default: true + pipelineOverride?: PipelineType; + onMetrics?: (metrics: Metrics) => void; +} +``` + +### Utility Functions + +**`detectCapabilities(): CapabilityInfo`** + +Detects browser capabilities required for background effects. Returns boolean flags for: + +- `webgl2`: WebGL2 support +- `worker`: Web Worker support +- `offscreenCanvas`: OffscreenCanvas support +- `requestVideoFrameCallback`: RequestVideoFrameCallback API support + +**`choosePipeline(cap: CapabilityInfo, preferWorker?: boolean): BackgroundEffectsRenderingPipeline`** + +Selects the optimal rendering pipeline based on browser capabilities. + +- `cap`: Capability information from `detectCapabilities()` +- `preferWorker`: If true, prefers worker-based pipeline when available +- Returns: `'worker-webgl2'` | `'main-webgl2'` | `'canvas2d'` | `'passthrough'` + +### Types + +- `EffectMode`: `'blur'` | `'virtual'` | `'passthrough'` +- `DebugMode`: `'off'` | `'maskOverlay'` | `'maskOnly'` | `'edgeOnly'` | `'classOverlay'` | `'classOnly'` +- `QualityMode`: `'auto'` | `'A'` | `'B'` | `'C'` | `'D'` +- `Metrics`: Performance metrics tracked during frame processing + +## Architecture + +### BackgroundEffectsRenderingPipeline Selection + +The module automatically selects the best available pipeline based on browser capabilities: + +1. **worker-webgl2** (preferred): Worker + OffscreenCanvas + WebGL2 + +- Best performance (background thread processing) +- Requires: Worker, OffscreenCanvas, WebGL2 + +2. **main-webgl2**: Main-thread WebGL2 + +- High quality (GPU-accelerated) +- Requires: WebGL2 + +3. **canvas2d**: Canvas2D compositing + +- Fallback (CPU-based, widely supported) +- Lower visual quality than WebGL2 + +4. **passthrough**: No processing + +- Last resort when no other pipeline is available + +### Processing BackgroundEffectsRenderingPipeline + +1. **Frame extraction**: `VideoSource` extracts frames using `requestVideoFrameCallback` (preferred) or + `requestAnimationFrame` (fallback) +2. **Segmentation**: MediaPipe Selfie Segmentation generates low-res mask (256x256, 256x144, or 160x96) +3. **Mask refinement**: WebGL pipelines apply joint bilateral filter + temporal smoothing + upsampling +4. **Compositing**: GPU-accelerated blur or virtual background replacement (WebGL) or CPU compositing (Canvas2D) +5. **Output**: Rendered to canvas, exposed via `canvas.captureStream()` + +### Post-Processing Today + +**WebGL pipelines (worker-webgl2 / main-webgl2)** + +- Mask upsample (linear sampling) from low-res segmentation to refined resolution +- Joint bilateral filtering using the video frame as guidance (edge-preserving smoothing) +- Temporal smoothing (EMA) using the previous refined mask +- Separable Gaussian blur (downsample + horizontal + vertical passes) +- Matte thresholds + soft edges for blur/virtual compositing + +**Canvas2D pipeline** + +- CPU compositing with blur filter and mask-based alpha +- Temporal smoothing for masks (EMA) and mask reuse between segmentation frames +- No bilateral refinement (kept lightweight for low-end devices) + +### Quality Tiers + +Quality tiers balance visual quality against performance: + +- **Tier A**: 256x256 segmentation, cadence 1, blur 1/2 res with radius 4 +- **Tier B**: 256x144 segmentation, cadence 2, blur 1/2 res with radius 3 +- **Tier C**: 160x96 segmentation, cadence 3, blur 1/4 res with radius 2 +- **Tier D**: Bypass (no processing) + +Use `setQuality('A' | 'B' | 'C' | 'D')` to force a tier, or `setQuality('auto')` to let the controller adapt based on +performance metrics. + +### Debug Modes + +- `maskOverlay`: Overlays green tint on mask areas +- `maskOnly`: Displays only the segmentation mask as grayscale +- `edgeOnly`: Highlights mask edges using edge detection +- `classOverlay`: Overlays class colors from multiclass segmentation +- `classOnly`: Shows class colors only (no video) + +## Module Structure + +``` +BackgroundEffects/ +├── effects/ +│ ├── backgroundEffectsController.ts # Main controller +│ ├── frameSource.ts # Frame extraction adapter +│ ├── videoSource.ts # Frame extraction +│ └── capability.ts # Capability detection +├── pipelines/ # BackgroundEffectsRenderingPipeline implementations +├── renderer/ +│ └── webGlRenderer.ts # WebGL2 rendering pipeline +├── segmentation/ +│ └── segmenter.ts # MediaPipe segmentation +├── quality/ +│ └── qualityController.ts # Adaptive quality control +├── worker/ +│ └── bgfx.worker.ts # Worker-based pipeline +├── shaders/ # GLSL shaders +├── debug/ +│ └── debugModes.ts # Debug mode utilities +├── shared/ # Shared helpers (mask, timestamps) +├── testBasic.ts # Basic pipeline test harness +├── backgroundEffectsWorkerTypes.ts # Type definitions +└── index.ts # Public API exports +``` + +## Implementation Details + +### Frame Transfer + +- **Frame source**: `FrameSource` produces `ImageBitmap` frames using `MediaStreamTrackProcessor` or a `VideoSource` + fallback. +- **Worker pipeline**: Transfers `ImageBitmap` frames to the worker; if a frame is in flight, the next frame is dropped. +- **Main pipeline**: Frames are processed directly on the main thread. + +### Background Sources + +- **Images**: Converted to ImageBitmap and transferred once (worker) or stored (main) +- **Videos**: Sampled at ~15fps and converted to ImageBitmap frames + +### Resource Management + +- Background sources (ImageBitmaps) are properly closed when replaced or stopped +- Worker is terminated on stop +- Renderer and segmenter are destroyed on stop +- Video source and output track are stopped on stop + +## Dependencies + +- Tier A defaults to the selfie segmentation model: `/assets/mediapipe-models/selfie_segmenter_landscape.tflite` +- Tier B/C/D use MediaPipe selfie segmentation: `/assets/mediapipe-models/selfie_segmenter_landscape.tflite` +- Multiclass segmentation is optional and can be provided via `segmentationModelByTier` or config override. +- MediaPipe WASM: `/min/mediapipe/wasm` + +## Notes + +- The Canvas2D fallback honors `mode`, `debugMode`, `backgroundSource`, and `blurStrength`, but visual quality is lower + than WebGL2. +- Passthrough mode is used when no other pipeline is available or when explicitly selected. +- The module uses MediaPipe assets from `/assets/mediapipe-models/selfie_segmenter_landscape.tflite` and + `/min/mediapipe/wasm`. Optional multiclass assets are not bundled by default. +- All runtime controls (`setMode`, `setBlurStrength`, etc.) work with both worker and main pipelines. + +## Future Exploration + +- Higher quality mask refinement (guided filter, morphological ops, or ML post-processing) +- Improved edge handling (halo suppression, adaptive matte thresholds, temporal stability) +- Color matching/relighting for virtual backgrounds +- Class-aware compositing (multi-class segmentation use cases) +- Dynamic post-processing knobs per quality tier and device class diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/backgroundEffectsWorkerTypes.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/backgroundEffectsWorkerTypes.ts new file mode 100644 index 00000000000..946e6dabc13 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/backgroundEffectsWorkerTypes.ts @@ -0,0 +1,432 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Type definitions and interfaces for the background effects system. + * + * This module exports all shared types used across the background effects pipeline, + * including effect modes, quality settings, worker messages, and configuration options. + */ + +/** + * Effect mode for background processing. + * - 'blur': Applies blur effect to the background + * - 'virtual': Replaces background with an image or video + * - 'passthrough': No processing, passes through original video + */ +export type EffectMode = 'blur' | 'virtual' | 'passthrough'; + +/** + * Processing mode (excludes passthrough, as passthrough doesn't require processing). + * Used by quality controller and renderer to determine processing parameters. + */ +export type Mode = Exclude; + +/** + * Debug visualization mode for inspecting segmentation masks. + * - 'off': Normal rendering with effects applied + * - 'maskOverlay': Overlays green tint on mask areas + * - 'maskOnly': Displays only the segmentation mask as grayscale + * - 'edgeOnly': Highlights mask edges using edge detection + * - 'classOverlay': Overlays class colors from multiclass segmentation + * - 'classOnly': Shows class colors only (no video) + */ +export type DebugMode = 'off' | 'maskOverlay' | 'maskOnly' | 'edgeOnly' | 'classOverlay' | 'classOnly'; + +export type QualityTier = 'superhigh' | 'high' | 'medium' | 'low' | 'bypass'; + +/** + * Quality mode for rendering performance. + * - 'auto': Adaptive quality based on performance metrics + * - QualityTier + */ +export type QualityMode = 'auto' | QualityTier; + +/** + * Optional mapping of quality tiers to segmentation model paths. + */ +export type SegmentationModelByTier = Partial>; + +/** + * Browser capability information detected at runtime. + */ +export interface CapabilityInfo { + offscreenCanvas: boolean; + worker: boolean; + webgl2: boolean; + requestVideoFrameCallback: boolean; +} + +export type QualityPolicyMode = 'auto' | 'conservative' | 'aggressive'; + +export interface QualityPolicyResult { + initialTier: QualityTier; + segmentationModelByTier?: SegmentationModelByTier; +} + +export type QualityPolicyResolver = (capabilities: CapabilityInfo) => QualityPolicyResult; + +/** + * Rendering pipeline selection. + * - 'worker-webgl2': OffscreenCanvas + WebGL2 in a Worker + * - 'main-webgl2': WebGL2 on the main thread + * - 'canvas2d': Canvas2D compositing fallback + * - 'passthrough': No processing + */ +export type PipelineType = 'worker-webgl2' | 'main-webgl2' | 'canvas2d' | 'passthrough'; + +/** + * Quality tier parameters that control rendering performance and visual quality. + * + * These parameters are generated by QualityController and used by WebGlRenderer + * to configure the rendering pipeline. They balance visual quality against + * performance by adjusting segmentation resolution, processing cadence, and + * post-processing effects. + */ +export interface QualityTierParams { + /** Quality tier identifier */ + tier: QualityTier; + /** Segmentation mask width in pixels. Lower values reduce CPU/ML cost. */ + segmentationWidth: number; + /** Segmentation mask height in pixels. Lower values reduce CPU/ML cost. */ + segmentationHeight: number; + /** Segmentation cadence (process every Nth frame). Higher values reduce CPU/ML cost. */ + segmentationCadence: number; + /** Scale factor for mask refinement pass (0-1). Lower values reduce GPU cost. */ + maskRefineScale: number; + /** Scale factor for blur downsampling (0-1). Lower values reduce GPU cost. */ + blurDownsampleScale: number; + /** Blur radius in pixels. Lower values reduce GPU cost. */ + blurRadius: number; + /** Joint bilateral filter radius in pixels. Lower values reduce GPU cost. */ + bilateralRadius: number; + /** Spatial sigma for joint bilateral filter. Controls spatial smoothing. */ + bilateralSpatialSigma: number; + /** Range sigma for joint bilateral filter. Controls edge preservation. */ + bilateralRangeSigma: number; + /** Lower threshold for soft matte edge (0-1). Controls where soft edges begin. */ + softLow: number; + /** Upper threshold for soft matte edge (0-1). Controls where soft edges end. */ + softHigh: number; + /** Lower threshold for matte cutoff (0-1). Pixels below this are considered background. */ + matteLow: number; + /** Upper threshold for matte cutoff (0-1). Pixels above this are considered foreground. */ + matteHigh: number; + /** Hysteresis value for matte thresholds to prevent flickering (0-1). */ + matteHysteresis: number; + /** Temporal smoothing alpha (0-1). Higher values increase temporal stability. */ + temporalAlpha: number; + /** If true, bypass all processing and pass through original frames. */ + bypass: boolean; +} + +/** + * Background source for virtual background mode (static image). + * + * Used internally by BackgroundEffectsController to store and manage + * background images for virtual background replacement. + */ +export interface BackgroundSourceImage { + /** Discriminator for type narrowing. */ + type: 'image'; + /** Image bitmap data (transferred to worker if using worker pipeline). */ + bitmap: ImageBitmap; + /** Image width in pixels. */ + width: number; + /** Image height in pixels. */ + height: number; +} + +/** + * Background source for virtual background mode (video frame). + * + * Used internally by BackgroundEffectsController to store and manage + * background video frames for virtual background replacement. + */ +export interface BackgroundSourceVideoFrame { + /** Discriminator for type narrowing. */ + type: 'video'; + /** Video frame bitmap data (transferred to worker if using worker pipeline). */ + bitmap: ImageBitmap; + /** Frame width in pixels. */ + width: number; + /** Frame height in pixels. */ + height: number; +} + +/** + * Performance metrics tracked during frame processing. + * + * These metrics are collected by the worker/main thread and sent to the + * main thread for monitoring and adaptive quality control. Averages are + * computed over a rolling window of recent frames. + * + * **Important**: These metrics measure **time allocation** (wall-clock time + * spent in each phase), not actual hardware CPU/GPU utilization. The percentages + * indicate what portion of total processing time was spent in each phase. + * + * - `avgSegmentationMs`: Time spent in ML segmentation (may run on CPU or GPU + * depending on delegate type, see `segmentationDelegate`) + * - `avgGpuMs`: Time spent in WebGL rendering operations (runs on GPU) + */ +export interface Metrics { + /** Average total frame processing time in milliseconds (segmentation + WebGL rendering). */ + avgTotalMs: number; + /** Average ML segmentation time in milliseconds. */ + avgSegmentationMs: number; + /** Average WebGL rendering time in milliseconds. */ + avgGpuMs: number; + /** Segmentation delegate type ('CPU' or 'GPU') indicating where segmentation runs. */ + segmentationDelegate: 'CPU' | 'GPU' | null; + /** Number of frames dropped due to processing errors or timeouts. */ + droppedFrames: number; + /** Current quality tier */ + tier: QualityTier; +} + +/** + * Configuration options for starting the background effects pipeline. + * + * All options are optional and have sensible defaults. Used when calling + * BackgroundEffectsController.start() to configure the processing pipeline. + */ +export interface StartOptions { + /** Target frames per second for adaptive quality control. Default: 30. */ + targetFps?: number; + /** Quality mode Default: 'auto'. */ + quality?: QualityMode; + /** Debug visualization mode. Default: 'off'. */ + debugMode?: DebugMode; + /** Effect mode ('blur', 'virtual', or 'passthrough'). Default: 'blur'. */ + mode?: EffectMode; + /** Blur strength (0-1) for blur effect mode. Default: 0.5. */ + blurStrength?: number; + /** Background image for virtual background mode (HTMLImageElement or ImageBitmap). */ + backgroundImage?: HTMLImageElement | ImageBitmap; + /** Background video element for virtual background mode. */ + backgroundVideo?: HTMLVideoElement; + /** Solid background color for virtual background mode (CSS color string). */ + backgroundColor?: string; + /** Path to MediaPipe segmentation model file. */ + segmentationModelPath?: string; + /** Per-tier segmentation model overrides (e.g., Tier A uses multiclass). */ + segmentationModelByTier?: SegmentationModelByTier; + /** Optional device capability policy for tier/model selection. */ + qualityPolicy?: QualityPolicyMode | QualityPolicyResolver; + /** Whether to prefer worker-based pipeline when available. Default: true. */ + useWorker?: boolean; + /** Override pipeline selection (primarily for testing). */ + pipelineOverride?: PipelineType; + /** Optional callback to receive performance metrics updates. */ + onMetrics?: (metrics: Metrics) => void; + /** Optional callback to receive model name updates. */ + onModelChange?: (model: string) => void; +} + +/** + * Worker initialization message sent from main thread to worker. + * + * Transfers OffscreenCanvas control to the worker and provides initial + * configuration. The canvas is transferred (not cloned) for performance. + */ +export interface WorkerInitMessage { + /** Message type discriminator. */ + type: 'init'; + /** OffscreenCanvas for WebGL rendering (transferred, not cloned). */ + canvas: OffscreenCanvas; + /** Initial canvas width in pixels. */ + width: number; + /** Initial canvas height in pixels. */ + height: number; + /** Device pixel ratio for high-DPI displays. */ + devicePixelRatio: number; + /** Required worker options (all fields must be provided). */ + options: Required; +} + +/** + * Worker frame message sent from main thread to worker. + * + * Transfers a video frame (ImageBitmap) to the worker for processing. + * The frame is transferred (not cloned) for performance. The worker + * responds with 'frameProcessed' when done. + */ +export interface WorkerFrameMessage { + /** Message type discriminator. */ + type: 'frame'; + /** Video frame as ImageBitmap (transferred, not cloned). */ + frame: ImageBitmap; + /** Frame timestamp in seconds (from video source). */ + timestamp: number; + /** Frame width in pixels. */ + width: number; + /** Frame height in pixels. */ + height: number; +} + +/** + * Worker update message for runtime configuration changes. + * + * Used to update worker state without reinitializing. Different message + * types require different optional fields based on the update type. + */ +export interface WorkerUpdateMessage { + /** Message type discriminator. Determines which optional fields are used. */ + type: + | 'setMode' + | 'setBlurStrength' + | 'setDebugMode' + | 'setQuality' + | 'setBackgroundImage' + | 'setBackgroundVideo' + | 'setDroppedFrames' + | 'stop'; + /** New effect mode (for 'setMode'). */ + mode?: EffectMode; + /** New blur strength 0-1 (for 'setBlurStrength'). */ + blurStrength?: number; + /** New debug mode (for 'setDebugMode'). */ + debugMode?: DebugMode; + /** New quality mode (for 'setQuality'). */ + quality?: QualityMode; + /** Background image bitmap (for 'setBackgroundImage', transferred). */ + image?: ImageBitmap; + /** Background video frame bitmap (for 'setBackgroundVideo', transferred). */ + video?: ImageBitmap; + /** Width in pixels (for 'setBackgroundImage'/'setBackgroundVideo'). */ + width?: number; + /** Height in pixels (for 'setBackgroundImage'/'setBackgroundVideo'). */ + height?: number; + /** Dropped frame count (for 'setDroppedFrames'). */ + droppedFrames?: number; +} + +/** + * Union type of all messages sent from main thread to worker. + * + * Used for type-safe message handling in the worker's onmessage handler. + */ +export type WorkerMessage = WorkerInitMessage | WorkerFrameMessage | WorkerUpdateMessage; + +/** + * Worker metrics message sent from worker to main thread. + * + * Sent periodically (after each frame) to provide performance metrics + * for monitoring and adaptive quality control. + */ +export interface WorkerMetricsMessage { + /** Message type discriminator. */ + type: 'metrics'; + /** Current performance metrics. */ + metrics: Metrics; +} + +/** + * Worker ready message sent from worker to main thread. + * + * Sent once after initialization completes, indicating the worker is + * ready to process frames. + */ +export interface WorkerReadyMessage { + /** Message type discriminator. */ + type: 'ready'; +} + +/** + * Worker frame processed message sent from worker to main thread. + * + * Sent after each frame is processed, used for backpressure control + * to prevent unbounded frame queues. + */ +export interface WorkerFrameProcessedMessage { + /** Message type discriminator. */ + type: 'frameProcessed'; +} + +/** + * Worker error message sent from worker to main thread. + * + * Sent when segmenter initialization fails. The worker continues in + * bypass mode (no segmentation) but notifies the main thread. + */ +export interface WorkerSegmentErrorMessage { + /** Message type discriminator. */ + type: 'segmenterError'; + /** Error message describing the failure. */ + error: string; +} + +/** + * Worker context lost message sent from worker to main thread. + * + * Sent when the worker's WebGL context is lost so the main thread can + * decide how to recover (e.g., fallback to passthrough). + */ +export interface WorkerContextLostMessage { + /** Message type discriminator. */ + type: 'contextLost'; +} + +export interface WorkerErrorMessage { + /** Message type system error. */ + type: 'workerError'; + message: string; + reason: string; + filename?: string; + lineno?: number; + colno?: number; +} + +/** + * Union type of all messages sent from worker to main thread. + * + * Used for type-safe message handling in the main thread's worker.onmessage handler. + */ +export type WorkerResponse = + | WorkerMetricsMessage + | WorkerReadyMessage + | WorkerFrameProcessedMessage + | WorkerSegmentErrorMessage + | WorkerErrorMessage + | WorkerContextLostMessage; + +/** + * Required worker options for initialization. + * + * All fields are required (unlike StartOptions which has defaults). + * Used internally when transferring options to the worker thread. + */ +export interface WorkerOptions { + /** Effect mode ('blur', 'virtual', or 'passthrough'). */ + mode: EffectMode; + /** Debug visualization mode. */ + debugMode: DebugMode; + /** Quality mode ('auto' or fixed tier). */ + quality: QualityMode; + /** Blur strength (0-1) for blur effect mode. */ + blurStrength: number; + /** Path to MediaPipe segmentation model file. */ + segmentationModelPath: string; + /** Per-tier segmentation model overrides (resolved in worker). */ + segmentationModelByTier: SegmentationModelByTier; + /** Initial tier used when quality is set to auto. */ + initialTier: QualityTier; + /** Target frames per second for adaptive quality control. */ + targetFps: number; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/debug/debugModes.test.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/debug/debugModes.test.ts new file mode 100644 index 00000000000..afe669c28e0 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/debug/debugModes.test.ts @@ -0,0 +1,77 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {isDebugMode, DebugModeValues} from './debugModes'; + +describe('DebugModes', () => { + describe('DebugModeValues', () => { + it('contains all valid debug mode values', () => { + expect(DebugModeValues).toEqual( + expect.arrayContaining(['off', 'maskOverlay', 'maskOnly', 'edgeOnly', 'classOverlay', 'classOnly']), + ); + }); + + it('has exactly 6 debug mode values', () => { + expect(DebugModeValues.length).toBe(6); + }); + }); + + describe('isDebugMode', () => { + it('returns true for all valid modes', () => { + DebugModeValues.forEach(mode => { + expect(isDebugMode(mode)).toBe(true); + }); + }); + + it.each(['', 'invalid', 'mask', 'OFF', ' off '])('returns false for %p', value => { + expect(isDebugMode(value)).toBe(false); + }); + + it('works as a type guard in TypeScript', () => { + const testValue: string = 'maskOverlay'; + + if (isDebugMode(testValue)) { + // TypeScript should narrow the type to DebugMode here + const debugMode: typeof testValue = testValue; + expect(debugMode).toBe('maskOverlay'); + // Verify it's one of the valid values + expect(DebugModeValues).toContain(debugMode); + } else { + fail('Type guard should have returned true for valid debug mode'); + } + }); + + it('works as a type guard for invalid values', () => { + const testValue: string = 'invalid'; + + if (isDebugMode(testValue)) { + fail('Type guard should have returned false for invalid debug mode'); + } else { + // TypeScript should keep the type as string here + const stringValue: string = testValue; + expect(stringValue).toBe('invalid'); + } + }); + + it.each([null, undefined, 0, true, {}, []])('returns false for non-string %p', value => { + // @ts-expect-error - testing runtime behavior with invalid input + expect(isDebugMode(value)).toBe(false); + }); + }); +}); diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/debug/debugModes.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/debug/debugModes.ts new file mode 100644 index 00000000000..35e630f1e3c --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/debug/debugModes.ts @@ -0,0 +1,46 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {DebugMode} from '../backgroundEffectsWorkerTypes'; + +/** + * Array of all valid debug mode values. + * + * Debug modes provide visualization tools for inspecting the segmentation mask: + * - 'off': Normal rendering with background effects applied + * - 'maskOverlay': Overlays a semi-transparent green tint on mask areas (foreground) + * - 'maskOnly': Displays only the segmentation mask as a grayscale image + * - 'edgeOnly': Highlights the edges of the mask using smoothstep edge detection + * - 'classOverlay': Overlays class colors from multiclass segmentation + * - 'classOnly': Shows class colors only (no video) + */ +export const DebugModeValues: DebugMode[] = ['off', 'maskOverlay', 'maskOnly', 'edgeOnly', 'classOverlay', 'classOnly']; + +/** + * Type guard function that checks if a string value is a valid DebugMode. + * + * This function provides runtime type safety when parsing debug mode values + * from external sources (e.g., URL parameters, user input, configuration files). + * + * @param value - The string value to check. + * @returns True if the value is a valid DebugMode, false otherwise. + */ +export function isDebugMode(value: string): value is DebugMode { + return (DebugModeValues as string[]).includes(value); +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/effects/backgroundEffectsController.test.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/backgroundEffectsController.test.ts new file mode 100644 index 00000000000..9de4884d950 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/backgroundEffectsController.test.ts @@ -0,0 +1,157 @@ +/* + * Wire + * Copyright (C) 2026 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {BackgroundEffectsController} from './backgroundEffectsController'; + +// Mocks +jest.mock('./capability', () => ({ + detectCapabilities: jest.fn(() => ({webgl2: true})), + choosePipeline: jest.fn(() => 'main-webgl2'), +})); + +jest.mock('../pipelines/mainWebGlPipeline', () => ({ + MainWebGlPipeline: jest.fn().mockImplementation(() => ({ + init: jest.fn().mockResolvedValue(undefined), + processFrame: jest.fn().mockResolvedValue(undefined), + updateConfig: jest.fn(), + stop: jest.fn(), + isOutputCanvasTransferred: jest.fn(() => false), + })), +})); + +jest.mock('../pipelines/passthroughPipeline', () => ({ + PassthroughPipeline: jest.fn().mockImplementation(() => ({ + init: jest.fn().mockResolvedValue(undefined), + processFrame: jest.fn().mockResolvedValue(undefined), + updateConfig: jest.fn(), + stop: jest.fn(), + isOutputCanvasTransferred: jest.fn(() => false), + })), +})); + +jest.mock('./frameSource', () => ({ + FrameSource: jest.fn().mockImplementation(() => ({ + start: jest.fn().mockResolvedValue(undefined), + stop: jest.fn(), + })), +})); + +jest.mock('Util/logger', () => ({ + getLogger: () => ({ + info: jest.fn(), + warn: jest.fn(), + }), +})); + +// Browser API mocks +global.document.createElement = jest.fn(() => ({ + width: 0, + height: 0, + getContext: jest.fn(() => ({})), + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + captureStream: jest.fn(() => ({ + getVideoTracks: () => [ + { + stop: jest.fn(), + }, + ], + })), +})) as any; + +describe('BackgroundEffectsController', () => { + let controller: BackgroundEffectsController; + let mockTrack: any; + + beforeEach(() => { + controller = new BackgroundEffectsController(); + + mockTrack = { + getSettings: () => ({ + width: 1280, + height: 720, + }), + addEventListener: jest.fn(), + }; + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should start and return outputTrack + stop function', async () => { + const result = await controller.start(mockTrack); + + expect(result).toHaveProperty('outputTrack'); + expect(typeof result.stop).toBe('function'); + }); + + it('should set mode and update pipeline config', async () => { + await controller.start(mockTrack); + + const spy = jest.spyOn(controller as any, 'updatePipelineConfig'); + + controller.setMode('virtual'); + + expect(spy).toHaveBeenCalled(); + }); + + it('should clamp blur strength between 0 and 1', async () => { + await controller.start(mockTrack); + + controller.setBlurStrength(2); // too high + expect((controller as any).blurStrength).toBe(1); + + controller.setBlurStrength(-1); // too low + expect((controller as any).blurStrength).toBe(0); + }); + + it('should stop and clean up resources', async () => { + const {stop} = await controller.start(mockTrack); + + await stop(); + + expect((controller as any).pipelineImpl).toBeNull(); + expect((controller as any).frameSource).toBeNull(); + expect((controller as any).outputTrack).toBeNull(); + }); + + it('should return false for isProcessing after stop', async () => { + await controller.start(mockTrack); + await controller.stop(); + + expect(controller.isProcessing()).toBe(false); + }); + + it('should update quality mode', async () => { + await controller.start(mockTrack); + + controller.setQuality('auto'); + + expect((controller as any).quality).toBe('auto'); + }); + + it('should set debug mode', async () => { + await controller.start(mockTrack); + + controller.setDebugMode('maskOnly'); + + expect((controller as any).debugMode).toBe('maskOnly'); + }); +}); diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/effects/backgroundEffectsController.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/backgroundEffectsController.ts new file mode 100644 index 00000000000..7201b65432f --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/backgroundEffectsController.ts @@ -0,0 +1,806 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Main controller for background effects processing pipeline. + * + * This class orchestrates the entire background effects system, managing: + * - BackgroundEffectsRenderingPipeline selection (worker-webgl2, main-webgl2, canvas2d, passthrough) + * - Frame processing and routing to appropriate pipeline + * - Runtime configuration (mode, quality, blur strength, debug mode) + * - Background source management (images and videos) + * - Resource lifecycle (initialization, cleanup) + * + * The controller automatically selects the best available pipeline based on + * browser capabilities and processes video frames through the selected pipeline + * to produce an output MediaStreamTrack with effects applied. + */ + +import {getLogger, Logger} from 'Util/logger'; + +import {choosePipeline, detectCapabilities} from './capability'; +import {FrameSource} from './frameSource'; + +import type { + CapabilityInfo, + DebugMode, + EffectMode, + Metrics, + PipelineType, + QualityMode, + QualityTier, + SegmentationModelByTier, + StartOptions, +} from '../backgroundEffectsWorkerTypes'; +import type {BackgroundEffectsRenderingPipeline, PipelineConfig} from '../pipelines/backgroundEffectsRenderingPipeline'; +import {Canvas2dPipeline} from '../pipelines/canvas2dPipeline'; +import {MainWebGlPipeline} from '../pipelines/mainWebGlPipeline'; +import {PassthroughPipeline} from '../pipelines/passthroughPipeline'; +import {WorkerWebGlPipeline} from '../pipelines/workerWebGlPipeline'; +import {resolveQualityPolicy, resolveSegmentationModelPath, TIER_DEFINITIONS} from '../quality'; + +/** + * Main controller for background effects processing. + * + * This class manages the complete background effects pipeline, from input + * MediaStreamTrack to output MediaStreamTrack with effects applied. It: + * + * 1. **Detects capabilities** and selects optimal pipeline + * 2. **Initializes components** (renderer, segmenter, quality controller) + * 3. **Processes frames** through the selected pipeline + * 4. **Manages resources** and handles cleanup + * 5. **Provides runtime controls** for mode, quality, and effects + * + * BackgroundEffectsRenderingPipeline selection priority: + * - worker-webgl2: Best performance (background thread processing) + * - main-webgl2: High quality (GPU-accelerated, main thread) + * - canvas2d: Fallback (CPU-based, widely supported) + * - passthrough: Last resort (no processing) + */ +export class BackgroundEffectsController { + /** Logger instance for debugging and warnings. */ + private readonly logger: Logger; + /** Whether running in development mode (enables additional logging). */ + private readonly isDev = process.env.NODE_ENV !== 'production'; + /** Frame source adapter for extracting frames from input track. */ + private frameSource: FrameSource | null = null; + /** Output canvas for rendering processed frames. */ + private outputCanvas: HTMLCanvasElement | null = null; + /** Output MediaStreamTrack from canvas.captureStream(). */ + private outputTrack: MediaStreamTrack | null = null; + /** Active pipeline implementation. */ + private pipelineImpl: BackgroundEffectsRenderingPipeline | null = null; + /** Current effect mode ('blur', 'virtual', or 'passthrough'). */ + private mode: EffectMode = 'blur'; + /** Current debug visualization mode. */ + private debugMode: DebugMode = 'off'; + /** Blur strength (0-1) for blur effect mode. */ + private blurStrength = 0.5; + /** Quality mode ('auto' for adaptive, or fixed tier. */ + private quality: QualityMode = 'auto'; + /** Target frames per second for adaptive quality control. */ + private targetFps = 15; + /** Per-tier segmentation model overrides. */ + private segmentationModelByTier: SegmentationModelByTier = { + superhigh: TIER_DEFINITIONS.superhigh.modelPath, + high: TIER_DEFINITIONS.high.modelPath, + medium: TIER_DEFINITIONS.medium.modelPath, + low: TIER_DEFINITIONS.low.modelPath, + bypass: TIER_DEFINITIONS.bypass.modelPath, + }; + /** Selected rendering pipeline. */ + private pipeline: PipelineType = 'passthrough'; + /** Cancel function for background video pump (stops video frame extraction). */ + private backgroundPumpCancel: (() => void) | null = null; + /** Counter for dropped frames (backpressure and frame skips). */ + private droppedFrames = 0; + /** WebGL context loss handler (main-webgl2 pipeline). */ + private webglContextLossHandler: ((event: Event) => void) | null = null; + /** WebGL context restoration handler (main-webgl2 pipeline). */ + private webglContextRestoreHandler: (() => void) | null = null; + /** Tracks whether the main WebGL context is currently lost. */ + private webglContextLost = false; + /** BackgroundEffectsRenderingPipeline to attempt to restore after context loss. */ + private webglRestorePipeline: PipelineType | null = null; + /** Last quality tier for main pipeline (for logging tier changes). */ + private lastMainTier: QualityTier | null = null; + /** Last quality tier for worker pipeline (for logging tier changes). */ + private lastWorkerTier: QualityTier | null = null; + /** Optional metrics callback for demo/telemetry use. */ + private onMetrics: ((metrics: Metrics) => void) | null = null; + + private onModelChange: ((model: string) => void) | null = null; + /** Tracks shutdown to avoid logging expected stop errors. */ + private isStopping = false; + private capabilityInfo: CapabilityInfo = { + offscreenCanvas: false, + worker: false, + webgl2: false, + requestVideoFrameCallback: false, + }; + private maxQualityTier: QualityTier = 'superhigh'; + + /** + * Creates a new background effects controller. + * + * Initializes the logger. All other components are initialized when start() is called. + */ + constructor() { + this.logger = getLogger('BackgroundEffectsController'); + } + + /** + * Starts the background effects pipeline. + * + * This method: + * 1. Applies configuration options + * 2. Detects browser capabilities and selects optimal pipeline + * 3. Initializes video source and output canvas + * 4. Initializes the selected pipeline (worker/main/canvas2d) + * 5. Starts frame processing loop + * 6. Creates output MediaStreamTrack from canvas + * 7. Sets up background sources if provided + * + * The pipeline processes frames from the input track and outputs a processed + * track via canvas.captureStream(). The output track can be used with + * RTCPeerConnection or other MediaStream APIs. + * + * @param inputTrack - Input video track (e.g., from getUserMedia). + * @param opts - Configuration options (all optional with defaults). + * @returns Promise resolving to output track and stop function. + */ + public async start( + inputTrack: MediaStreamTrack, + opts: StartOptions = {}, + ): Promise<{outputTrack: MediaStreamTrack; stop: () => void}> { + this.isStopping = false; + // Apply configuration options (use defaults if not provided) + this.mode = opts.mode ?? this.mode; + this.debugMode = opts.debugMode ?? this.debugMode; + this.blurStrength = opts.blurStrength ?? this.blurStrength; + this.quality = opts.quality ?? this.quality; + this.targetFps = opts.targetFps ?? this.targetFps; + // Detect capabilities and select optimal pipeline + const cap = detectCapabilities(); + this.capabilityInfo = cap; + const policy = resolveQualityPolicy(cap, opts.qualityPolicy ?? 'auto'); + + if (opts.segmentationModelPath) { + this.segmentationModelByTier = { + superhigh: opts.segmentationModelPath, + high: opts.segmentationModelPath, + medium: opts.segmentationModelPath, + low: opts.segmentationModelPath, + bypass: opts.segmentationModelPath, + }; + } else if (opts.segmentationModelByTier || policy.segmentationModelByTier) { + this.segmentationModelByTier = { + superhigh: TIER_DEFINITIONS.superhigh.modelPath, + high: TIER_DEFINITIONS.high.modelPath, + medium: TIER_DEFINITIONS.medium.modelPath, + low: TIER_DEFINITIONS.low.modelPath, + bypass: TIER_DEFINITIONS.bypass.modelPath, + ...policy.segmentationModelByTier, + ...opts.segmentationModelByTier, + }; + } else { + this.segmentationModelByTier = { + superhigh: TIER_DEFINITIONS.superhigh.modelPath, + high: TIER_DEFINITIONS.high.modelPath, + medium: TIER_DEFINITIONS.medium.modelPath, + low: TIER_DEFINITIONS.low.modelPath, + bypass: TIER_DEFINITIONS.bypass.modelPath, + }; + } + this.onMetrics = opts.onMetrics ?? null; + this.onModelChange = opts.onModelChange ?? null; + this.droppedFrames = 0; + this.pipelineImpl = null; + + const chosenPipeline = choosePipeline(cap, opts.useWorker !== false); + this.pipeline = opts.pipelineOverride ?? chosenPipeline; + + this.logger.info('Background effects capabilities', cap); + this.logger.info('Background effects pipeline', { + chosen: chosenPipeline, + override: opts.pipelineOverride ?? null, + active: this.pipeline, + }); + + // Initialize frame source for frame extraction + this.frameSource = new FrameSource(inputTrack); + // Create output canvas for rendering + this.outputCanvas = document.createElement('canvas'); + + // Set canvas dimensions from input track settings + const settings = inputTrack.getSettings(); + this.outputCanvas.width = settings.width ?? 640; + this.outputCanvas.height = settings.height ?? 480; + + // Initialize selected pipeline + await this.initPipeline(this.pipeline); + // Start frame processing loop + await this.frameSource.start( + async (frame, timestamp, width, height) => { + if (!this.outputCanvas) { + try { + frame.close(); + } catch { + // Ignore close errors. + } + return; + } + // Skip frames with invalid dimensions + if (width === 0 || height === 0) { + try { + frame.close(); + } catch { + // Ignore close errors. + } + return; + } + // Resize canvas if dimensions changed (skip if transferred to worker) + if ( + !this.pipelineImpl?.isOutputCanvasTransferred() && + (this.outputCanvas.width !== width || this.outputCanvas.height !== height) + ) { + try { + this.outputCanvas.width = width; + this.outputCanvas.height = height; + } catch (error) { + this.logger.warn('Failed to resize output canvas', error); + } + } + + try { + await this.handleFrame(frame, timestamp, width, height); + } catch (error) { + try { + frame.close(); + } catch { + // Ignore close errors. + } + if (this.isStopping) { + return; + } + this.logger.warn('Frame handling failed', error); + } + }, + () => { + const count = this.handleFrameDrop(); + this.pipelineImpl?.notifyDroppedFrames(count); + }, + ); + + // Create output MediaStreamTrack from canvas + const captureStream = this.outputCanvas.captureStream(this.targetFps); + this.outputTrack = captureStream.getVideoTracks()[0]; + + // Stop pipeline when input track ends + inputTrack.addEventListener('ended', async () => await this.stop()); + + // Set background sources if provided + if (opts.backgroundImage) { + this.setBackgroundSource(opts.backgroundImage); + } + if (opts.backgroundVideo) { + this.setBackgroundSource(opts.backgroundVideo); + } + if (opts.backgroundColor) { + this.setBackgroundColor(opts.backgroundColor); + } + + return { + outputTrack: this.outputTrack, + stop: async () => await this.stop(), + }; + } + + /** + * Sets the effect mode. + * + * Changes the processing mode at runtime. Updates worker if using worker pipeline. + * + * @param mode - Effect mode ('blur', 'virtual', or 'passthrough'). + */ + public setMode(mode: EffectMode): void { + this.mode = mode; + this.logger.info('Background effects mode', mode); + this.updatePipelineConfig(); + } + + /** + * Sets the blur strength for blur effect mode. + * + * Clamps value to valid range [0, 1]. Updates worker if using worker pipeline. + * + * @param value - Blur strength (0 = no blur, 1 = maximum blur). + */ + public setBlurStrength(value: number): void { + this.blurStrength = Math.max(0, Math.min(1, value)); + this.updatePipelineConfig(); + } + + /** + * Sets the background source for virtual background mode. + * + * Supports three source types: + * - HTMLImageElement: Static image (converted to ImageBitmap) + * - HTMLVideoElement: Video (pumped at ~15fps, converted to ImageBitmap frames) + * - ImageBitmap: Direct bitmap (transferred to worker if using worker pipeline) + * + * For worker pipeline, the bitmap is transferred (not cloned) for performance. + * For main pipeline, the bitmap is stored and passed to renderer. + * + * @param source - Background image, video element, or ImageBitmap. + */ + public setBackgroundSource(source: HTMLImageElement | HTMLVideoElement | ImageBitmap): void { + if (!(source instanceof HTMLVideoElement)) { + this.backgroundPumpCancel?.(); + this.backgroundPumpCancel = null; + } + if (source instanceof HTMLImageElement) { + createImageBitmap(source) + .then(bitmap => { + if (!this.pipelineImpl) { + bitmap.close(); + return; + } + this.pipelineImpl.setBackgroundImage(bitmap, source.naturalWidth, source.naturalHeight); + }) + .catch((error: unknown) => this.logger.warn('Failed to set background image', error)); + return; + } + + if (source instanceof HTMLVideoElement) { + this.startBackgroundVideoPump(source); + return; + } + + if (!this.pipelineImpl) { + source.close(); + return; + } + this.pipelineImpl.setBackgroundImage(source, source.width, source.height); + } + + /** + * Sets a solid-color background for virtual background mode. + * + * Creates a 1x1 ImageBitmap filled with the requested color and reuses the + * existing background source pipeline. + * + * @param color - CSS color string (e.g., '#112233' or 'rgb(0, 0, 0)'). + */ + public setBackgroundColor(color: string): void { + this.backgroundPumpCancel?.(); + this.backgroundPumpCancel = null; + this.createSolidColorBitmap(color) + .then(bitmap => { + if (!this.pipelineImpl) { + bitmap.close(); + return; + } + this.pipelineImpl.setBackgroundImage(bitmap, 1, 1); + }) + .catch((error: unknown) => this.logger.warn('Failed to set solid background color', error)); + } + + /** + * Sets the debug visualization mode. + * + * Updates worker if using worker pipeline. Debug modes provide visualization + * tools for inspecting segmentation masks. + * + * @param mode - Debug mode ('off', 'maskOverlay', 'maskOnly', or 'edgeOnly'). + */ + public setDebugMode(mode: DebugMode): void { + this.debugMode = mode; + this.updatePipelineConfig(); + } + + /** + * Sets the quality mode. + * + * Changes quality mode at runtime. 'auto' enables adaptive quality based on + * performance metrics, while fixed tiers ('A'/'B'/'C'/'D') use constant quality. + * Updates worker if using worker pipeline. + * + * @param mode - Quality mode ('auto' or fixed tier 'A'/'B'/'C'/'D'). + */ + public setQuality(mode: QualityMode): void { + this.quality = mode; + if (this.isDev) { + this.logger.info('Background effects quality mode', mode); + } + this.updatePipelineConfig(); + } + + public getQuality(): QualityMode { + return this.quality; + } + + /** + * Stops the background effects pipeline and cleans up all resources. + * + * This method: + * 1. Stops background video pump if active + * 2. Closes pending frames and background sources + * 3. Terminates worker if using worker pipeline + * 4. Destroys renderer and segmenter + * 5. Stops video source and output track + * 6. Clears all references + * + * Should be called when the pipeline is no longer needed to free all resources + * and prevent memory leaks. + */ + public async stop(): Promise { + this.isStopping = true; + this.backgroundPumpCancel?.(); + this.backgroundPumpCancel = null; + this.pipelineImpl?.stop(); + this.pipelineImpl = null; + + this.frameSource?.stop(); + this.frameSource = null; + + this.outputTrack?.stop(); + this.outputTrack = null; + + this.detachWebGLContextHandlers(); + this.webglContextLost = false; + this.webglRestorePipeline = null; + this.outputCanvas = null; + this.onMetrics = null; + this.pipeline = 'passthrough'; + } + + private async initPipeline(type: PipelineType): Promise { + if (!this.outputCanvas) { + return; + } + this.detachWebGLContextHandlers(); + this.pipelineImpl?.stop(); + this.pipelineImpl = this.createPipeline(type); + this.pipeline = type; + + const config: PipelineConfig = { + mode: this.mode, + debugMode: this.debugMode, + blurStrength: this.blurStrength, + quality: this.quality, + }; + + const cap = detectCapabilities(); + const policy = resolveQualityPolicy(cap, 'auto'); + const initialTier = this.quality === 'auto' ? policy.initialTier : this.quality; + const segmentationModelPath = resolveSegmentationModelPath(initialTier, this.segmentationModelByTier, undefined); + + try { + await this.pipelineImpl.init({ + outputCanvas: this.outputCanvas, + targetFps: this.targetFps, + segmentationModelPath, + segmentationModelByTier: this.segmentationModelByTier, + initialTier, + maxTier: this.maxQualityTier, + config, + onMetrics: this.onMetrics, + onTierChange: tier => this.handleTierChange(tier), + onDroppedFrame: () => this.handleFrameDrop(), + getDroppedFrames: () => this.droppedFrames, + onWorkerSegmenterError: error => { + if (this.isDev) { + this.logger.warn('Worker segmenter init failed', error); + } + }, + onWorkerContextLoss: () => this.handleWorkerContextLoss(), + }); + } catch (error) { + this.logger.warn('BackgroundEffectsRenderingPipeline init failed, falling back to passthrough', error); + this.pipelineImpl?.stop(); + this.pipelineImpl = new PassthroughPipeline(); + this.pipeline = 'passthrough'; + await this.pipelineImpl.init({ + outputCanvas: this.outputCanvas, + targetFps: this.targetFps, + segmentationModelPath, + segmentationModelByTier: this.segmentationModelByTier, + initialTier, + maxTier: this.maxQualityTier, + config, + onMetrics: this.onMetrics, + onTierChange: tier => this.handleTierChange(tier), + onDroppedFrame: () => this.handleFrameDrop(), + getDroppedFrames: () => this.droppedFrames, + onWorkerContextLoss: () => this.handleWorkerContextLoss(), + }); + } + + if (this.pipeline === 'main-webgl2') { + this.bindWebGLContextHandlers(); + } + } + + private createPipeline(type: PipelineType): BackgroundEffectsRenderingPipeline { + switch (type) { + case 'worker-webgl2': + return new WorkerWebGlPipeline(); + case 'main-webgl2': + return new MainWebGlPipeline(); + case 'canvas2d': + return new Canvas2dPipeline(); + default: + return new PassthroughPipeline(); + } + } + + private updatePipelineConfig(): void { + if (!this.pipelineImpl) { + return; + } + this.pipelineImpl.updateConfig({ + mode: this.mode, + debugMode: this.debugMode, + blurStrength: this.blurStrength, + quality: this.quality, + }); + } + + private bindWebGLContextHandlers(): void { + if (!this.outputCanvas || this.webglContextLossHandler || this.pipeline !== 'main-webgl2') { + return; + } + this.webglContextLossHandler = event => { + event.preventDefault(); + this.handleWebGLContextLost(); + }; + this.webglContextRestoreHandler = () => { + void this.handleWebGLContextRestored(); + }; + this.outputCanvas.addEventListener('webglcontextlost', this.webglContextLossHandler as EventListener, { + passive: false, + }); + this.outputCanvas.addEventListener('webglcontextrestored', this.webglContextRestoreHandler as EventListener); + } + + private detachWebGLContextHandlers(): void { + if (!this.outputCanvas || !this.webglContextLossHandler || !this.webglContextRestoreHandler) { + return; + } + this.outputCanvas.removeEventListener('webglcontextlost', this.webglContextLossHandler as EventListener); + this.outputCanvas.removeEventListener('webglcontextrestored', this.webglContextRestoreHandler as EventListener); + this.webglContextLossHandler = null; + this.webglContextRestoreHandler = null; + } + + private handleWebGLContextLost(): void { + if (this.webglContextLost || this.pipeline !== 'main-webgl2') { + return; + } + this.webglContextLost = true; + this.webglRestorePipeline = this.pipeline; + this.logger.warn('WebGL context lost; falling back to passthrough'); + void this.initPipeline('passthrough'); + } + + private async handleWebGLContextRestored(): Promise { + if (!this.webglContextLost || this.webglRestorePipeline !== 'main-webgl2') { + return; + } + if (!this.outputCanvas) { + return; + } + this.logger.info('WebGL context restored; restarting main pipeline'); + try { + await this.initPipeline('main-webgl2'); + } catch (error) { + this.logger.warn('Failed to restore WebGL pipeline; staying in passthrough', error); + } finally { + this.webglContextLost = false; + this.webglRestorePipeline = null; + } + } + + /** + * Handles WebGL context loss in the worker pipeline. + * + * When the worker's WebGL context is lost (e.g., due to GPU driver issues, + * system sleep, or resource constraints), this method falls back to passthrough + * mode to ensure video continues to work, albeit without effects. + * + * Only handles context loss for worker-webgl2 pipeline. Main-thread WebGL + * context loss is handled separately via bindWebGLContextHandlers(). + * + * @returns Nothing. + */ + private handleWorkerContextLoss(): void { + if (this.isStopping || this.pipeline !== 'worker-webgl2') { + return; + } + this.logger.warn('Worker WebGL context lost; falling back to passthrough'); + void this.initPipeline('passthrough'); + } + + /** + * Routes a frame to the appropriate pipeline for processing. + * + * Dispatches the frame to the selected pipeline: + * - worker-webgl2: Sends to worker via postMessage + * - main-webgl2: Renders on main thread + * - canvas2d: Renders using Canvas2D API + * - passthrough: Passes through without processing + * + * @param frame - Video frame as ImageBitmap. + * @param timestamp - Frame timestamp in seconds. + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + */ + private async handleFrame(frame: ImageBitmap, timestamp: number, width: number, height: number): Promise { + if (!this.pipelineImpl) { + frame.close(); + return; + } + await this.pipelineImpl.processFrame(frame, timestamp, width, height); + } + + private handleFrameDrop(): number { + this.droppedFrames += 1; + return this.droppedFrames; + } + + private handleTierChange(tier: QualityTier): void { + const newModel = resolveSegmentationModelPath(tier, this.segmentationModelByTier, undefined); + if (this.onModelChange !== null) { + this.onModelChange(newModel); + } + this.logger.info('Quality tier changed', tier, newModel); + + if (this.pipeline === 'worker-webgl2') { + this.maybeLogWorkerTierChange(tier); + return; + } + this.maybeLogMainTierChange(tier); + } + + /** + * Logs quality tier changes for main pipeline (development only). + * + * @param tier - New quality tier. + */ + private maybeLogMainTierChange(tier: QualityTier): void { + if (!this.isDev) { + return; + } + if (this.lastMainTier !== tier) { + this.logger.info('Main pipeline quality tier change', {from: this.lastMainTier, to: tier}); + this.lastMainTier = tier; + } + } + + /** + * Logs quality tier changes for worker pipeline (development only). + * + * @param tier - New quality tier. + */ + private maybeLogWorkerTierChange(tier: QualityTier): void { + if (this.lastWorkerTier !== tier) { + this.logger.info('Worker pipeline quality tier change', {from: this.lastWorkerTier, to: tier}); + this.lastWorkerTier = tier; + } + } + + private async createSolidColorBitmap(color: string): Promise { + const canvas = + typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); + const isOffscreen = typeof OffscreenCanvas !== 'undefined' && canvas instanceof OffscreenCanvas; + if (!isOffscreen) { + (canvas as HTMLCanvasElement).width = 1; + (canvas as HTMLCanvasElement).height = 1; + } + const ctx = canvas.getContext('2d'); + if (!ctx) { + throw new Error('Failed to create 2D context for solid background.'); + } + ctx.fillStyle = color; + ctx.fillRect(0, 0, 1, 1); + return createImageBitmap(canvas as OffscreenCanvas | HTMLCanvasElement); + } + + /** + * Starts background video frame extraction pump. + * + * Extracts frames from an HTMLVideoElement at ~15fps and sends them to the + * renderer/worker for virtual background mode. Uses requestVideoFrameCallback + * (preferred) or requestAnimationFrame (fallback) for frame timing. + * + * The pump runs continuously until cancelled via backgroundPumpCancel. + * + * @param video - HTMLVideoElement to extract frames from. + */ + private startBackgroundVideoPump(video: HTMLVideoElement): void { + this.backgroundPumpCancel?.(); + + let lastTimestamp = 0; + const targetInterval = 1000 / 15; + let active = true; + let rVFCHandle: number | null = null; + let rafHandle: number | null = null; + + const pump = async (now: number) => { + if (!active) { + return; + } + if (now - lastTimestamp < targetInterval) { + schedule(); + return; + } + lastTimestamp = now; + + try { + const bitmap = await createImageBitmap(video); + if (!this.pipelineImpl) { + bitmap.close(); + } else { + this.pipelineImpl.setBackgroundVideoFrame(bitmap, video.videoWidth, video.videoHeight); + } + } catch (error) { + this.logger.warn('Failed to capture background video frame', error); + } + + schedule(); + }; + + const schedule = () => { + if ('requestVideoFrameCallback' in video) { + rVFCHandle = (video as any).requestVideoFrameCallback((now: number) => pump(now)); + } else { + rafHandle = window.requestAnimationFrame(pump); + } + }; + + schedule(); + this.backgroundPumpCancel = () => { + active = false; + if (rVFCHandle !== null && 'cancelVideoFrameCallback' in video) { + (video as any).cancelVideoFrameCallback(rVFCHandle); + } + if (rafHandle !== null) { + window.cancelAnimationFrame(rafHandle); + } + this.pipelineImpl?.clearBackground(); + }; + } + + isProcessing() { + return this.pipelineImpl !== null; + } + + public getCapabilityInfo(): CapabilityInfo { + return this.capabilityInfo; + } + + public setMaxQualityTier(quality: QualityTier) { + this.maxQualityTier = quality; + } + + public getMaxQualityTier(): QualityTier { + return this.maxQualityTier; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/effects/capability.test.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/capability.test.ts new file mode 100644 index 00000000000..20729d71fdf --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/capability.test.ts @@ -0,0 +1,461 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {choosePipeline, detectCapabilities} from './capability'; +import {CapabilityInfo} from 'Repositories/media/backgroundEffects/backgroundEffectsWorkerTypes'; + +describe('capability', () => { + // Store original globals to restore after tests + const originalOffscreenCanvas = global.OffscreenCanvas; + const originalWorker = global.Worker; + const originalHTMLVideoElement = global.HTMLVideoElement; + const originalDocument = global.document; + + beforeEach(() => { + // Reset globals before each test using Object.defineProperty for safer mocking + Object.defineProperty(global, 'OffscreenCanvas', { + value: undefined, + writable: true, + configurable: true, + }); + Object.defineProperty(global, 'Worker', { + value: undefined, + writable: true, + configurable: true, + }); + Object.defineProperty(global, 'HTMLVideoElement', { + value: undefined, + writable: true, + configurable: true, + }); + // Don't delete document/window as jsdom needs them - just mock their methods + }); + + afterEach(() => { + // Restore original globals + if (originalOffscreenCanvas !== undefined) { + Object.defineProperty(global, 'OffscreenCanvas', { + value: originalOffscreenCanvas, + writable: true, + configurable: true, + }); + } else { + delete (global as any).OffscreenCanvas; + } + if (originalWorker !== undefined) { + Object.defineProperty(global, 'Worker', { + value: originalWorker, + writable: true, + configurable: true, + }); + } else { + delete (global as any).Worker; + } + if (originalHTMLVideoElement !== undefined) { + Object.defineProperty(global, 'HTMLVideoElement', { + value: originalHTMLVideoElement, + writable: true, + configurable: true, + }); + } else { + delete (global as any).HTMLVideoElement; + } + if (originalDocument !== undefined) { + (global as any).document = originalDocument; + } else { + delete (global as any).document; + } + // Restore document methods + jest.restoreAllMocks(); + }); + + describe('detectCapabilities', () => { + it('detects all capabilities when all are available', () => { + // Mock OffscreenCanvas + Object.defineProperty(global, 'OffscreenCanvas', { + value: class {}, + writable: true, + configurable: true, + }); + // Mock Worker + Object.defineProperty(global, 'Worker', { + value: class {}, + writable: true, + configurable: true, + }); + // Mock HTMLVideoElement with requestVideoFrameCallback + class MockHTMLVideoElement {} + Object.defineProperty(MockHTMLVideoElement.prototype, 'requestVideoFrameCallback', { + value: () => {}, + writable: true, + configurable: true, + }); + Object.defineProperty(global, 'HTMLVideoElement', { + value: MockHTMLVideoElement, + writable: true, + configurable: true, + }); + // Mock document and canvas with WebGL2 + const mockCanvas = { + getContext: jest.fn().mockReturnValue({}), + }; + jest.spyOn(document, 'createElement').mockReturnValue(mockCanvas as any); + const caps = detectCapabilities(); + + expect(caps.offscreenCanvas).toBe(true); + expect(caps.worker).toBe(true); + expect(caps.webgl2).toBe(true); + expect(caps.requestVideoFrameCallback).toBe(true); + }); + + it('detects no capabilities when none are available', () => { + // No globals set - all should be false + const caps = detectCapabilities(); + + expect(caps.offscreenCanvas).toBe(false); + expect(caps.worker).toBe(false); + expect(caps.webgl2).toBe(false); + expect(caps.requestVideoFrameCallback).toBe(false); + }); + + it('detects OffscreenCanvas when available', () => { + Object.defineProperty(global, 'OffscreenCanvas', { + value: class {}, + writable: true, + configurable: true, + }); + + const caps = detectCapabilities(); + + expect(caps.offscreenCanvas).toBe(true); + expect(caps.worker).toBe(false); + expect(caps.webgl2).toBe(false); + expect(caps.requestVideoFrameCallback).toBe(false); + }); + + it('detects Worker when available', () => { + Object.defineProperty(global, 'Worker', { + value: class {}, + writable: true, + configurable: true, + }); + + const caps = detectCapabilities(); + + expect(caps.offscreenCanvas).toBe(false); + expect(caps.worker).toBe(true); + expect(caps.webgl2).toBe(false); + expect(caps.requestVideoFrameCallback).toBe(false); + }); + + it('detects WebGL2 when available', () => { + const mockCanvas = { + getContext: jest.fn().mockReturnValue({}), + }; + jest.spyOn(document, 'createElement').mockReturnValue(mockCanvas as any); + + const caps = detectCapabilities(); + + expect(caps.offscreenCanvas).toBe(false); + expect(caps.worker).toBe(false); + expect(caps.webgl2).toBe(true); + expect(caps.requestVideoFrameCallback).toBe(false); + }); + + it('detects WebGL2 as false when document.createElement returns canvas without webgl2 context', () => { + // Test the case where document exists but getContext('webgl2') returns null + // This simulates the behavior when document is undefined (webgl2 check returns false) + const mockCanvas = { + getContext: jest.fn().mockReturnValue(null), + }; + jest.spyOn(document, 'createElement').mockReturnValue(mockCanvas as any); + + const caps = detectCapabilities(); + + expect(caps.webgl2).toBe(false); + }); + + it('detects WebGL2 as false when getContext returns null', () => { + const mockCanvas = { + getContext: jest.fn().mockReturnValue(null), + }; + jest.spyOn(document, 'createElement').mockReturnValue(mockCanvas as any); + + const caps = detectCapabilities(); + + expect(caps.webgl2).toBe(false); + }); + + it('detects requestVideoFrameCallback when available', () => { + class MockHTMLVideoElement {} + Object.defineProperty(MockHTMLVideoElement.prototype, 'requestVideoFrameCallback', { + value: () => {}, + writable: true, + configurable: true, + }); + Object.defineProperty(global, 'HTMLVideoElement', { + value: MockHTMLVideoElement, + writable: true, + configurable: true, + }); + + const caps = detectCapabilities(); + + expect(caps.offscreenCanvas).toBe(false); + expect(caps.worker).toBe(false); + expect(caps.webgl2).toBe(false); + expect(caps.requestVideoFrameCallback).toBe(true); + }); + + it('detects requestVideoFrameCallback as false when HTMLVideoElement is undefined', () => { + // HTMLVideoElement is undefined + const caps = detectCapabilities(); + + expect(caps.requestVideoFrameCallback).toBe(false); + }); + + it('detects requestVideoFrameCallback as false when method not in prototype', () => { + class MockHTMLVideoElement {} + // Don't add requestVideoFrameCallback to prototype + Object.defineProperty(global, 'HTMLVideoElement', { + value: MockHTMLVideoElement, + writable: true, + configurable: true, + }); + + const caps = detectCapabilities(); + + expect(caps.requestVideoFrameCallback).toBe(false); + }); + + it('handles partial capabilities correctly', () => { + // Only OffscreenCanvas and Worker, but no WebGL2 + Object.defineProperty(global, 'OffscreenCanvas', { + value: class {}, + writable: true, + configurable: true, + }); + Object.defineProperty(global, 'Worker', { + value: class {}, + writable: true, + configurable: true, + }); + const mockCanvas = { + getContext: jest.fn().mockReturnValue(null), // WebGL2 not available + }; + jest.spyOn(document, 'createElement').mockReturnValue(mockCanvas as any); + + const caps = detectCapabilities(); + + expect(caps.offscreenCanvas).toBe(true); + expect(caps.worker).toBe(true); + expect(caps.webgl2).toBe(false); + expect(caps.requestVideoFrameCallback).toBe(false); + }); + }); + + describe('choosePipeline', () => { + it('selects worker-webgl2 when all capabilities available and preferWorker=true', () => { + const caps: CapabilityInfo = { + offscreenCanvas: true, + worker: true, + webgl2: true, + requestVideoFrameCallback: true, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('worker-webgl2'); + }); + + it('selects main-webgl2 when all capabilities available but preferWorker=false', () => { + const caps: CapabilityInfo = { + offscreenCanvas: true, + worker: true, + webgl2: true, + requestVideoFrameCallback: true, + }; + + const pipeline = choosePipeline(caps, false); + + expect(pipeline).toBe('main-webgl2'); + }); + + it('selects main-webgl2 when webgl2 available but worker missing', () => { + const caps: CapabilityInfo = { + offscreenCanvas: true, + worker: false, + webgl2: true, + requestVideoFrameCallback: true, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('main-webgl2'); + }); + + it('selects main-webgl2 when webgl2 available but offscreenCanvas missing', () => { + const caps: CapabilityInfo = { + offscreenCanvas: false, + worker: true, + webgl2: true, + requestVideoFrameCallback: true, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('main-webgl2'); + }); + + it('selects main-webgl2 when only webgl2 is available', () => { + const caps: CapabilityInfo = { + offscreenCanvas: false, + worker: false, + webgl2: true, + requestVideoFrameCallback: false, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('main-webgl2'); + }); + + it('selects canvas2d when webgl2 unavailable but document exists', () => { + // document exists (jsdom provides it), so canvas2d should be selected + const caps: CapabilityInfo = { + offscreenCanvas: false, + worker: false, + webgl2: false, + requestVideoFrameCallback: false, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('canvas2d'); + }); + + it('selects passthrough when no capabilities and no document', () => { + const previousDescriptor = Object.getOwnPropertyDescriptor(global, 'document'); + Object.defineProperty(global, 'document', {value: undefined, configurable: true}); + try { + const caps: CapabilityInfo = { + offscreenCanvas: false, + worker: false, + webgl2: false, + requestVideoFrameCallback: false, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('passthrough'); + } finally { + if (previousDescriptor) { + Object.defineProperty(global, 'document', previousDescriptor); + } else { + delete (global as any).document; + } + } + }); + + it('selects canvas2d when no webgl2 but document exists (jsdom environment)', () => { + // In jsdom, document exists, so canvas2d is selected + const caps: CapabilityInfo = { + offscreenCanvas: false, + worker: false, + webgl2: false, + requestVideoFrameCallback: false, + }; + + const pipeline = choosePipeline(caps, true); + + // In jsdom environment, document exists so canvas2d is returned + // In real worker environment (document undefined), passthrough would be returned + expect(pipeline).toBe('canvas2d'); + }); + + it('prioritizes worker-webgl2 over main-webgl2 when both possible', () => { + const caps: CapabilityInfo = { + offscreenCanvas: true, + worker: true, + webgl2: true, + requestVideoFrameCallback: true, + }; + + const workerPipeline = choosePipeline(caps, true); + const mainPipeline = choosePipeline(caps, false); + + expect(workerPipeline).toBe('worker-webgl2'); + expect(mainPipeline).toBe('main-webgl2'); + }); + + it('prioritizes main-webgl2 over canvas2d when webgl2 available', () => { + const caps: CapabilityInfo = { + offscreenCanvas: false, + worker: false, + webgl2: true, + requestVideoFrameCallback: false, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('main-webgl2'); + }); + + it('prioritizes canvas2d over passthrough when document available', () => { + // document exists (jsdom provides it) + const caps: CapabilityInfo = { + offscreenCanvas: false, + worker: false, + webgl2: false, + requestVideoFrameCallback: false, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('canvas2d'); + }); + + it('handles edge case: webgl2 true but preferWorker false with all other capabilities', () => { + const caps: CapabilityInfo = { + offscreenCanvas: true, + worker: true, + webgl2: true, + requestVideoFrameCallback: true, + }; + + const pipeline = choosePipeline(caps, false); + + expect(pipeline).toBe('main-webgl2'); + }); + + it('handles edge case: only requestVideoFrameCallback available', () => { + // document exists (jsdom provides it) + const caps: CapabilityInfo = { + offscreenCanvas: false, + worker: false, + webgl2: false, + requestVideoFrameCallback: true, + }; + + const pipeline = choosePipeline(caps, true); + + expect(pipeline).toBe('canvas2d'); + }); + }); +}); diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/effects/capability.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/capability.ts new file mode 100644 index 00000000000..37629aeea78 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/capability.ts @@ -0,0 +1,124 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {CapabilityInfo} from '../backgroundEffectsWorkerTypes'; + +/** + * Detects browser capabilities required for background effects rendering. + * + * This function performs runtime feature detection for web APIs needed by + * different rendering pipelines. Detection is performed synchronously and + * is safe to call in any environment (browser, worker, Node.js). + * + * Detection methods: + * - OffscreenCanvas: Checks for global OffscreenCanvas constructor + * - Worker: Checks for global Worker constructor + * - WebGL2: Creates a test canvas and attempts to get WebGL2 context + * - requestVideoFrameCallback: Checks for method on HTMLVideoElement prototype + * + * @returns Capability information object with boolean flags for each capability. + * + * @example + * ```typescript + * const caps = detectCapabilities(); + * if (caps.webgl2 && caps.worker && caps.offscreenCanvas) { + * // Can use worker-based WebGL2 pipeline + * } + * ``` + */ +export function detectCapabilities(): CapabilityInfo { + // Check for OffscreenCanvas support (enables worker-based rendering) + const offscreenCanvas = typeof OffscreenCanvas !== 'undefined'; + // Check for Web Worker support (enables background thread processing) + const worker = typeof Worker !== 'undefined'; + // Check for requestVideoFrameCallback (better than requestAnimationFrame for video) + const requestVideoFrameCallback = + typeof HTMLVideoElement !== 'undefined' && 'requestVideoFrameCallback' in HTMLVideoElement.prototype; + // Check for WebGL2 support (requires DOM for canvas creation) + const webgl2 = (() => { + if (typeof document === 'undefined') { + return false; + } + const canvas = document.createElement('canvas'); + return !!canvas.getContext('webgl2'); + })(); + + return { + offscreenCanvas, + worker, + webgl2, + requestVideoFrameCallback, + }; +} + +/** + * Selects the optimal rendering pipeline based on browser capabilities. + * + * BackgroundEffectsRenderingPipeline selection follows a priority order, selecting the highest-quality + * pipeline that the browser supports: + * + * 1. **worker-webgl2** (highest quality, best performance) + * - Requires: WebGL2, Worker, OffscreenCanvas, and preferWorker=true + * - Benefits: Offloads rendering to background thread, avoids main thread blocking + * + * 2. **main-webgl2** (high quality, good performance) + * - Requires: WebGL2 + * - Benefits: GPU-accelerated rendering with advanced effects + * - Trade-off: Runs on main thread (may impact UI responsiveness) + * + * 3. **canvas2d** (medium quality, acceptable performance) + * - Requires: Document/DOM (browser environment) + * - Benefits: Widely supported, no WebGL requirement + * - Trade-off: CPU-based rendering, limited effects quality + * + * 4. **passthrough** (no processing, fallback only) + * - Used when: No DOM available or all other pipelines unavailable + * - Behavior: Passes through original video track unchanged + * + * @param cap - Capability information from detectCapabilities(). + * @param preferWorker - If true, prefers worker-based pipeline when available. + * If false, skips worker pipeline even if supported. + * @returns The selected pipeline identifier. + * + * @example + * ```typescript + * const caps = detectCapabilities(); + * const pipeline = choosePipeline(caps, true); + * // pipeline will be one of: 'worker-webgl2', 'main-webgl2', 'canvas2d', 'passthrough' + * ``` + */ +export function choosePipeline( + cap: CapabilityInfo, + preferWorker: boolean, +): 'worker-webgl2' | 'main-webgl2' | 'canvas2d' | 'passthrough' { + // Priority 1: Worker + OffscreenCanvas + WebGL2 (best performance) + if (cap.webgl2 && cap.worker && cap.offscreenCanvas && preferWorker) { + return 'worker-webgl2'; + } + // Priority 2: Main-thread WebGL2 (GPU-accelerated, but blocks main thread) + if (cap.webgl2) { + return 'main-webgl2'; + } + // Priority 3: Canvas2D (CPU-based, widely supported) + if (typeof document !== 'undefined') { + return 'canvas2d'; + } + // Priority 4: Passthrough (no processing, last resort) + return 'passthrough'; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/effects/frameSource.test.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/frameSource.test.ts new file mode 100644 index 00000000000..267b2a30430 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/frameSource.test.ts @@ -0,0 +1,206 @@ +/* + * Wire + * Copyright (C) 2026 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {FrameSource} from './frameSource'; + +// Logger mock +jest.mock('Util/logger', () => ({ + getLogger: () => ({ + warn: jest.fn(), + info: jest.fn(), + }), +})); + +// VideoSource mock +jest.mock('./videoSource', () => ({ + VideoSource: jest.fn().mockImplementation(() => ({ + element: {}, + start: jest.fn().mockResolvedValue(undefined), + stop: jest.fn(), + })), +})); + +describe('FrameSource', () => { + let mockTrack: any; + + beforeEach(() => { + mockTrack = {}; + jest.clearAllMocks(); + }); + + afterEach(() => { + delete (window as any).MediaStreamTrackProcessor; + }); + + const createMockBitmap = () => ({ + width: 640, + height: 480, + close: jest.fn(), + }); + + // Mock createImageBitmap + beforeAll(() => { + global.createImageBitmap = jest.fn(() => Promise.resolve(createMockBitmap())) as any; + }); + + it('should use MediaStreamTrackProcessor if available', async () => { + const readMock = jest + .fn() + .mockResolvedValueOnce({ + done: false, + value: { + timestamp: 1_000_000, + close: jest.fn(), + }, + }) + .mockResolvedValueOnce({done: true}); + + const reader = { + read: readMock, + releaseLock: jest.fn(), + }; + + const processorMock = jest.fn().mockImplementation(() => ({ + readable: { + getReader: () => reader, + }, + })); + + (window as any).MediaStreamTrackProcessor = processorMock; + + const fs = new FrameSource(mockTrack); + + const onFrame = jest.fn(); + + await fs.start(onFrame); + + // Wait for async loop + await new Promise(r => setTimeout(r, 0)); + + expect(processorMock).toHaveBeenCalled(); + expect(onFrame).toHaveBeenCalled(); + }); + + it('should fallback to video element if processor not available', async () => { + const fs = new FrameSource(mockTrack); + const onFrame = jest.fn(); + + await fs.start(onFrame); + + const {VideoSource} = require('./videoSource'); + + expect(VideoSource).toHaveBeenCalled(); + }); + + it('should call onFrame with correct params (video fallback)', async () => { + const {VideoSource} = require('./videoSource'); + + let callback: any; + + VideoSource.mockImplementation(() => ({ + element: {}, + start: jest.fn(cb => { + callback = cb; + return Promise.resolve(); + }), + stop: jest.fn(), + })); + + const fs = new FrameSource(mockTrack); + + const onFrame = jest.fn(); + + await fs.start(onFrame); + + // simulate frame + await callback(1, 640, 480); + + expect(onFrame).toHaveBeenCalled(); + const args = onFrame.mock.calls[0]; + + expect(args[1]).toBe(1); // timestamp + expect(args[2]).toBe(640); + expect(args[3]).toBe(480); + }); + + it('should call onDrop when processing is busy', async () => { + const {VideoSource} = require('./videoSource'); + + let callback: any; + + VideoSource.mockImplementation(() => ({ + element: {}, + start: jest.fn(cb => { + callback = cb; + return Promise.resolve(); + }), + stop: jest.fn(), + })); + + const fs = new FrameSource(mockTrack); + + const onFrame = jest.fn(async () => { + // simulate long processing + await new Promise(r => setTimeout(r, 10)); + }); + + const onDrop = jest.fn(); + + await fs.start(onFrame, onDrop); + + // first frame (starts processing) + callback(1, 640, 480); + + // second frame arrives while busy + callback(2, 640, 480); + + expect(onDrop).toHaveBeenCalled(); + }); + + it('should stop and clean up resources', async () => { + const {VideoSource} = require('./videoSource'); + + const stopMock = jest.fn(); + + VideoSource.mockImplementation(() => ({ + element: {}, + start: jest.fn().mockResolvedValue(undefined), + stop: stopMock, + })); + + const fs = new FrameSource(mockTrack); + + await fs.start(jest.fn()); + + await fs.stop(); + + expect(stopMock).toHaveBeenCalled(); + }); + + it('should not start twice', async () => { + const fs = new FrameSource(mockTrack); + const onFrame = jest.fn(); + + await fs.start(onFrame); + await fs.start(onFrame); + + // no crash / doppelte init + expect(true).toBe(true); + }); +}); diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/effects/frameSource.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/frameSource.ts new file mode 100644 index 00000000000..9c4c40ff1de --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/frameSource.ts @@ -0,0 +1,246 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Frame source adapter for MediaStreamTrack input. + * + * This class provides a unified interface for extracting frames from a + * MediaStreamTrack. It prefers WebCodecs MediaStreamTrackProcessor with + * backpressure (single frame in flight) and falls back to HTMLVideoElement + * callbacks when unavailable. + */ + +import {getLogger, Logger} from 'Util/logger'; + +import {VideoSource} from './videoSource'; + +/** + * Callback function for processing extracted video frames. + * + * Invoked for each frame extracted from the MediaStreamTrack. The frame + * is provided as an ImageBitmap along with its timestamp and dimensions. + * The callback can be async to allow for frame processing. + * + * @param frame - Video frame as ImageBitmap (caller will close after callback). + * @param timestampSeconds - Frame timestamp in seconds. + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + * @returns Promise or void (async processing is supported). + */ +export type FrameCallback = ( + frame: ImageBitmap, + timestampSeconds: number, + width: number, + height: number, +) => Promise | void; + +export class FrameSource { + private readonly logger: Logger; + private processor: any | null = null; + private processorAbort: AbortController | null = null; + private processorReader: ReadableStreamDefaultReader | null = null; + private videoSource: VideoSource | null = null; + private processing = false; + private running = false; + + constructor(private readonly track: MediaStreamTrack) { + this.logger = getLogger('FrameSource'); + } + + /** + * Starts frame extraction from the MediaStreamTrack. + * + * Prefers WebCodecs MediaStreamTrackProcessor if available (better performance + * and backpressure handling), otherwise falls back to HTMLVideoElement-based + * extraction. Only one extraction can be active at a time. + * + * @param onFrame - Callback invoked for each extracted frame. + * @param onDrop - Optional callback invoked when frames are dropped (video element fallback only). + * @returns Promise that resolves when extraction starts. + */ + public async start(onFrame: FrameCallback, onDrop?: () => void): Promise { + if (this.running) { + return; + } + this.running = true; + + const Processor = (window as any)?.MediaStreamTrackProcessor; + if (Processor) { + await this.startWithProcessor(Processor, onFrame); + return; + } + + await this.startWithVideoElement(onFrame, onDrop); + } + + /** + * Stops frame extraction and releases resources. + * + * Cancels any active processor readers, stops video element extraction, + * and clears all references. Safe to call multiple times. + * + * @returns Nothing. + */ + public async stop(): Promise { + this.running = false; + if (this.processorAbort) { + this.processorAbort.abort(); + this.processorAbort = null; + } + if (this.processorReader) { + try { + await this.processorReader.cancel(); + } catch (error) { + this.logger.warn('FrameSource cancel failed', error); + } + this.processorReader = null; + } + this.processor = null; + this.videoSource?.stop(); + this.videoSource = null; + this.processing = false; + } + + /** + * Starts frame extraction using WebCodecs MediaStreamTrackProcessor. + * + * Creates a processor and reads frames from its readable stream. Converts + * VideoFrames to ImageBitmaps and invokes the callback. Handles backpressure + * naturally through the stream API. Falls back to video element if processor + * creation fails. + * + * @param Processor - MediaStreamTrackProcessor constructor. + * @param onFrame - Callback invoked for each extracted frame. + * @returns Promise that resolves when extraction starts. + */ + private async startWithProcessor( + Processor: new (opts: {track: MediaStreamTrack}) => {readable: ReadableStream}, + onFrame: FrameCallback, + ): Promise { + try { + this.processor = new Processor({track: this.track}); + } catch (error) { + this.logger.warn('FrameSource processor init failed, falling back to video element', error); + await this.startWithVideoElement(onFrame); + return; + } + + const readable = this.processor.readable; + const abortController = new AbortController(); + this.processorAbort = abortController; + const reader = readable.getReader(); + this.processorReader = reader; + + void (async () => { + try { + while (this.running && !abortController.signal.aborted) { + const result = await reader.read(); + if (result.done) { + break; + } + const frame = result.value; + let bitmap: ImageBitmap | null = null; + try { + if (!this.running) { + frame.close(); + continue; + } + bitmap = await createImageBitmap(frame); + const timestampSeconds = Number.isFinite(frame.timestamp) + ? frame.timestamp / 1_000_000 + : performance.now() / 1000; + const width = bitmap.width; + const height = bitmap.height; + await onFrame(bitmap, timestampSeconds, width, height); + } catch (error) { + if (bitmap) { + try { + bitmap.close(); + } catch { + // Ignore bitmap close errors. + } + } + this.logger.warn('FrameSource processor frame failed', error); + } finally { + frame.close(); + } + } + } catch (error: unknown) { + if (abortController.signal.aborted || (error instanceof DOMException && error?.name === 'AbortError')) { + return; + } + this.logger.warn('FrameSource processor pipe failed', error); + } finally { + try { + reader.releaseLock(); + } catch { + // Ignore release errors. + } + if (this.processorReader === reader) { + this.processorReader = null; + } + } + })(); + } + + /** + * Starts frame extraction using HTMLVideoElement fallback. + * + * Creates a hidden video element, attaches the track, and uses + * requestVideoFrameCallback (preferred) or requestAnimationFrame (fallback) + * to extract frames. Uses a processing flag to prevent concurrent frame + * processing and calls onDrop when frames are dropped. + * + * @param onFrame - Callback invoked for each extracted frame. + * @param onDrop - Optional callback invoked when frames are dropped. + * @returns Promise that resolves when extraction starts. + */ + private async startWithVideoElement(onFrame: FrameCallback, onDrop?: () => void): Promise { + this.videoSource = new VideoSource(this.track); + await this.videoSource.start(async (timestamp, width, height) => { + if (!this.running) { + return; + } + if (!this.videoSource) { + return; + } + if (this.processing) { + onDrop?.(); + return; + } + this.processing = true; + let bitmap: ImageBitmap | null = null; + try { + bitmap = await createImageBitmap(this.videoSource.element); + await onFrame(bitmap, timestamp, bitmap.width, bitmap.height); + } catch (error) { + if (bitmap) { + try { + bitmap.close(); + } catch { + // Ignore bitmap close errors. + } + } + this.logger.warn('FrameSource video element frame failed', error); + } finally { + this.processing = false; + } + }); + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/effects/videoSource.test.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/videoSource.test.ts new file mode 100644 index 00000000000..3664a4523d5 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/videoSource.test.ts @@ -0,0 +1,506 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {VideoSource} from './videoSource'; + +describe('VideoSource', () => { + const originalMediaStream = global.MediaStream; + let mockTrack: MediaStreamTrack; + let mockVideoElement: HTMLVideoElement; + let mockMediaStream: MediaStream; + let createElementSpy: jest.SpyInstance; + let requestAnimationFrameSpy: jest.SpyInstance; + let cancelAnimationFrameSpy: jest.SpyInstance; + + beforeEach(() => { + // Mock MediaStreamTrack + mockTrack = { + id: 'test-track-id', + kind: 'video', + } as MediaStreamTrack; + + // Mock MediaStream + mockMediaStream = { + getVideoTracks: jest.fn().mockReturnValue([mockTrack]), + } as any; + + // Mock HTMLVideoElement + mockVideoElement = { + autoplay: false, + muted: false, + playsInline: false, + srcObject: null, + videoWidth: 640, + videoHeight: 480, + currentTime: 0, + paused: true, + ended: false, + play: jest.fn().mockResolvedValue(undefined), + pause: jest.fn(), + load: jest.fn(), + } as any; + + // Make read-only properties writable for testing + Object.defineProperty(mockVideoElement, 'videoWidth', { + writable: true, + configurable: true, + value: 640, + }); + Object.defineProperty(mockVideoElement, 'videoHeight', { + writable: true, + configurable: true, + value: 480, + }); + Object.defineProperty(mockVideoElement, 'paused', { + writable: true, + configurable: true, + value: true, + }); + Object.defineProperty(mockVideoElement, 'ended', { + writable: true, + configurable: true, + value: false, + }); + + // Mock document.createElement + createElementSpy = jest.spyOn(document, 'createElement').mockReturnValue(mockVideoElement); + + // Mock MediaStream constructor + global.MediaStream = jest.fn().mockImplementation(() => mockMediaStream) as any; + + // Mock requestAnimationFrame + let rafIdCounter = 1; + requestAnimationFrameSpy = jest + .spyOn(window, 'requestAnimationFrame') + .mockImplementation((callback: FrameRequestCallback) => { + return rafIdCounter++ as any; + }); + + // Mock cancelAnimationFrame + cancelAnimationFrameSpy = jest.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + if (originalMediaStream !== undefined) { + global.MediaStream = originalMediaStream; + } else { + delete (global as any).MediaStream; + } + }); + + describe('constructor', () => { + it('creates video element with correct properties', () => { + const source = new VideoSource(mockTrack); + + expect(source).toBeDefined(); + expect(createElementSpy).toHaveBeenCalledWith('video'); + expect(mockVideoElement.autoplay).toBe(true); + expect(mockVideoElement.muted).toBe(true); + expect(mockVideoElement.playsInline).toBe(true); + }); + + it('sets srcObject with MediaStream containing the track', () => { + const source = new VideoSource(mockTrack); + + expect(source).toBeDefined(); + expect(global.MediaStream).toHaveBeenCalledWith([mockTrack]); + expect(mockVideoElement.srcObject).toBe(mockMediaStream); + }); + }); + + describe('element getter', () => { + it('returns the video element', () => { + const source = new VideoSource(mockTrack); + + expect(source.element).toBe(mockVideoElement); + }); + }); + + describe('start', () => { + it('calls play() on video element', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + + await source.start(onFrame); + + expect(mockVideoElement.play).toHaveBeenCalled(); + }); + + it('handles play() errors gracefully', async () => { + const onFrame = jest.fn(); + const playError = new Error('Play failed'); + mockVideoElement.play = jest.fn().mockRejectedValue(playError); + + const source = new VideoSource(mockTrack); + await source.start(onFrame); + + // Should not throw - error is caught and logged + expect(mockVideoElement.play).toHaveBeenCalled(); + }); + + it('uses requestVideoFrameCallback when available', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + let rvfcCallback: ((now: number, metadata: VideoFrameCallbackMetadata) => void) | null = null; + let rvfcHandle = 1; + + // Mock requestVideoFrameCallback + (mockVideoElement as any).requestVideoFrameCallback = jest.fn().mockImplementation((callback: any) => { + rvfcCallback = callback; + return rvfcHandle++; + }); + + await source.start(onFrame); + + expect((mockVideoElement as any).requestVideoFrameCallback).toHaveBeenCalled(); + expect(requestAnimationFrameSpy).not.toHaveBeenCalled(); + + // Simulate callback invocation + if (rvfcCallback) { + (rvfcCallback as (now: number, metadata: VideoFrameCallbackMetadata) => void)(1000, { + mediaTime: 1.5, + expectedDisplayTime: 1000, + width: 640, + height: 480, + } as any); + expect(onFrame).toHaveBeenCalledWith(1.5, 640, 480); + } + }); + + it('schedules next frame in requestVideoFrameCallback callback', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + let rvfcCallback: ((now: number, metadata: VideoFrameCallbackMetadata) => void) | null = null; + let rvfcHandle = 1; + const rvfcSpy = jest.fn().mockImplementation((callback: any) => { + rvfcCallback = callback; + return rvfcHandle++; + }); + + (mockVideoElement as any).requestVideoFrameCallback = rvfcSpy; + + await source.start(onFrame); + + // First call during start + expect(rvfcSpy).toHaveBeenCalledTimes(1); + + // Simulate callback - should schedule next frame + if (rvfcCallback) { + (rvfcCallback as (now: number, metadata: VideoFrameCallbackMetadata) => void)(1000, { + mediaTime: 1.5, + expectedDisplayTime: 1000, + width: 640, + height: 480, + } as any); + expect(rvfcSpy).toHaveBeenCalledTimes(2); + } + }); + + it('falls back to requestAnimationFrame when requestVideoFrameCallback not available', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + + // Don't add requestVideoFrameCallback to video element + await source.start(onFrame); + + expect(requestAnimationFrameSpy).toHaveBeenCalled(); + }); + + it('calls onFrame with currentTime when using requestAnimationFrame', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + let rafCallback: FrameRequestCallback | null = null; + + requestAnimationFrameSpy.mockImplementation((callback: FrameRequestCallback) => { + rafCallback = callback; + return 1 as any; + }); + + mockVideoElement.currentTime = 2.5; + + await source.start(onFrame); + + // Simulate requestAnimationFrame callback + if (rafCallback) { + (rafCallback as FrameRequestCallback)(1000); + expect(onFrame).toHaveBeenCalledWith(2.5, 640, 480); + } + }); + + it('deduplicates frames when currentTime has not changed', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + let rafCallback: FrameRequestCallback | null = null; + + requestAnimationFrameSpy.mockImplementation((callback: FrameRequestCallback) => { + rafCallback = callback; + return 1 as any; + }); + + mockVideoElement.currentTime = 2.5; + + await source.start(onFrame); + + // First call - should trigger onFrame + if (rafCallback) { + (rafCallback as FrameRequestCallback)(1000); + expect(onFrame).toHaveBeenCalledTimes(1); + } + + // Second call with same currentTime - should NOT trigger onFrame + if (rafCallback) { + (rafCallback as FrameRequestCallback)(1001); + expect(onFrame).toHaveBeenCalledTimes(1); // Still 1, not 2 + } + + // Third call with different currentTime - should trigger onFrame + mockVideoElement.currentTime = 3.0; + if (rafCallback) { + (rafCallback as FrameRequestCallback)(1002); + expect(onFrame).toHaveBeenCalledTimes(2); + expect(onFrame).toHaveBeenLastCalledWith(3.0, 640, 480); + } + }); + + it('uses videoWidth and videoHeight from video element', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + let rafCallback: FrameRequestCallback | null = null; + + requestAnimationFrameSpy.mockImplementation((callback: FrameRequestCallback) => { + rafCallback = callback; + return 1 as any; + }); + + Object.defineProperty(mockVideoElement, 'videoWidth', {value: 1280, writable: true, configurable: true}); + Object.defineProperty(mockVideoElement, 'videoHeight', {value: 720, writable: true, configurable: true}); + mockVideoElement.currentTime = 1.0; + + await source.start(onFrame); + + if (rafCallback) { + (rafCallback as FrameRequestCallback)(1000); + expect(onFrame).toHaveBeenCalledWith(1.0, 1280, 720); + } + }); + + it('uses 0 for width/height when videoWidth/videoHeight are 0', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + let rafCallback: FrameRequestCallback | null = null; + + requestAnimationFrameSpy.mockImplementation((callback: FrameRequestCallback) => { + rafCallback = callback; + return 1 as any; + }); + + Object.defineProperty(mockVideoElement, 'videoWidth', {value: 0, writable: true, configurable: true}); + Object.defineProperty(mockVideoElement, 'videoHeight', {value: 0, writable: true, configurable: true}); + mockVideoElement.currentTime = 1.0; + + await source.start(onFrame); + + if (rafCallback) { + (rafCallback as FrameRequestCallback)(1000); + expect(onFrame).toHaveBeenCalledWith(1.0, 0, 0); + } + }); + + it('continues requestAnimationFrame loop', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + let rafCallback: FrameRequestCallback | null = null; + let rafCallCount = 0; + + requestAnimationFrameSpy.mockImplementation((callback: FrameRequestCallback) => { + rafCallback = callback; + rafCallCount++; + return rafCallCount as any; + }); + + await source.start(onFrame); + + // Initial call + expect(rafCallCount).toBe(1); + + // Simulate multiple animation frames + if (rafCallback) { + mockVideoElement.currentTime = 1.0; + (rafCallback as FrameRequestCallback)(1000); + expect(rafCallCount).toBe(2); // Should schedule next frame + + mockVideoElement.currentTime = 2.0; + (rafCallback as FrameRequestCallback)(1001); + expect(rafCallCount).toBe(3); // Should schedule next frame + } + }); + }); + + describe('stop', () => { + it('cancels requestVideoFrameCallback when active', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + const rvfcHandle = 1; + const cancelRvfcSpy = jest.fn(); + + (mockVideoElement as any).requestVideoFrameCallback = jest.fn().mockReturnValue(rvfcHandle); + (mockVideoElement as any).cancelVideoFrameCallback = cancelRvfcSpy; + + await source.start(onFrame); + source.stop(); + + expect(cancelRvfcSpy).toHaveBeenCalledWith(rvfcHandle); + }); + + it('does not call cancelVideoFrameCallback when rVFCHandle is null', () => { + const source = new VideoSource(mockTrack); + const cancelRvfcSpy = jest.fn(); + + (mockVideoElement as any).cancelVideoFrameCallback = cancelRvfcSpy; + + source.stop(); + + expect(cancelRvfcSpy).not.toHaveBeenCalled(); + }); + + it('cancels requestAnimationFrame when active', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + const rafId = 123; + + requestAnimationFrameSpy.mockReturnValue(rafId as any); + + await source.start(onFrame); + source.stop(); + + expect(cancelAnimationFrameSpy).toHaveBeenCalledWith(rafId); + }); + + it('does not call cancelAnimationFrame when rafId is null', () => { + const source = new VideoSource(mockTrack); + + source.stop(); + + expect(cancelAnimationFrameSpy).not.toHaveBeenCalled(); + }); + + it('pauses video if not paused and not ended', () => { + const source = new VideoSource(mockTrack); + Object.defineProperty(mockVideoElement, 'paused', {value: false, writable: true, configurable: true}); + Object.defineProperty(mockVideoElement, 'ended', {value: false, writable: true, configurable: true}); + + source.stop(); + + expect(mockVideoElement.pause).toHaveBeenCalled(); + }); + + it('does not pause video if already paused', () => { + const source = new VideoSource(mockTrack); + Object.defineProperty(mockVideoElement, 'paused', {value: true, writable: true, configurable: true}); + Object.defineProperty(mockVideoElement, 'ended', {value: false, writable: true, configurable: true}); + + source.stop(); + + expect(mockVideoElement.pause).not.toHaveBeenCalled(); + }); + + it('does not pause video if already ended', () => { + const source = new VideoSource(mockTrack); + Object.defineProperty(mockVideoElement, 'paused', {value: false, writable: true, configurable: true}); + Object.defineProperty(mockVideoElement, 'ended', {value: true, writable: true, configurable: true}); + + source.stop(); + + expect(mockVideoElement.pause).not.toHaveBeenCalled(); + }); + + it('sets srcObject to null', () => { + const source = new VideoSource(mockTrack); + + source.stop(); + + expect(mockVideoElement.srcObject).toBeNull(); + }); + + it('calls load() on video element', () => { + const source = new VideoSource(mockTrack); + + source.stop(); + + expect(mockVideoElement.load).toHaveBeenCalled(); + }); + + it('resets rVFCHandle to null', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + const cancelRvfcSpy = jest.fn(); + + (mockVideoElement as any).requestVideoFrameCallback = jest.fn().mockReturnValue(1); + (mockVideoElement as any).cancelVideoFrameCallback = cancelRvfcSpy; + + await source.start(onFrame); + source.stop(); + + expect(cancelRvfcSpy).toHaveBeenCalledTimes(1); + + // Call stop again - should not try to cancel again (rVFCHandle is now null) + cancelRvfcSpy.mockClear(); + source.stop(); + + expect(cancelRvfcSpy).not.toHaveBeenCalled(); + }); + + it('resets rafId to null', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + + requestAnimationFrameSpy.mockReturnValue(123 as any); + + await source.start(onFrame); + source.stop(); + + // Call stop again - should not try to cancel again + cancelAnimationFrameSpy.mockClear(); + source.stop(); + + expect(cancelAnimationFrameSpy).not.toHaveBeenCalled(); + }); + + it('performs all cleanup operations', async () => { + const source = new VideoSource(mockTrack); + const onFrame = jest.fn(); + + // Use requestAnimationFrame path (not requestVideoFrameCallback) + // Delete requestVideoFrameCallback so 'in' check returns false + delete (mockVideoElement as any).requestVideoFrameCallback; + requestAnimationFrameSpy.mockReturnValue(123 as any); + Object.defineProperty(mockVideoElement, 'paused', {value: false, writable: true, configurable: true}); + Object.defineProperty(mockVideoElement, 'ended', {value: false, writable: true, configurable: true}); + + await source.start(onFrame); + source.stop(); + + expect(cancelAnimationFrameSpy).toHaveBeenCalledWith(123); + expect(mockVideoElement.pause).toHaveBeenCalled(); + expect(mockVideoElement.srcObject).toBeNull(); + expect(mockVideoElement.load).toHaveBeenCalled(); + }); + }); +}); diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/effects/videoSource.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/videoSource.ts new file mode 100644 index 00000000000..aa646ffacf4 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/effects/videoSource.ts @@ -0,0 +1,180 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Video source wrapper for extracting frames from MediaStreamTrack. + * + * This class provides a convenient interface for processing video frames from + * a MediaStreamTrack (e.g., from getUserMedia or RTCPeerConnection). It: + * - Creates and manages an HTMLVideoElement for playback + * - Provides frame callbacks using the most efficient available API + * - Prefers requestVideoFrameCallback (better timing) over requestAnimationFrame (fallback) + * - Handles video element lifecycle (play, pause, cleanup) + */ + +import {getLogger, Logger} from 'Util/logger'; + +/** + * Callback function invoked for each video frame. + * + * @param timestamp - Frame timestamp in seconds (mediaTime for rVFC, currentTime for RAF). + * @param width - Video width in pixels. + * @param height - Video height in pixels. + */ +export type FrameCallback = (timestamp: number, width: number, height: number) => void; + +/** + * Wrapper around HTMLVideoElement for frame extraction from MediaStreamTrack. + * + * This class manages a hidden video element that plays a MediaStreamTrack and + * provides callbacks for each frame. It automatically selects the best available + * frame callback API: + * - requestVideoFrameCallback: Preferred, provides accurate media timestamps + * - requestAnimationFrame: Fallback, uses currentTime (less accurate but widely supported) + */ +export class VideoSource { + /** Logger instance for debugging and warnings. */ + private readonly logger: Logger; + /** Hidden video element used for frame extraction. */ + private readonly videoEl: HTMLVideoElement; + /** RequestAnimationFrame handle (used when rVFC unavailable). */ + private rafId: number | null = null; + /** RequestVideoFrameCallback handle (preferred method). */ + private rVFCHandle: number | null = null; + /** Last processed frame time (for RAF deduplication). */ + private lastTime = -1; + + /** + * Creates a new video source from a MediaStreamTrack. + * + * Sets up a hidden video element with autoplay, muted, and playsInline attributes + * to ensure smooth playback without user interaction. The track is attached to + * the video element via a MediaStream. + * + * @param track - MediaStreamTrack to extract frames from (e.g., from getUserMedia). + */ + constructor(track: MediaStreamTrack) { + this.logger = getLogger('VideoSource'); + this.videoEl = document.createElement('video'); + // Configure for automatic playback without user interaction + this.videoEl.autoplay = true; + this.videoEl.muted = true; // Required for autoplay in most browsers + this.videoEl.playsInline = true; // Prevents fullscreen on mobile + // Attach track to video element + this.videoEl.srcObject = new MediaStream([track]); + } + + /** + * Gets the underlying HTMLVideoElement. + * + * Useful for direct access to video properties or for attaching event listeners. + * The element is configured for automatic playback and is hidden from the user. + * + * @returns The video element instance. + */ + public get element(): HTMLVideoElement { + return this.videoEl; + } + + /** + * Starts frame extraction and invokes callback for each frame. + * + * This method: + * 1. Starts video playback + * 2. Selects the best available frame callback API + * 3. Invokes the callback for each frame with timestamp and dimensions + * + * Frame callback selection: + * - **requestVideoFrameCallback** (preferred): Provides accurate mediaTime timestamps, + * synchronized with video playback. More efficient and accurate than RAF. + * - **requestAnimationFrame** (fallback): Uses currentTime, less accurate but widely + * supported. Includes deduplication to avoid duplicate frames. + * + * @param onFrame - Callback function invoked for each video frame. + */ + public async start(onFrame: FrameCallback): Promise { + // Start video playback (required for frame extraction) + await this.videoEl.play().catch((error: unknown) => this.logger.warn('VideoSource play failed', error)); + + // Helper functions to get current video dimensions + const width = () => this.videoEl.videoWidth || 0; + const height = () => this.videoEl.videoHeight || 0; + + // Prefer requestVideoFrameCallback (more accurate timestamps) + if ('requestVideoFrameCallback' in this.videoEl) { + const callback = (now: number, metadata: VideoFrameCallbackMetadata) => { + // Use mediaTime for accurate frame timing + onFrame(metadata.mediaTime, width(), height()); + // Schedule next frame callback (rVFC requires re-registration) + this.rVFCHandle = (this.videoEl as any).requestVideoFrameCallback(callback); + }; + // Start the callback chain + this.rVFCHandle = (this.videoEl as any).requestVideoFrameCallback(callback); + return; + } + + // Fallback to requestAnimationFrame (less accurate but widely supported) + const rafLoop = () => { + const currentTime = this.videoEl.currentTime; + // Deduplicate: only process if time has changed (avoid duplicate frames) + if (currentTime !== this.lastTime) { + this.lastTime = currentTime; + // Use currentTime as timestamp (less accurate than mediaTime) + onFrame(currentTime, width(), height()); + } + // Schedule next frame check + this.rafId = window.requestAnimationFrame(rafLoop); + }; + + // Start the RAF loop + this.rafId = window.requestAnimationFrame(rafLoop); + } + + /** + * Stops frame extraction and cleans up resources. + * + * This method: + * 1. Cancels any active frame callbacks (rVFC or RAF) + * 2. Pauses video playback if active + * 3. Clears the video source and resets the element + * + * Should be called when the video source is no longer needed to prevent + * memory leaks and stop frame processing. + */ + public stop(): void { + // Cancel requestVideoFrameCallback if active + if (this.rVFCHandle !== null && 'cancelVideoFrameCallback' in this.videoEl) { + (this.videoEl as any).cancelVideoFrameCallback(this.rVFCHandle); + this.rVFCHandle = null; + } + // Cancel requestAnimationFrame if active + if (this.rafId !== null) { + window.cancelAnimationFrame(this.rafId); + this.rafId = null; + } + + // Pause video playback if still playing + if (!this.videoEl.paused && !this.videoEl.ended) { + this.videoEl.pause(); + } + // Clear video source and reset element state + this.videoEl.srcObject = null; + this.videoEl.load(); + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/index.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/index.ts new file mode 100644 index 00000000000..7551934c250 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/index.ts @@ -0,0 +1,105 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Detects browser capabilities required for background effects. + * + * @returns Capability information with boolean flags for each required API. + * @example + * ```typescript + * const caps = detectCapabilities(); + * if (caps.webgl2 && caps.worker) { + * // Can use worker-based pipeline + * } + * ``` + */ +export {detectCapabilities} from './effects/capability'; + +/** + * Selects the optimal rendering pipeline based on browser capabilities. + * + * @param cap - Capability information from detectCapabilities(). + * @param preferWorker - If true, prefers worker-based pipeline when available. + * @returns Selected pipeline identifier. + * @example + * ```typescript + * const caps = detectCapabilities(); + * const pipeline = choosePipeline(caps, true); + * ``` + */ +export {choosePipeline} from './effects/capability'; + +/** + * Video source wrapper for extracting frames from MediaStreamTrack. + * + * Provides frame callbacks using requestVideoFrameCallback (preferred) or + * requestAnimationFrame (fallback). Used internally by BackgroundEffectsController. + */ +export {VideoSource} from './effects/videoSource'; + +/** + * Effect mode type ('blur', 'virtual', or 'passthrough'). + */ +export type {EffectMode} from './backgroundEffectsWorkerTypes'; + +/** + * Debug visualization mode type. + */ +export type {DebugMode} from './backgroundEffectsWorkerTypes'; + +export type {QualityTier} from './backgroundEffectsWorkerTypes'; + +/** + * Quality mode type ('auto' or fixed QualityTier). + */ +export type {QualityMode} from './backgroundEffectsWorkerTypes'; + +/** + * BackgroundEffectsRenderingPipeline selection type. + */ +export type {PipelineType} from './backgroundEffectsWorkerTypes'; + +/** + * Configuration options for starting the background effects pipeline. + */ +export type {StartOptions} from './backgroundEffectsWorkerTypes'; + +/** + * Performance metrics tracked during frame processing. + */ +export type {Metrics} from './backgroundEffectsWorkerTypes'; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/backgroundEffectsRenderingPipeline.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/backgroundEffectsRenderingPipeline.ts new file mode 100644 index 00000000000..d5cf0e8cf10 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/backgroundEffectsRenderingPipeline.ts @@ -0,0 +1,189 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type { + DebugMode, + EffectMode, + Metrics, + PipelineType, + QualityMode, + QualityTier, + SegmentationModelByTier, +} from '../backgroundEffectsWorkerTypes'; +import type {MaskPostProcessorFactory} from '../segmentation/maskPostProcessor'; +import type {SegmenterFactory} from '../segmentation/segmenterTypes'; + +/** + * Runtime configuration for a pipeline instance. + * + * These values can be updated at runtime via updateConfig() without + * reinitializing the pipeline. + */ +export interface PipelineConfig { + /** Effect mode ('blur', 'virtual', or 'passthrough'). */ + mode: EffectMode; + /** Debug visualization mode. */ + debugMode: DebugMode; + /** Blur strength (0-1) for blur effect mode. */ + blurStrength: number; + /** Quality mode */ + quality: QualityMode; +} + +/** + * Initialization parameters for a pipeline instance. + * + * All fields are required except for optional factories used for testing. + * The pipeline uses these values during initialization to set up resources, + * segmenters, and quality controllers. + */ +export interface PipelineInit { + /** Output canvas for rendering processed frames. */ + outputCanvas: HTMLCanvasElement; + /** Target frames per second for adaptive quality control. */ + targetFps: number; + /** Path to MediaPipe segmentation model file. */ + segmentationModelPath: string; + /** Per-tier segmentation model path overrides. */ + segmentationModelByTier: SegmentationModelByTier; + /** Initial quality tier when quality mode is 'auto'. */ + initialTier: QualityTier; + /** Runtime configuration for the pipeline. */ + config: PipelineConfig; + /** Optional factory for creating segmenter instances (for testing). */ + createSegmenter?: SegmenterFactory; + /** Optional factory for creating mask post-processors (for testing). */ + createMaskPostProcessor?: MaskPostProcessorFactory; + /** Optional callback to receive performance metrics updates. */ + onMetrics: ((metrics: Metrics) => void) | null; + /** Callback invoked when quality tier changes (adaptive quality mode). */ + onTierChange: (tier: QualityTier) => void; + /** Callback invoked when a frame is dropped, returns dropped frame count. */ + onDroppedFrame: () => number; + /** Function that returns the current dropped frame count. */ + getDroppedFrames: () => number; + /** Optional callback invoked when worker segmenter initialization fails. */ + onWorkerSegmenterError?: (error: string) => void; + /** Optional callback invoked when worker WebGL context is lost. */ + onWorkerContextLoss?: () => void; + maxTier: QualityTier | null; +} + +/** + * Interface for background effects rendering pipelines. + * + * Pipelines process video frames through segmentation and rendering stages + * to produce output frames with background effects applied. Different pipeline + * implementations use different rendering backends (WebGL2, Canvas2D, Worker) + * and have different performance characteristics. + * + * All pipelines must implement this interface to be used by BackgroundEffectsController. + */ +export interface BackgroundEffectsRenderingPipeline { + /** BackgroundEffectsRenderingPipeline type identifier. */ + readonly type: PipelineType; + /** + * Initializes the pipeline with configuration and resources. + * + * Sets up rendering contexts, segmenters, quality controllers, and other + * resources needed for frame processing. Must be called before processFrame(). + * + * @param init - BackgroundEffectsRenderingPipeline initialization parameters. + * @returns Promise that resolves when initialization is complete. + */ + init(init: PipelineInit): Promise; + /** + * Processes a single video frame through the pipeline. + * + * Performs segmentation (if cadence allows), applies mask post-processing, + * and renders the frame with background effects applied to the output canvas. + * + * @param frame - Input video frame as ImageBitmap (will be closed after processing). + * @param timestamp - Frame timestamp in seconds. + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + * @returns Promise that resolves when frame processing is complete. + */ + processFrame(frame: ImageBitmap, timestamp: number, width: number, height: number): Promise; + /** + * Updates runtime configuration without reinitializing. + * + * Updates effect mode, debug mode, blur strength, and quality settings. + * Some pipelines may need to update internal state (e.g., quality controller tier). + * + * @param config - New pipeline configuration. + */ + updateConfig(config: PipelineConfig): void; + /** + * Sets a static background image for virtual background mode. + * + * The bitmap is stored and used for all subsequent frames until cleared + * or replaced. For worker pipelines, the bitmap is transferred (not cloned). + * + * @param bitmap - Background image as ImageBitmap. + * @param width - Image width in pixels. + * @param height - Image height in pixels. + */ + setBackgroundImage(bitmap: ImageBitmap, width: number, height: number): void; + /** + * Sets a video frame as background for virtual background mode. + * + * Similar to setBackgroundImage but intended for video backgrounds that + * update each frame. The bitmap is stored and used for subsequent frames. + * + * @param bitmap - Background video frame as ImageBitmap. + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + */ + setBackgroundVideoFrame(bitmap: ImageBitmap, width: number, height: number): void; + /** + * Clears the background source. + * + * Releases any stored background image/video and resets background state. + */ + clearBackground(): void; + /** + * Notifies the pipeline of dropped frames for metrics tracking. + * + * Used by worker pipelines to sync dropped frame counts from main thread. + * Main-thread pipelines may ignore this (no-op). + * + * @param count - Number of dropped frames. + */ + notifyDroppedFrames(count: number): void; + /** + * Returns whether the output canvas has been transferred to a worker. + * + * Used by BackgroundEffectsController to determine if canvas resizing + * should be skipped (transferred canvases cannot be resized from main thread). + * + * @returns True if canvas is transferred to worker, false otherwise. + */ + isOutputCanvasTransferred(): boolean; + /** + * Stops the pipeline and releases all resources. + * + * Closes segmenters, destroys renderers, releases background sources, + * and clears all references. Should be called when the pipeline is no + * longer needed to prevent memory leaks. + */ + stop(): void; + + getCurrentModelPath(): string | null; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/canvas2dPipeline.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/canvas2dPipeline.ts new file mode 100644 index 00000000000..f4045e38540 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/canvas2dPipeline.ts @@ -0,0 +1,829 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {getLogger, Logger} from 'Util/logger'; + +import type { + BackgroundEffectsRenderingPipeline, + PipelineConfig, + PipelineInit, +} from './backgroundEffectsRenderingPipeline'; + +import type {QualityTier, QualityTierParams, SegmentationModelByTier} from '../backgroundEffectsWorkerTypes'; +import { + buildMetrics, + computeBlurRadius, + createMetricsWindow, + isProcessingMode, + pushMetricsSample, + QualityController, + resetMetricsWindow, + resolveQualityTierForEffectMode, + resolveSegmentationModelPath, +} from '../quality'; +import type {MaskPostProcessor} from '../segmentation/maskPostProcessor'; +import {NoopMaskPostProcessor} from '../segmentation/maskPostProcessor'; +import {MediaPipeSegmenterFactory} from '../segmentation/mediaPipeSegmenter'; +import type {SegmenterFactory, SegmenterLike} from '../segmentation/segmenterTypes'; + +/** + * Canvas2D-based rendering pipeline for background effects. + * + * This pipeline uses CPU-based Canvas2D API for rendering, making it a + * fallback option when WebGL2 is unavailable. It performs segmentation + * using CPU delegate and renders effects using Canvas2D compositing operations. + * + * Performance characteristics: + * - Lower quality than WebGL pipelines (no GPU acceleration) + * - Runs on main thread (may impact UI responsiveness) + * - Widely supported across browsers + * - Uses CPU-based blur filter (less efficient than GPU blur) + * + * Rendering approach: + * - Segmentation: CPU-based ML inference + * - Background blur: Canvas2D filter API + * - Virtual background: Canvas2D drawImage with compositing + * - Mask compositing: destination-in blend mode + */ +export class Canvas2dPipeline implements BackgroundEffectsRenderingPipeline { + public readonly type = 'canvas2d' as const; + private readonly logger: Logger; + private outputCanvas: HTMLCanvasElement | null = null; + private canvasCtx: CanvasRenderingContext2D | null = null; + private foregroundCanvas: HTMLCanvasElement | null = null; + private foregroundCtx: CanvasRenderingContext2D | null = null; + private debugCanvas: HTMLCanvasElement | null = null; + private debugCtx: CanvasRenderingContext2D | null = null; + private maskCanvas: HTMLCanvasElement | null = null; + private maskCtx: CanvasRenderingContext2D | null = null; + private maskScratchCanvas: HTMLCanvasElement | null = null; + private maskScratchCtx: CanvasRenderingContext2D | null = null; + private hasMask = false; + private segmenter: SegmenterLike | null = null; + private segmenterFactory: SegmenterFactory = MediaPipeSegmenterFactory; + private segmentationModelByTier: SegmentationModelByTier = {}; + private currentModelPath: string | null = null; + private maskPostProcessor: MaskPostProcessor = new NoopMaskPostProcessor(); + private qualityController: QualityController | null = null; + private config: PipelineConfig | null = null; + private onMetrics: PipelineInit['onMetrics'] = null; + private onTierChange: PipelineInit['onTierChange'] | null = null; + private getDroppedFrames: PipelineInit['getDroppedFrames'] | null = null; + private readonly metricsMaxSamples = 30; + private readonly metricsWindow = createMetricsWindow(this.metricsMaxSamples); + private mainFrameCount = 0; + private canvasFrameToken = 0; + private canvasPassthroughLogged = false; + private background: {bitmap: ImageBitmap; width: number; height: number} | null = null; + + constructor() { + this.logger = getLogger('Canvas2dPipeline'); + } + + /** + * Initializes the Canvas2D pipeline. + * + * Sets up Canvas2D contexts (output, foreground, debug), initializes + * segmenter with CPU delegate, quality controller, and mask post-processor. + * Creates foreground and debug canvases for compositing operations. + * + * If segmentation initialization fails, the pipeline will run in passthrough + * mode (no effects applied). + * + * @param init - BackgroundEffectsRenderingPipeline initialization parameters. + * @throws Error if Canvas2D context is unavailable. + */ + public async init(init: PipelineInit): Promise { + this.outputCanvas = init.outputCanvas; + this.config = init.config; + this.onMetrics = init.onMetrics; + this.onTierChange = init.onTierChange; + this.getDroppedFrames = init.getDroppedFrames; + this.mainFrameCount = 0; + this.canvasFrameToken = 0; + this.canvasPassthroughLogged = false; + this.maskCanvas = null; + this.maskCtx = null; + this.maskScratchCanvas = null; + this.maskScratchCtx = null; + this.hasMask = false; + + const ctx = this.outputCanvas.getContext('2d'); + if (!ctx) { + throw new Error('Canvas2D context unavailable'); + } + this.canvasCtx = ctx; + this.foregroundCanvas = document.createElement('canvas'); + this.foregroundCanvas.width = this.outputCanvas.width; + this.foregroundCanvas.height = this.outputCanvas.height; + this.foregroundCtx = this.foregroundCanvas.getContext('2d'); + this.debugCanvas = document.createElement('canvas'); + this.debugCanvas.width = this.outputCanvas.width; + this.debugCanvas.height = this.outputCanvas.height; + this.debugCtx = this.debugCanvas.getContext('2d'); + + this.segmenterFactory = init.createSegmenter ?? MediaPipeSegmenterFactory; + this.segmentationModelByTier = init.segmentationModelByTier; + const postProcessorFactory = init.createMaskPostProcessor ?? { + create: () => new NoopMaskPostProcessor(), + }; + this.maskPostProcessor = postProcessorFactory.create(); + this.qualityController = new QualityController(init.targetFps); + if (this.qualityController && this.config?.quality === 'auto') { + this.qualityController.setTier(init.initialTier); + } else if (this.qualityController && this.config?.quality !== 'auto') { + this.qualityController.setTier(this.config.quality); + } + const startingTier = init.config.quality === 'auto' ? init.initialTier : init.config.quality; + const shouldInitSegmenter = startingTier !== 'bypass' && init.config.mode !== 'passthrough'; + if (shouldInitSegmenter) { + this.segmenter = this.segmenterFactory.create({ + modelPath: init.segmentationModelPath, + delegate: 'CPU', + }); + this.currentModelPath = init.segmentationModelPath; + try { + await this.segmenter.init(); + } catch (error) { + this.logger.warn('Segmentation init failed, canvas2d will pass through', error); + this.segmenter = null; + } + } else { + this.segmenter = null; + this.currentModelPath = null; + } + } + + /** + * Updates pipeline configuration at runtime. + * + * Updates internal config and quality controller tier if quality + * mode is fixed (not 'auto'). Adaptive quality updates happen + * automatically during frame processing. + * + * @param config - New pipeline configuration. + */ + public updateConfig(config: PipelineConfig): void { + this.config = config; + if (this.qualityController && config.quality !== 'auto') { + this.qualityController.setTier(config.quality); + } + } + + /** + * Processes a video frame through the Canvas2D pipeline. + * + * Processing steps: + * 1. Checks for bypass mode (passthrough or no segmenter) and returns early if needed + * 2. Resolves quality tier parameters (adaptive or fixed) + * 3. Checks quality tier bypass and returns early if needed + * 4. Ensures segmenter is initialized for current tier + * 5. Captures frame index before incrementing for cadence calculation + * 6. Performs segmentation (if cadence allows, respecting frame token) + * 7. Applies mask post-processing + * 8. Applies temporal mask smoothing via updateMaskCanvas() (if mask available) + * 9. Renders background (blur using Canvas2D filter or virtual background) + * 10. Composites foreground using temporally-smoothed mask with destination-in blend mode + * 11. Updates quality controller inline (if in 'auto' mode) and performance metrics + * + * Supports debug visualization modes (maskOverlay, maskOnly, edgeOnly, + * classOverlay, classOnly). Handles frame token validation to prevent + * race conditions when frames arrive faster than processing. Uses temporally-smoothed + * masks (maskForEffect) instead of raw masks for better visual stability. + * + * @param frame - Input video frame as ImageBitmap (will be closed after processing). + * @param _timestamp - Frame timestamp (unused, uses performance.now() for segmentation). + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + * @returns Promise that resolves when frame processing is complete. + */ + public async processFrame(frame: ImageBitmap, _timestamp: number, width: number, height: number): Promise { + if (!this.outputCanvas || !this.canvasCtx || !this.config) { + frame.close(); + return; + } + const ctx = this.canvasCtx; + this.foregroundCanvas!.width = width; + this.foregroundCanvas!.height = height; + if (this.debugCanvas) { + this.debugCanvas.width = width; + this.debugCanvas.height = height; + } + ctx.clearRect(0, 0, width, height); + + if (this.config.mode === 'passthrough' || !this.segmenter || !this.qualityController) { + if (!this.segmenter && !this.canvasPassthroughLogged) { + this.logger.warn('Canvas2D pipeline running without segmenter; output will be passthrough'); + this.canvasPassthroughLogged = true; + } + ctx.drawImage(frame, 0, 0, width, height); + frame.close(); + return; + } + + const qualityTier = this.resolveQuality(); + if (qualityTier.bypass) { + ctx.drawImage(frame, 0, 0, width, height); + frame.close(); + return; + } + if (!qualityTier.bypass) { + await this.ensureSegmenterForTier(qualityTier.tier); + } + const frameIndex = this.mainFrameCount; + this.mainFrameCount += 1; + const token = ++this.canvasFrameToken; + + let result: + | {mask: ImageBitmap | null; classMask: ImageBitmap | null; durationMs: number; release: () => void} + | {mask: null; classMask: ImageBitmap | null; durationMs: number; release: () => void}; + if (qualityTier.segmentationCadence > 0 && frameIndex % qualityTier.segmentationCadence === 0) { + this.segmenter.configure(qualityTier.segmentationWidth, qualityTier.segmentationHeight); + const timestampMs = performance.now(); + const includeClassMask = this.config.debugMode === 'classOverlay' || this.config.debugMode === 'classOnly'; + const segmentation = await this.segmenter.segment(frame, timestampMs, {includeClassMask}); + const processed = await this.maskPostProcessor.process(segmentation, { + qualityTier, + mode: this.config.mode, + timestampMs, + frameSize: {width, height}, + }); + if (processed !== segmentation) { + segmentation.mask?.close(); + segmentation.classMask?.close(); + segmentation.release(); + } + result = { + mask: processed.mask, + classMask: processed.classMask, + durationMs: processed.durationMs, + release: processed.release, + }; + } else { + result = {mask: null, classMask: null, durationMs: 0, release: () => {}}; + } + + if (token !== this.canvasFrameToken) { + result.mask?.close(); + result.release(); + frame.close(); + return; + } + + const mask = result.mask; + const classMask = result.classMask; + if (mask) { + this.updateMaskCanvas(mask, width, height, qualityTier.temporalAlpha); + } + const maskForEffect = this.hasMask ? this.maskCanvas : null; + const renderStart = performance.now(); + try { + const isClassDebug = this.config.debugMode === 'classOverlay' || this.config.debugMode === 'classOnly'; + const activeMask = isClassDebug ? classMask : maskForEffect; + if (this.config.debugMode !== 'off') { + ctx.clearRect(0, 0, width, height); + if (!activeMask) { + ctx.drawImage(frame, 0, 0, width, height); + } else if (this.config.debugMode === 'maskOnly') { + ctx.drawImage(activeMask, 0, 0, width, height); + } else if (this.config.debugMode === 'maskOverlay') { + ctx.drawImage(frame, 0, 0, width, height); + ctx.globalAlpha = 0.5; + ctx.fillStyle = '#00ff00'; + ctx.globalCompositeOperation = 'source-atop'; + ctx.drawImage(activeMask, 0, 0, width, height); + ctx.globalCompositeOperation = 'source-over'; + ctx.globalAlpha = 1; + } else if (this.config.debugMode === 'edgeOnly') { + this.renderEdgeMask(ctx, activeMask, width, height); + } else if (this.config.debugMode === 'classOnly' && classMask) { + this.renderClassMask(ctx, classMask, width, height, false, frame); + } else if (this.config.debugMode === 'classOverlay' && classMask) { + this.renderClassMask(ctx, classMask, width, height, true, frame); + } + const renderMs = performance.now() - renderStart; + const totalMs = result.durationMs + renderMs; + let metricsTier = qualityTier.tier; + if (this.config.quality === 'auto' && this.qualityController && isProcessingMode(this.config.mode)) { + const updatedTier = this.qualityController.update( + {totalMs, segmentationMs: result.durationMs, gpuMs: renderMs}, + this.config.mode, + ); + metricsTier = updatedTier.tier; + this.onTierChange?.(updatedTier.tier); + } + this.updateMetrics(totalMs, result.durationMs, renderMs, metricsTier); + return; + } + + ctx.clearRect(0, 0, width, height); + const blurPx = computeBlurRadius(qualityTier, this.config.blurStrength, false); + if (this.config.mode === 'virtual' && this.background) { + ctx.drawImage(this.background.bitmap, 0, 0, this.background.width, this.background.height, 0, 0, width, height); + } else { + ctx.filter = `blur(${blurPx}px)`; + ctx.drawImage(frame, 0, 0, width, height); + ctx.filter = 'none'; + } + + if (maskForEffect && this.foregroundCtx) { + this.foregroundCtx.clearRect(0, 0, width, height); + this.foregroundCtx.drawImage(frame, 0, 0, width, height); + this.foregroundCtx.globalCompositeOperation = 'destination-in'; + this.foregroundCtx.drawImage(maskForEffect, 0, 0, width, height); + this.foregroundCtx.globalCompositeOperation = 'source-over'; + ctx.drawImage(this.foregroundCanvas!, 0, 0, width, height); + } else { + ctx.drawImage(frame, 0, 0, width, height); + } + + const renderMs = performance.now() - renderStart; + const totalMs = result.durationMs + renderMs; + let metricsTier = qualityTier.tier; + if (this.config.quality === 'auto' && this.qualityController && isProcessingMode(this.config.mode)) { + const updatedTier = this.qualityController.update( + {totalMs, segmentationMs: result.durationMs, gpuMs: renderMs}, + this.config.mode, + ); + metricsTier = updatedTier.tier; + this.onTierChange?.(updatedTier.tier); + } + this.updateMetrics(totalMs, result.durationMs, renderMs, metricsTier); + } finally { + mask?.close(); + classMask?.close(); + result.release(); + frame.close(); + } + } + + /** + * Sets a static background image for virtual background mode. + * + * Stores the bitmap and uses it for all subsequent frames until cleared + * or replaced. The previous background bitmap is closed to prevent leaks. + * + * @param bitmap - Background image as ImageBitmap. + * @param width - Image width in pixels. + * @param height - Image height in pixels. + */ + public setBackgroundImage(bitmap: ImageBitmap, width: number, height: number): void { + this.background?.bitmap?.close(); + this.background = {bitmap, width, height}; + } + + /** + * Sets a video frame as background for virtual background mode. + * + * Delegates to setBackgroundImage since Canvas2D pipeline treats + * video frames the same as static images. + * + * @param bitmap - Background video frame as ImageBitmap. + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + */ + public setBackgroundVideoFrame(bitmap: ImageBitmap, width: number, height: number): void { + this.setBackgroundImage(bitmap, width, height); + } + + /** + * Clears the background source. + * + * Closes the stored background bitmap and resets background state. + */ + public clearBackground(): void { + this.background?.bitmap?.close(); + this.background = null; + } + + /** + * Notifies of dropped frames (no-op for Canvas2D). + * + * Canvas2D pipeline doesn't need dropped frame notifications since it + * processes frames synchronously on the main thread. + * + * @param _count - Dropped frame count (ignored). + */ + public notifyDroppedFrames(_count: number): void { + // No-op for canvas2d. + } + + /** + * Returns whether output canvas is transferred (always false). + * + * Canvas2D pipeline always runs on the main thread. + * + * @returns Always false. + */ + public isOutputCanvasTransferred(): boolean { + return false; + } + + /** + * Renders class mask visualization for debug modes. + * + * Converts class mask ImageBitmap to colored visualization by mapping + * class IDs to HSV-based colors. Supports overlay and standalone modes. + * Uses a debug canvas for pixel manipulation before rendering. + * + * @param ctx - Canvas2D rendering context for output. + * @param mask - Class mask ImageBitmap with class IDs encoded in RGB channels. + * @param width - Mask width in pixels. + * @param height - Mask height in pixels. + * @param overlay - If true, overlays colored mask on video frame; if false, shows only mask. + * @param frame - Original video frame (used for overlay mode). + */ + private renderClassMask( + ctx: CanvasRenderingContext2D, + mask: ImageBitmap, + width: number, + height: number, + overlay: boolean, + frame: ImageBitmap, + ): void { + if (!this.debugCanvas || !this.debugCtx) { + if (overlay) { + ctx.drawImage(frame, 0, 0, width, height); + } + ctx.drawImage(mask, 0, 0, width, height); + return; + } + this.debugCtx.clearRect(0, 0, width, height); + this.debugCtx.drawImage(mask, 0, 0, width, height); + const imageData = this.debugCtx.getImageData(0, 0, width, height); + const data = imageData.data; + for (let i = 0; i < data.length; i += 4) { + const classId = data[i]; + if (classId === 0) { + data[i] = 0; + data[i + 1] = 0; + data[i + 2] = 0; + data[i + 3] = overlay ? 0 : 255; + continue; + } + const [r, g, b] = this.classColor(classId); + data[i] = r; + data[i + 1] = g; + data[i + 2] = b; + data[i + 3] = 255; + } + this.debugCtx.putImageData(imageData, 0, 0); + if (overlay) { + ctx.drawImage(frame, 0, 0, width, height); + ctx.globalAlpha = 0.6; + ctx.drawImage(this.debugCanvas, 0, 0, width, height); + ctx.globalAlpha = 1; + } else { + ctx.drawImage(this.debugCanvas, 0, 0, width, height); + } + } + + /** + * Generates a color for a class ID using HSV color space. + * + * Uses a fixed saturation and value with hue derived from class ID + * to ensure distinct colors for different classes. The hue is multiplied + * by 0.13 and wrapped to ensure good color distribution. + * + * @param classId - Class identifier (0-255). + * @returns RGB color tuple [r, g, b] with values 0-255. + */ + private classColor(classId: number): [number, number, number] { + const hue = (classId * 0.13) % 1; + const [r, g, b] = this.hsvToRgb(hue, 0.75, 0.95); + return [Math.round(r * 255), Math.round(g * 255), Math.round(b * 255)]; + } + + /** + * Converts HSV color to RGB color space. + * + * Implements standard HSV to RGB conversion algorithm using the + * hexagonal color model. + * + * @param h - Hue (0-1). + * @param s - Saturation (0-1). + * @param v - Value/brightness (0-1). + * @returns RGB color tuple [r, g, b] with values 0-1. + */ + private hsvToRgb(h: number, s: number, v: number): [number, number, number] { + const i = Math.floor(h * 6); + const f = h * 6 - i; + const p = v * (1 - s); + const q = v * (1 - f * s); + const t = v * (1 - (1 - f) * s); + const mod = i % 6; + if (mod === 0) { + return [v, t, p]; + } + if (mod === 1) { + return [q, v, p]; + } + if (mod === 2) { + return [p, v, t]; + } + if (mod === 3) { + return [p, q, v]; + } + if (mod === 4) { + return [t, p, v]; + } + return [v, p, q]; + } + + /** + * Updates the mask canvas with temporal smoothing using exponential moving average. + * + * Implements temporal mask smoothing by blending the new mask with the previous + * mask using an alpha value. Uses ping-pong buffering (maskCanvas and maskScratchCanvas) + * to avoid reading and writing to the same buffer. + * + * Behavior: + * - If no previous mask exists or alpha >= 1: replaces mask completely + * - If alpha <= 0: keeps previous mask unchanged + * - Otherwise: blends previous mask (1-alpha) with new mask (alpha) + * + * @param mask - New mask ImageBitmap to blend in. + * @param width - Mask width in pixels (uses mask.width if not provided). + * @param height - Mask height in pixels (uses mask.height if not provided). + * @param alpha - Blending factor (0-1) for temporal smoothing. Higher values + * make the mask more responsive to changes. + */ + private updateMaskCanvas(mask: ImageBitmap, width: number, height: number, alpha: number): void { + const resolvedWidth = width || mask.width; + const resolvedHeight = height || mask.height; + this.ensureMaskCanvas(resolvedWidth, resolvedHeight); + if (!this.maskCanvas || !this.maskCtx) { + return; + } + const clampedAlpha = Math.max(0, Math.min(1, alpha)); + if (!this.hasMask || clampedAlpha >= 1) { + this.maskCtx.globalAlpha = 1; + this.maskCtx.clearRect(0, 0, resolvedWidth, resolvedHeight); + this.maskCtx.drawImage(mask, 0, 0, resolvedWidth, resolvedHeight); + this.hasMask = true; + return; + } + if (clampedAlpha <= 0) { + this.hasMask = true; + return; + } + if (!this.maskScratchCanvas || !this.maskScratchCtx) { + this.maskCtx.globalAlpha = 1; + this.maskCtx.clearRect(0, 0, resolvedWidth, resolvedHeight); + this.maskCtx.drawImage(mask, 0, 0, resolvedWidth, resolvedHeight); + this.hasMask = true; + return; + } + this.maskScratchCtx.globalAlpha = 1 - clampedAlpha; + this.maskScratchCtx.clearRect(0, 0, resolvedWidth, resolvedHeight); + this.maskScratchCtx.drawImage(this.maskCanvas, 0, 0, resolvedWidth, resolvedHeight); + this.maskScratchCtx.globalAlpha = clampedAlpha; + this.maskScratchCtx.drawImage(mask, 0, 0, resolvedWidth, resolvedHeight); + this.maskScratchCtx.globalAlpha = 1; + this.swapMaskBuffers(); + this.hasMask = true; + } + + /** + * Renders edge visualization of a mask for debug mode. + * + * Extracts edges from the mask using smoothstep functions to create a band-pass + * filter that highlights mask boundaries. The result is a grayscale image where + * edges appear as bright regions. + * + * Edge detection algorithm: + * - Uses smoothstep(0.4, 0.6, value) to detect rising edges + * - Subtracts smoothstep(0.6, 0.8, value) to detect falling edges + * - Result highlights the transition region (edges) of the mask + * + * @param ctx - Canvas2D rendering context for output. + * @param mask - Mask image source to extract edges from. + * @param width - Output width in pixels. + * @param height - Output height in pixels. + */ + private renderEdgeMask(ctx: CanvasRenderingContext2D, mask: CanvasImageSource, width: number, height: number): void { + if (!this.maskScratchCanvas || !this.maskScratchCtx || !this.maskCanvas) { + ctx.drawImage(mask, 0, 0, width, height); + return; + } + const maskWidth = this.maskCanvas.width || width; + const maskHeight = this.maskCanvas.height || height; + if (this.maskScratchCanvas.width !== maskWidth || this.maskScratchCanvas.height !== maskHeight) { + this.maskScratchCanvas.width = maskWidth; + this.maskScratchCanvas.height = maskHeight; + } + this.maskScratchCtx.clearRect(0, 0, maskWidth, maskHeight); + this.maskScratchCtx.drawImage(mask, 0, 0, maskWidth, maskHeight); + const imageData = this.maskScratchCtx.getImageData(0, 0, maskWidth, maskHeight); + const data = imageData.data; + for (let i = 0; i < data.length; i += 4) { + const value = data[i] / 255; + const edge = this.smoothstep(0.4, 0.6, value) - this.smoothstep(0.6, 0.8, value); + const gray = Math.round(edge * 255); + data[i] = gray; + data[i + 1] = gray; + data[i + 2] = gray; + data[i + 3] = 255; + } + this.maskScratchCtx.putImageData(imageData, 0, 0); + ctx.drawImage(this.maskScratchCanvas, 0, 0, width, height); + } + + /** + * Smooth interpolation function (smoothstep) for edge detection. + * + * Implements the standard smoothstep function that provides smooth interpolation + * between 0 and 1 using a cubic Hermite polynomial. Returns 0 for x <= edge0, + * 1 for x >= edge1, and smoothly interpolates between them. + * + * Used for edge detection in renderEdgeMask to create smooth transitions + * rather than hard thresholds. + * + * @param edge0 - Lower edge threshold. + * @param edge1 - Upper edge threshold. + * @param x - Input value to interpolate. + * @returns Interpolated value between 0 and 1. + */ + private smoothstep(edge0: number, edge1: number, x: number): number { + const t = Math.max(0, Math.min(1, (x - edge0) / (edge1 - edge0))); + return t * t * (3 - 2 * t); + } + + /** + * Ensures mask canvas buffers are created and properly sized. + * + * Creates maskCanvas and maskScratchCanvas if they don't exist, and resizes + * them to match the specified dimensions. These canvases are used for temporal + * mask smoothing with ping-pong buffering. + * + * @param width - Required canvas width in pixels. + * @param height - Required canvas height in pixels. + */ + private ensureMaskCanvas(width: number, height: number): void { + if (!this.maskCanvas) { + this.maskCanvas = document.createElement('canvas'); + this.maskCtx = this.maskCanvas.getContext('2d'); + } + if (!this.maskScratchCanvas) { + this.maskScratchCanvas = document.createElement('canvas'); + this.maskScratchCtx = this.maskScratchCanvas.getContext('2d'); + } + if (this.maskCanvas && (this.maskCanvas.width !== width || this.maskCanvas.height !== height)) { + this.maskCanvas.width = width; + this.maskCanvas.height = height; + } + if ( + this.maskScratchCanvas && + (this.maskScratchCanvas.width !== width || this.maskScratchCanvas.height !== height) + ) { + this.maskScratchCanvas.width = width; + this.maskScratchCanvas.height = height; + } + } + + /** + * Swaps mask canvas buffers for ping-pong rendering. + * + * Exchanges maskCanvas with maskScratchCanvas and their associated contexts. + * Used after blending operations to prepare for the next frame, allowing + * reading from one buffer while writing to the other. + */ + private swapMaskBuffers(): void { + const canvas = this.maskCanvas; + this.maskCanvas = this.maskScratchCanvas; + this.maskScratchCanvas = canvas; + const ctx = this.maskCtx; + this.maskCtx = this.maskScratchCtx; + this.maskScratchCtx = ctx; + } + + /** + * Stops the pipeline and releases all resources. + * + * Closes segmenter, releases background bitmaps, destroys canvases, + * resets quality controller, and clears all references. Should be called + * when the pipeline is no longer needed to prevent memory leaks. + */ + public stop(): void { + this.background?.bitmap?.close(); + this.background = null; + this.maskPostProcessor.reset(); + this.segmenter?.close(); + this.segmenter = null; + this.currentModelPath = null; + this.qualityController = null; + this.outputCanvas = null; + this.canvasCtx = null; + this.foregroundCanvas = null; + this.foregroundCtx = null; + this.debugCanvas = null; + this.debugCtx = null; + this.maskCanvas = null; + this.maskCtx = null; + this.maskScratchCanvas = null; + this.maskScratchCtx = null; + this.hasMask = false; + this.config = null; + this.onMetrics = null; + this.onTierChange = null; + this.getDroppedFrames = null; + resetMetricsWindow(this.metricsWindow); + } + + /** + * Resolves quality tier parameters for the current configuration. + * + * Delegates to resolveQualityTierForEffectMode with the current quality + * controller, quality mode, and effect mode. Returns bypass tier if + * config is unavailable. + * + * @returns Quality tier parameters for current configuration. + */ + private resolveQuality(): QualityTierParams { + if (!this.config) { + return resolveQualityTierForEffectMode(null, 'auto', 'blur'); + } + return resolveQualityTierForEffectMode(this.qualityController, this.config.quality, this.config.mode); + } + + /** + * Ensures the segmenter is initialized for the specified quality tier. + * + * If the tier requires a different model than currently loaded, swaps + * the segmenter instance. Skips swap if tier is bypass or if the + * desired model is already loaded. Uses CPU delegate for Canvas2D pipeline. + * + * @param tier - Quality tier + */ + private async ensureSegmenterForTier(tier: QualityTier): Promise { + if (tier === 'bypass') { + return; + } + const desiredPath = resolveSegmentationModelPath( + tier, + this.segmentationModelByTier, + this.currentModelPath ?? undefined, + ); + if (this.currentModelPath === desiredPath && this.segmenter) { + return; + } + const nextSegmenter = this.segmenterFactory.create({ + modelPath: desiredPath, + delegate: 'CPU', + }); + try { + await nextSegmenter.init(); + } catch (error) { + this.logger.warn('Segmentation model swap failed, keeping previous model', error); + nextSegmenter.close(); + return; + } + this.segmenter?.close(); + this.segmenter = nextSegmenter; + this.currentModelPath = desiredPath; + } + + /** + * Updates performance metrics and invokes metrics callback. + * + * Adds a new sample to the metrics window and invokes the onMetrics callback + * with aggregated metrics. Note that quality controller updates now happen + * inline in processFrame() when in 'auto' mode, not in this method. + * + * Uses 'CPU' as the segmentation delegate since Canvas2D uses CPU inference. + * + * @param totalMs - Total frame processing time in milliseconds. + * @param segmentationMs - Segmentation processing time in milliseconds. + * @param gpuMs - Rendering time in milliseconds (Canvas2D operations). + * @param tier - Current quality tier. + */ + private updateMetrics(totalMs: number, segmentationMs: number, gpuMs: number, tier: QualityTier): void { + if (!this.onMetrics || !this.getDroppedFrames) { + return; + } + pushMetricsSample(this.metricsWindow, {totalMs, segmentationMs, gpuMs}); + // Canvas2dPipeline uses CPU delegate + const segmentationDelegate = this.segmenter?.getDelegate?.() ?? 'CPU'; + this.onMetrics(buildMetrics(this.metricsWindow, this.getDroppedFrames(), tier, segmentationDelegate)); + } + + public getCurrentModelPath(): string | null { + return ''; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/mainWebGlPipeline.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/mainWebGlPipeline.ts new file mode 100644 index 00000000000..42597fbed73 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/mainWebGlPipeline.ts @@ -0,0 +1,456 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {getLogger, Logger} from 'Util/logger'; + +import type { + BackgroundEffectsRenderingPipeline, + PipelineConfig, + PipelineInit, +} from './backgroundEffectsRenderingPipeline'; + +import type {QualityTier, QualityTierParams, SegmentationModelByTier} from '../backgroundEffectsWorkerTypes'; +import { + buildMetrics, + createMetricsWindow, + isProcessingMode, + pushMetricsSample, + QualityController, + resetMetricsWindow, + resolveQualityTierForEffectMode, + resolveSegmentationModelPath, +} from '../quality'; +import {WebGlRenderer} from '../renderer/webGlRenderer'; +import type {MaskPostProcessor} from '../segmentation/maskPostProcessor'; +import {NoopMaskPostProcessor} from '../segmentation/maskPostProcessor'; +import {MediaPipeSegmenterFactory} from '../segmentation/mediaPipeSegmenter'; +import type {SegmenterFactory, SegmenterLike} from '../segmentation/segmenterTypes'; +import {buildMaskInput, type MaskInput, type MaskSource} from '../shared/mask'; + +/** + * Main-thread WebGL2 rendering pipeline for background effects. + * + * This pipeline uses GPU-accelerated WebGL2 rendering on the main thread. + * It performs segmentation using GPU delegate and renders effects using + * a multi-pass WebGL rendering pipeline with shader programs. + * + * Performance characteristics: + * - High quality rendering with GPU acceleration + * - Runs on main thread (may impact UI responsiveness) + * - Requires WebGL2 support + * - Uses GPU-accelerated segmentation (faster than CPU) + * + * Rendering approach: + * - Segmentation: GPU-based ML inference (MediaPipe GPU delegate) + * - Background blur: Multi-pass Gaussian blur with downsampling + * - Virtual background: WebGL texture compositing + * - Mask refinement: Joint bilateral filtering, temporal smoothing + */ +export class MainWebGlPipeline implements BackgroundEffectsRenderingPipeline { + public readonly type = 'main-webgl2' as const; + private readonly logger: Logger; + private renderer: WebGlRenderer | null = null; + private segmenter: SegmenterLike | null = null; + private segmenterFactory: SegmenterFactory = MediaPipeSegmenterFactory; + private segmentationModelByTier: SegmentationModelByTier = {}; + private currentModelPath: string | null = null; + private maskPostProcessor: MaskPostProcessor = new NoopMaskPostProcessor(); + private qualityController: QualityController | null = null; + private outputCanvas: HTMLCanvasElement | null = null; + private background: {bitmap: ImageBitmap; width: number; height: number} | null = null; + private config: PipelineConfig | null = null; + private onMetrics: PipelineInit['onMetrics'] = null; + private onTierChange: PipelineInit['onTierChange'] | null = null; + private getDroppedFrames: PipelineInit['getDroppedFrames'] | null = null; + private readonly metricsMaxSamples = 30; + private readonly metricsWindow = createMetricsWindow(this.metricsMaxSamples); + private mainFrameCount = 0; + + constructor() { + this.logger = getLogger('MainWebGlPipeline'); + } + + /** + * Initializes the main WebGL2 pipeline. + * + * Sets up WebGL renderer, initializes segmenter with GPU delegate, + * quality controller, and mask post-processor. Creates WebGL context + * on the output canvas and compiles all shader programs. + * + * If segmentation initialization fails, the pipeline throws an error + * (caller should fall back to another pipeline). + * + * @param init - BackgroundEffectsRenderingPipeline initialization parameters. + * @throws Error if segmentation initialization fails. + */ + public async init(init: PipelineInit): Promise { + this.outputCanvas = init.outputCanvas; + this.config = init.config; + this.onMetrics = init.onMetrics; + this.onTierChange = init.onTierChange; + this.getDroppedFrames = init.getDroppedFrames; + this.mainFrameCount = 0; + + this.renderer = new WebGlRenderer(this.outputCanvas, this.outputCanvas.width, this.outputCanvas.height); + this.segmenterFactory = init.createSegmenter ?? MediaPipeSegmenterFactory; + this.segmentationModelByTier = init.segmentationModelByTier; + const postProcessorFactory = init.createMaskPostProcessor ?? { + create: () => new NoopMaskPostProcessor(), + }; + this.maskPostProcessor = postProcessorFactory.create(); + this.qualityController = new QualityController(init.targetFps); + if (this.qualityController && this.config?.quality === 'auto') { + this.qualityController.setTier(init.initialTier); + } else if (this.qualityController && this.config?.quality !== 'auto') { + this.qualityController.setTier(this.config.quality); + } + const startingTier = init.config.quality === 'auto' ? init.initialTier : init.config.quality; + const shouldInitSegmenter = startingTier !== 'bypass' && init.config.mode !== 'passthrough'; + if (shouldInitSegmenter) { + this.segmenter = this.segmenterFactory.create({ + modelPath: init.segmentationModelPath, + delegate: 'GPU', + canvas: this.outputCanvas, + }); + this.currentModelPath = init.segmentationModelPath; + try { + await this.segmenter.init(); + } catch (error) { + this.logger.warn('Segmentation init failed, falling back to passthrough', error); + this.segmenter?.close(); + this.segmenter = null; + this.renderer?.destroy(); + this.renderer = null; + throw error; + } + } else { + this.segmenter = null; + this.currentModelPath = null; + } + } + + /** + * Updates pipeline configuration at runtime. + * + * Updates internal config and quality controller tier if quality + * mode is fixed (not 'auto'). Adaptive quality updates happen + * automatically during frame processing. + * + * @param config - New pipeline configuration. + */ + public updateConfig(config: PipelineConfig): void { + this.config = config; + if (this.qualityController && config.quality !== 'auto') { + this.qualityController.setTier(config.quality); + } + } + + /** + * Processes a video frame through the main WebGL2 pipeline. + * + * Processing steps: + * 1. Resolves quality tier parameters (adaptive or fixed) + * 2. Ensures segmenter is initialized for current tier + * 3. Performs segmentation (if cadence allows, GPU-accelerated) + * 4. Applies mask post-processing + * 5. Configures WebGL renderer with current settings + * 6. Renders frame through multi-pass WebGL pipeline + * 7. Updates quality controller and performance metrics + * + * The renderer performs mask refinement, temporal smoothing, blur passes, + * and final compositing using GPU shaders for optimal performance. + * + * @param frame - Input video frame as ImageBitmap (will be closed after processing). + * @param timestamp - Frame timestamp in seconds (used for temporal smoothing). + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + * @returns Promise that resolves when frame processing is complete. + */ + public async processFrame(frame: ImageBitmap, timestamp: number, width: number, height: number): Promise { + if (!this.renderer || !this.outputCanvas || !this.config) { + frame.close(); + return; + } + + const qualityTier = this.resolveQuality(); + if (!qualityTier.bypass) { + await this.ensureSegmenterForTier(qualityTier.tier); + } + const frameIndex = this.mainFrameCount; + this.mainFrameCount += 1; + + let maskInput: MaskInput | null = null; + let maskBitmap: ImageBitmap | null = null; + let releaseMaskResources: (() => void) | null = null; + let segmentationMs = 0; + if (!qualityTier.bypass && qualityTier.segmentationCadence > 0 && this.segmenter && this.qualityController) { + if (frameIndex % qualityTier.segmentationCadence === 0) { + this.segmenter.configure(qualityTier.segmentationWidth, qualityTier.segmentationHeight); + const segStart = performance.now(); + const timestampMs = timestamp * 1000; + const includeClassMask = this.config.debugMode === 'classOverlay' || this.config.debugMode === 'classOnly'; + const result = await this.segmenter.segment(frame, timestampMs, {includeClassMask}); + segmentationMs = performance.now() - segStart; + const processed = await this.maskPostProcessor.process(result, { + qualityTier, + mode: this.config.mode, + timestampMs, + frameSize: {width, height}, + }); + if (processed !== result) { + result.mask?.close(); + result.classMask?.close(); + result.release(); + } + const useClassMask = includeClassMask && processed.classMask; + const maskSource: MaskSource = useClassMask + ? { + mask: processed.classMask, + maskTexture: null, + width: processed.width, + height: processed.height, + release: processed.release, + } + : processed; + if (useClassMask) { + processed.mask?.close(); + } else { + processed.classMask?.close(); + } + const maskResult = buildMaskInput(maskSource); + maskInput = maskResult.maskInput; + maskBitmap = maskResult.maskBitmap; + releaseMaskResources = maskResult.release; + } + } + + if (this.outputCanvas.width !== width || this.outputCanvas.height !== height) { + this.outputCanvas.width = width; + this.outputCanvas.height = height; + } + + this.renderer.configure( + width, + height, + qualityTier, + this.config.mode, + this.config.debugMode, + this.config.blurStrength, + ); + + if (this.background) { + this.renderer.setBackground(this.background.bitmap, this.background.width, this.background.height); + } + + const gpuStart = performance.now(); + let gpuMs = 0; + try { + this.renderer.render(frame, maskInput); + } finally { + gpuMs = performance.now() - gpuStart; + maskBitmap?.close(); + releaseMaskResources?.(); + frame.close(); + } + + if (this.config.quality === 'auto' && this.qualityController && isProcessingMode(this.config.mode)) { + const updatedTier = this.qualityController.update( + {totalMs: segmentationMs + gpuMs, segmentationMs, gpuMs}, + this.config.mode, + ); + this.onTierChange?.(updatedTier.tier); + } + + this.updateMetrics(segmentationMs + gpuMs, segmentationMs, gpuMs, qualityTier.tier); + } + + /** + * Sets a static background image for virtual background mode. + * + * Stores the bitmap and uploads it to the WebGL renderer as a texture. + * The previous background bitmap is closed to prevent leaks. + * + * @param bitmap - Background image as ImageBitmap. + * @param width - Image width in pixels. + * @param height - Image height in pixels. + */ + public setBackgroundImage(bitmap: ImageBitmap, width: number, height: number): void { + this.background?.bitmap?.close(); + this.background = {bitmap, width, height}; + this.renderer?.setBackground(bitmap, width, height); + } + + /** + * Sets a video frame as background for virtual background mode. + * + * Delegates to setBackgroundImage since the renderer treats video + * frames the same as static images (updates texture each frame). + * + * @param bitmap - Background video frame as ImageBitmap. + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + */ + public setBackgroundVideoFrame(bitmap: ImageBitmap, width: number, height: number): void { + this.setBackgroundImage(bitmap, width, height); + } + + /** + * Clears the background source. + * + * Closes the stored background bitmap, resets background state, and + * clears the renderer's background texture. + */ + public clearBackground(): void { + this.background?.bitmap?.close(); + this.background = null; + this.renderer?.setBackground(null, 0, 0); + } + + /** + * Notifies of dropped frames (no-op for main pipeline). + * + * Main WebGL pipeline doesn't need dropped frame notifications since it + * processes frames synchronously on the main thread. + * + * @param _count - Dropped frame count (ignored). + */ + public notifyDroppedFrames(_count: number): void { + // No-op for main pipeline. + } + + /** + * Returns whether output canvas is transferred (always false). + * + * Main WebGL pipeline always runs on the main thread. + * + * @returns Always false. + */ + public isOutputCanvasTransferred(): boolean { + return false; + } + + /** + * Stops the pipeline and releases all resources. + * + * Closes segmenter, destroys WebGL renderer (releases all GPU resources), + * releases background bitmaps, resets quality controller, and clears all + * references. Should be called when the pipeline is no longer needed. + */ + public stop(): void { + this.background?.bitmap?.close(); + this.background = null; + this.maskPostProcessor.reset(); + this.segmenter?.close(); + this.segmenter = null; + this.currentModelPath = null; + this.renderer?.destroy(); + this.renderer = null; + this.qualityController = null; + this.outputCanvas = null; + this.config = null; + this.onMetrics = null; + this.onTierChange = null; + this.getDroppedFrames = null; + resetMetricsWindow(this.metricsWindow); + } + + /** + * Resolves quality tier parameters for the current configuration. + * + * Delegates to resolveQualityTierForEffectMode with the current quality + * controller, quality mode, and effect mode. Returns bypass tier if + * config is unavailable. + * + * @returns Quality tier parameters for current configuration. + */ + private resolveQuality(): QualityTierParams { + if (!this.config) { + return resolveQualityTierForEffectMode(null, 'auto', 'blur'); + } + return resolveQualityTierForEffectMode(this.qualityController, this.config.quality, this.config.mode); + } + + /** + * Ensures the segmenter is initialized for the specified quality tier. + * + * If the tier requires a different model than currently loaded, swaps + * the segmenter instance. Skips swap if tier is 'D' (bypass), output canvas + * is unavailable, or if the desired model is already loaded. Uses GPU delegate + * for main WebGL pipeline. + * + * @param tier - Quality tier ). + */ + private async ensureSegmenterForTier(tier: QualityTier): Promise { + if (!this.outputCanvas) { + return; + } + if (tier === 'bypass') { + return; + } + const desiredPath = resolveSegmentationModelPath( + tier, + this.segmentationModelByTier, + this.currentModelPath ?? undefined, + ); + if (this.currentModelPath === desiredPath && this.segmenter) { + return; + } + const nextSegmenter = this.segmenterFactory.create({ + modelPath: desiredPath, + delegate: 'GPU', + canvas: this.outputCanvas, + }); + try { + await nextSegmenter.init(); + } catch (error) { + this.logger.warn('Segmentation model swap failed, keeping previous model', error); + nextSegmenter.close(); + return; + } + this.segmenter?.close(); + this.segmenter = nextSegmenter; + this.currentModelPath = desiredPath; + } + + /** + * Updates performance metrics and invokes metrics callback. + * + * Adds a new sample to the metrics window, updates quality tier if in + * adaptive mode, and invokes the onMetrics callback with aggregated metrics. + * Uses 'GPU' as the segmentation delegate since MainWebGlPipeline uses GPU inference. + * + * @param totalMs - Total frame processing time in milliseconds. + * @param segmentationMs - Segmentation processing time in milliseconds. + * @param gpuMs - WebGL rendering time in milliseconds. + * @param tier - Current quality tier. + */ + private updateMetrics(totalMs: number, segmentationMs: number, gpuMs: number, tier: QualityTier): void { + if (!this.onMetrics || !this.getDroppedFrames) { + return; + } + pushMetricsSample(this.metricsWindow, {totalMs, segmentationMs, gpuMs}); + // MainWebGlPipeline uses GPU delegate + const segmentationDelegate = this.segmenter?.getDelegate?.() ?? 'GPU'; + this.onMetrics(buildMetrics(this.metricsWindow, this.getDroppedFrames(), tier, segmentationDelegate)); + } + + public getCurrentModelPath(): string | null { + return ''; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/passthroughPipeline.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/passthroughPipeline.ts new file mode 100644 index 00000000000..ca4203b3db4 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/passthroughPipeline.ts @@ -0,0 +1,166 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type { + BackgroundEffectsRenderingPipeline, + PipelineConfig, + PipelineInit, +} from './backgroundEffectsRenderingPipeline'; + +/** + * Passthrough pipeline that renders frames without any processing. + * + * This pipeline serves as a fallback when other pipelines are unavailable + * or when passthrough mode is explicitly requested. It simply copies input + * frames to the output canvas without applying any background effects. + * + * Use cases: + * - Fallback when WebGL2/Canvas2D unavailable + * - Explicit passthrough mode (no effects) + * - Error recovery after pipeline failures + */ +export class PassthroughPipeline implements BackgroundEffectsRenderingPipeline { + public readonly type = 'passthrough' as const; + private outputCanvas: HTMLCanvasElement | null = null; + + /** + * Initializes the passthrough pipeline. + * + * Stores a reference to the output canvas. No other initialization is needed + * since no processing resources are required. + * + * @param init - BackgroundEffectsRenderingPipeline initialization parameters. + * @returns Promise that resolves immediately. + */ + public async init(init: PipelineInit): Promise { + this.outputCanvas = init.outputCanvas; + } + + /** + * Updates pipeline configuration (no-op for passthrough). + * + * Passthrough pipeline ignores all configuration changes since it doesn't + * apply any effects or processing. + * + * @param _config - New configuration (ignored). + */ + public updateConfig(_config: PipelineConfig): void { + // No-op. + } + + /** + * Processes a frame by copying it directly to the output canvas. + * + * Draws the input frame to the output canvas using Canvas2D without + * any processing or effects applied. The frame is closed after drawing. + * + * @param frame - Input video frame as ImageBitmap. + * @param _timestamp - Frame timestamp (unused). + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + * @returns Promise that resolves immediately after drawing. + */ + public async processFrame(frame: ImageBitmap, _timestamp: number, width: number, height: number): Promise { + if (!this.outputCanvas) { + frame.close(); + return; + } + const ctx = this.outputCanvas.getContext('2d'); + if (!ctx) { + frame.close(); + return; + } + ctx.drawImage(frame, 0, 0, width, height); + frame.close(); + } + + /** + * Sets background image (no-op, closes bitmap immediately). + * + * Passthrough pipeline doesn't use backgrounds, so the bitmap is + * immediately closed to prevent memory leaks. + * + * @param bitmap - Background image bitmap (closed immediately). + * @param _width - Image width (unused). + * @param _height - Image height (unused). + */ + public setBackgroundImage(bitmap: ImageBitmap, _width: number, _height: number): void { + bitmap.close(); + } + + /** + * Sets background video frame (no-op, closes bitmap immediately). + * + * Passthrough pipeline doesn't use backgrounds, so the bitmap is + * immediately closed to prevent memory leaks. + * + * @param bitmap - Background video frame bitmap (closed immediately). + * @param _width - Frame width (unused). + * @param _height - Frame height (unused). + */ + public setBackgroundVideoFrame(bitmap: ImageBitmap, _width: number, _height: number): void { + bitmap.close(); + } + + /** + * Clears background (no-op for passthrough). + * + * Passthrough pipeline doesn't maintain background state. + */ + public clearBackground(): void { + // No-op. + } + + /** + * Notifies of dropped frames (no-op for passthrough). + * + * Passthrough pipeline doesn't track dropped frames since it has + * minimal processing overhead. + * + * @param _count - Dropped frame count (ignored). + */ + public notifyDroppedFrames(_count: number): void { + // No-op. + } + + /** + * Returns whether output canvas is transferred (always false). + * + * Passthrough pipeline always runs on the main thread. + * + * @returns Always false. + */ + public isOutputCanvasTransferred(): boolean { + return false; + } + + /** + * Stops the pipeline and releases resources. + * + * Clears the output canvas reference. No other cleanup is needed + * since passthrough doesn't allocate processing resources. + */ + public stop(): void { + this.outputCanvas = null; + } + + public getCurrentModelPath(): string | null { + return ''; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/workerWebGlPipeline.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/workerWebGlPipeline.ts new file mode 100644 index 00000000000..1b76610f73f --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/pipelines/workerWebGlPipeline.ts @@ -0,0 +1,345 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type { + BackgroundEffectsRenderingPipeline, + PipelineConfig, + PipelineInit, +} from './backgroundEffectsRenderingPipeline'; + +import type {QualityTier, WorkerOptions, WorkerResponse} from '../backgroundEffectsWorkerTypes'; + +/** + * Worker-thread WebGL2 rendering pipeline for background effects. + * + * This pipeline transfers the output canvas to a Web Worker and performs + * all processing (segmentation and rendering) in a background thread. This + * provides the best performance by avoiding main thread blocking. + * + * Performance characteristics: + * - Highest quality and performance + * - Offloads processing to background thread (no UI blocking) + * - Requires WebGL2, Worker, and OffscreenCanvas support + * - Uses backpressure (single frame in flight) to prevent queue buildup + * + * Communication: + * - Main thread sends frames via postMessage (ImageBitmap transferred, not cloned) + * - Worker processes frames and renders to OffscreenCanvas + * - Worker sends metrics and completion notifications back to main thread + * - All configuration updates are sent via postMessage + */ +export class WorkerWebGlPipeline implements BackgroundEffectsRenderingPipeline { + public readonly type = 'worker-webgl2' as const; + private worker: Worker | null = null; + private outputCanvasTransferred = false; + private workerFrameInFlight = false; + private workerFrameResolve: (() => void) | null = null; + private workerFrameReject: ((error: Error) => void) | null = null; + private onMetrics: PipelineInit['onMetrics'] = null; + private onTierChange: PipelineInit['onTierChange'] | null = null; + private onDroppedFrame: PipelineInit['onDroppedFrame'] | null = null; + private onWorkerSegmenterError: PipelineInit['onWorkerSegmenterError'] | null = null; + private onWorkerContextLoss: PipelineInit['onWorkerContextLoss'] | null = null; + private lastTier: QualityTier | null = null; + + /** + * Initializes the worker WebGL2 pipeline. + * + * Transfers the output canvas to an OffscreenCanvas, creates a Web Worker, + * and sends initialization message with configuration. Sets up message handlers + * for metrics, tier changes, segmenter errors, context loss, and frame completion. + * + * The canvas is transferred (not cloned) to the worker, so the main thread + * can no longer access it directly after this call. + * + * Context loss handling: + * - Listens for 'contextLost' messages from the worker + * - Invokes onWorkerContextLoss callback when WebGL context is lost + * - The callback should handle fallback to another pipeline + * + * @param init - BackgroundEffectsRenderingPipeline initialization parameters. + * @returns Promise that resolves when worker initialization is complete. + */ + public async init(init: PipelineInit): Promise { + this.onMetrics = init.onMetrics; + this.onTierChange = init.onTierChange; + this.onDroppedFrame = init.onDroppedFrame; + this.onWorkerSegmenterError = init.onWorkerSegmenterError ?? null; + this.onWorkerContextLoss = init.onWorkerContextLoss ?? null; + this.workerFrameInFlight = false; + this.workerFrameResolve = null; + this.workerFrameReject = null; + + const offscreen = init.outputCanvas.transferControlToOffscreen(); + this.outputCanvasTransferred = true; + this.worker = new Worker(new URL('../worker/bgfx.worker.ts', import.meta.url), {type: 'module'}); + + const workerOptions: WorkerOptions = { + mode: init.config.mode, + debugMode: init.config.debugMode, + quality: init.config.quality, + blurStrength: init.config.blurStrength, + segmentationModelPath: init.segmentationModelPath, + segmentationModelByTier: init.segmentationModelByTier, + initialTier: init.initialTier, + targetFps: init.targetFps, + }; + + this.worker.onmessage = (event: MessageEvent) => { + if (event.data.type === 'metrics') { + const metrics = event.data.metrics; + if (this.lastTier !== metrics.tier) { + this.onTierChange?.(metrics.tier); + this.lastTier = metrics.tier; + } + this.onMetrics?.(metrics); + } + if (event.data.type === 'segmenterError') { + this.onWorkerSegmenterError?.(event.data.error); + } + if (event.data.type === 'workerError') { + this.onWorkerContextLoss?.(); + } + if (event.data.type === 'contextLost') { + this.onWorkerContextLoss?.(); + } + if (event.data.type === 'frameProcessed') { + this.workerFrameInFlight = false; + this.workerFrameResolve?.(); + this.workerFrameResolve = null; + this.workerFrameReject = null; + } + }; + + this.worker.postMessage( + { + type: 'init', + canvas: offscreen, + width: init.outputCanvas.width, + height: init.outputCanvas.height, + devicePixelRatio: window.devicePixelRatio, + options: workerOptions, + }, + [offscreen], + ); + } + + /** + * Updates pipeline configuration at runtime. + * + * Sends configuration updates to the worker via postMessage. The worker + * applies these changes without reinitializing. If the worker is not + * initialized, this method does nothing. + * + * @param config - New pipeline configuration. + */ + public updateConfig(config: PipelineConfig): void { + if (!this.worker) { + return; + } + this.worker.postMessage({type: 'setMode', mode: config.mode}); + this.worker.postMessage({type: 'setDebugMode', debugMode: config.debugMode}); + this.worker.postMessage({type: 'setBlurStrength', blurStrength: config.blurStrength}); + this.worker.postMessage({type: 'setQuality', quality: config.quality}); + } + + /** + * Processes a video frame by sending it to the worker. + * + * Implements backpressure: if a frame is already in flight, drops the + * new frame and increments dropped frame counter. Otherwise, transfers + * the frame to the worker and waits for completion notification. + * + * The frame is transferred (not cloned) to the worker for performance. + * The promise resolves when the worker sends 'frameProcessed' message. + * + * @param frame - Input video frame as ImageBitmap (transferred to worker). + * @param timestamp - Frame timestamp in seconds. + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + * @returns Promise that resolves when worker finishes processing the frame. + */ + public async processFrame(frame: ImageBitmap, timestamp: number, width: number, height: number): Promise { + if (!this.worker) { + frame.close(); + return; + } + + if (this.workerFrameInFlight) { + const droppedCount = this.onDroppedFrame ? this.onDroppedFrame() : 0; + if (droppedCount > 0) { + this.notifyDroppedFrames(droppedCount); + } + frame.close(); + return; + } + + this.workerFrameInFlight = true; + const done = new Promise((resolve, reject) => { + this.workerFrameResolve = resolve; + this.workerFrameReject = reject; + }); + this.worker.postMessage( + { + type: 'frame', + frame, + timestamp: timestamp ?? performance.now() / 1000, + width: width ?? frame.width, + height: height ?? frame.height, + }, + [frame], + ); + await done; + } + + /** + * Sets a static background image for virtual background mode. + * + * Transfers the bitmap to the worker via postMessage. The bitmap is + * transferred (not cloned) for performance. If the worker is not + * initialized, closes the bitmap immediately. + * + * @param bitmap - Background image as ImageBitmap (transferred to worker). + * @param width - Image width in pixels. + * @param height - Image height in pixels. + */ + public setBackgroundImage(bitmap: ImageBitmap, width: number, height: number): void { + if (!this.worker) { + bitmap.close(); + return; + } + + if (bitmap.width === 0 || bitmap.height === 0) { + return; + } + + try { + this.worker.postMessage( + { + type: 'setBackgroundImage', + image: bitmap, + width, + height, + }, + [bitmap], + ); + } catch { + bitmap.close(); + } + } + + /** + * Sets a video frame as background for virtual background mode. + * + * Transfers the bitmap to the worker via postMessage with 'setBackgroundVideo' + * message type. The bitmap is transferred (not cloned) for performance. + * If the worker is not initialized, closes the bitmap immediately. + * + * @param bitmap - Background video frame as ImageBitmap (transferred to worker). + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + */ + public setBackgroundVideoFrame(bitmap: ImageBitmap, width: number, height: number): void { + if (!this.worker) { + bitmap.close(); + return; + } + this.worker.postMessage( + { + type: 'setBackgroundVideo', + video: bitmap, + width, + height, + }, + [bitmap], + ); + } + + /** + * Clears the background source. + * + * Sends a message to the worker to clear the background. If the worker + * is not initialized, does nothing. + */ + public clearBackground(): void { + if (!this.worker) { + return; + } + this.worker.postMessage({type: 'setBackgroundImage', image: null, width: 0, height: 0}); + } + + /** + * Notifies the worker of dropped frame count. + * + * Sends the dropped frame count to the worker for metrics tracking. + * The worker uses this to include dropped frames in performance metrics. + * + * @param count - Number of dropped frames. + */ + public notifyDroppedFrames(count: number): void { + if (!this.worker) { + return; + } + this.worker.postMessage({type: 'setDroppedFrames', droppedFrames: count}); + } + + /** + * Returns whether output canvas is transferred (always true after init). + * + * The canvas is transferred to the worker during init(), so it's always + * transferred after initialization completes. + * + * @returns True if canvas is transferred to worker, false if not yet initialized. + */ + public isOutputCanvasTransferred(): boolean { + return this.outputCanvasTransferred; + } + + /** + * Stops the pipeline and releases all resources. + * + * Rejects any pending frame promise, sends stop message to worker, + * terminates the worker thread, and clears all references. Should be + * called when the pipeline is no longer needed. + */ + public stop(): void { + if (this.workerFrameReject) { + this.workerFrameReject(new Error('Worker stopped')); + this.workerFrameReject = null; + this.workerFrameResolve = null; + this.workerFrameInFlight = false; + } + if (this.worker) { + this.worker.postMessage({type: 'stop'}); + this.worker.terminate(); + this.worker = null; + } + this.outputCanvasTransferred = false; + this.onMetrics = null; + this.onTierChange = null; + this.onDroppedFrame = null; + this.onWorkerSegmenterError = null; + this.onWorkerContextLoss = null; + this.lastTier = null; + } + + getCurrentModelPath(): string | null { + return ''; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/blur.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/blur.ts new file mode 100644 index 00000000000..92fb47f7c6e --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/blur.ts @@ -0,0 +1,60 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {QualityTierParams} from '../backgroundEffectsWorkerTypes'; + +/** Minimum blur radius in pixels (at blurStrength = 0). */ +const BASE_MIN_RADIUS = 2; +/** Maximum blur radius in pixels (at blurStrength = 1). */ +const BASE_MAX_RADIUS = 12; +/** Reference tier radius used for scaling calculations (tier A default). */ +const BASE_TIER_RADIUS = 4; +/** Reference downsampling scale used for scaling calculations (tier A default). */ +const BASE_DOWNSAMPLE_SCALE = 0.5; +/** Maximum blur radius supported by the shader (hardware/implementation limit). */ +const MAX_SHADER_RADIUS = 16; + +/** + * Computes the effective blur radius based on quality tier, blur strength, and downsampling. + * + * The blur radius is calculated as: + * 1. Base radius: Interpolated between BASE_MIN_RADIUS and BASE_MAX_RADIUS based on blurStrength (0-1) + * 2. Tier scale: Multiplied by (quality.blurRadius / BASE_TIER_RADIUS) to adjust for quality tier + * 3. Downsample scale: Optionally multiplied by (quality.blurDownsampleScale / BASE_DOWNSAMPLE_SCALE) + * to account for downsampled rendering (blur appears stronger at lower resolution) + * 4. Final radius: Clamped to [0, MAX_SHADER_RADIUS] + * + * @param quality - Quality tier parameters containing blurRadius and blurDownsampleScale. + * @param blurStrength - User-controlled blur strength (0-1), clamped internally. + * @param includeDownsampleScale - If true, applies downsampling scale factor to account for + * lower-resolution blur rendering making blur appear stronger. + * @returns Computed blur radius in pixels, clamped to valid shader range. + */ +export const computeBlurRadius = ( + quality: QualityTierParams, + blurStrength: number, + includeDownsampleScale: boolean, +): number => { + const clampedStrength = Math.max(0, Math.min(1, blurStrength)); + const baseRadius = BASE_MIN_RADIUS + clampedStrength * (BASE_MAX_RADIUS - BASE_MIN_RADIUS); + const tierScale = quality.blurRadius / BASE_TIER_RADIUS; + const downsampleScale = includeDownsampleScale ? quality.blurDownsampleScale / BASE_DOWNSAMPLE_SCALE : 1; + const radius = baseRadius * tierScale * downsampleScale; + return Math.max(0, Math.min(MAX_SHADER_RADIUS, radius)); +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/capabilityPolicy.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/capabilityPolicy.ts new file mode 100644 index 00000000000..ed64095a9b2 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/capabilityPolicy.ts @@ -0,0 +1,146 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {TIER_DEFINITIONS} from './definitions'; + +import type { + CapabilityInfo, + QualityPolicyMode, + QualityPolicyResult, + QualityPolicyResolver, + QualityTier, +} from '../backgroundEffectsWorkerTypes'; + +/** + * Downgrades a tier by one level (A→B→C→D). + * + * @param tier - Current tier to downgrade. + * @returns Next lower tier, or 'D' if already at minimum. + */ +const downgradeTier = (tier: QualityTier): QualityTier => { + if (tier === 'superhigh') { + return 'high'; + } + if (tier === 'high') { + return 'medium'; + } + if (tier === 'medium') { + return 'low'; + } + return 'bypass'; +}; + +/** + * Upgrades a tier by one level (D→C→B→A). + * + * @param tier - Current tier to upgrade. + * @returns Next higher tier, or 'A' if already at maximum. + */ +const upgradeTier = (tier: QualityTier): QualityTier => { + if (tier === 'bypass') { + return 'low'; + } + if (tier === 'low') { + return 'medium'; + } + if (tier === 'medium') { + return 'high'; + } + return 'superhigh'; +}; + +/** + * Determines the baseline quality tier based on browser capabilities. + * + * Tier selection priority: + * - Tier A: WebGL2 + Worker + OffscreenCanvas (full GPU acceleration in worker) + * - Tier B: WebGL2 only (GPU acceleration on main thread) + * - Tier C: Canvas2D available (CPU-based rendering fallback) + * - Tier D: No rendering capabilities (bypass mode) + * + * @param capabilities - Browser capability information. + * @returns Baseline quality tier appropriate for the available capabilities. + */ +export const baselineTierForCapabilities = (capabilities: CapabilityInfo): QualityTier => { + if (capabilities.webgl2 && capabilities.worker && capabilities.offscreenCanvas) { + return 'superhigh'; + } + if (capabilities.webgl2) { + return 'medium'; + } + if (typeof document !== 'undefined') { + return 'low'; + } + return 'bypass'; +}; + +/** + * Applies a quality policy mode to adjust the tier. + * + * Policy modes: + * - 'conservative': Downgrades tier by one level (more stable, lower quality) + * - 'aggressive': Upgrades tier by one level (higher quality, may be less stable) + * - 'balanced': No adjustment (uses baseline tier) + * + * @param tier - Baseline tier to adjust. + * @param policy - Quality policy mode to apply. + * @returns Adjusted tier based on the policy mode. + */ +export const applyPolicyMode = (tier: QualityTier, policy: QualityPolicyMode): QualityTier => { + if (policy === 'conservative') { + return downgradeTier(tier); + } + if (policy === 'aggressive') { + return upgradeTier(tier); + } + return tier; +}; + +/** + * Resolves the initial quality tier and model configuration based on capabilities and policy. + * + * Resolution process: + * 1. If policy is a function: Calls it with capabilities and returns the result + * 2. If policy is a mode string: + * a. Determines baseline tier from capabilities + * b. Applies policy mode adjustment (conservative/aggressive/balanced) + * c. Configures model path overrides (non-WebGL2 browsers use tier B model for tier A) + * + * Model path override: When WebGL2 is not available, tier A uses tier B's model path. + * + * @param capabilities - Browser capability information. + * @param policy - Quality policy mode ('conservative', 'aggressive', 'balanced') or + * a custom resolver function that returns QualityPolicyResult. + * @returns Quality policy result containing initial tier and optional model path overrides. + */ +export function resolveQualityPolicy( + capabilities: CapabilityInfo, + policy: QualityPolicyMode | QualityPolicyResolver, +): QualityPolicyResult { + if (typeof policy === 'function') { + return policy(capabilities); + } + + let initialTier = baselineTierForCapabilities(capabilities); + initialTier = applyPolicyMode(initialTier, policy); + + const segmentationModelByTier = capabilities.webgl2 ? undefined : {superhigh: TIER_DEFINITIONS.superhigh.modelPath}; + + return {initialTier, segmentationModelByTier}; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/definitions.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/definitions.ts new file mode 100644 index 00000000000..48f45763872 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/definitions.ts @@ -0,0 +1,256 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {Mode, QualityTier, QualityTierParams, SegmentationModelByTier} from '../backgroundEffectsWorkerTypes'; + +/** + * Performance tier parameters that control rendering quality and resource usage. + * These parameters are applied before mode-specific overlays. + */ +export interface PerfTierParams { + /** Quality tier identifier. */ + tier: QualityTier; + /** Width of the segmentation mask in pixels. Lower values reduce CPU/ML cost. */ + segmentationWidth: number; + /** Height of the segmentation mask in pixels. Lower values reduce CPU/ML cost. */ + segmentationHeight: number; + /** Segmentation cadence (process every Nth frame). Higher values reduce CPU/ML cost. */ + segmentationCadence: number; + /** Scale factor for mask refinement pass (0-1). Lower values reduce GPU cost. */ + maskRefineScale: number; + /** Scale factor for blur downsampling (0-1). Lower values reduce GPU cost. */ + blurDownsampleScale: number; + /** Blur radius in pixels. Lower values reduce GPU cost. */ + blurRadius: number; + /** Joint bilateral filter radius in pixels. Lower values reduce GPU cost. */ + bilateralRadius: number; + /** Spatial sigma for joint bilateral filter. Controls spatial smoothing. */ + bilateralSpatialSigma: number; + /** Range sigma for joint bilateral filter. Controls edge preservation. */ + bilateralRangeSigma: number; + /** If true, bypass all processing and pass through original frames. */ + bypass: boolean; +} + +/** + * Tier definition that groups performance parameters with the default model path. + */ +export interface TierDefinition extends PerfTierParams { + /** Default segmentation model path for this tier. */ + modelPath: string; +} + +/** + * Mode-specific overlay parameters that adjust matte processing and temporal smoothing. + * These values are applied on top of tier parameters to optimize for each effect mode. + */ +export interface ModeOverlay { + /** Temporal smoothing alpha (0-1). Higher values increase temporal stability. */ + temporalAlpha: number; + /** Lower threshold for soft matte edge (0-1). Controls where soft edges begin. */ + softLow: number; + /** Upper threshold for soft matte edge (0-1). Controls where soft edges end. */ + softHigh: number; + /** Lower threshold for matte cutoff (0-1). Pixels below this are considered background. */ + matteLow: number; + /** Upper threshold for matte cutoff (0-1). Pixels above this are considered foreground. */ + matteHigh: number; + /** Hysteresis value for matte thresholds to prevent flickering (0-1). */ + matteHysteresis: number; +} + +/** + * Quality tier definitions. + */ +export const TIER_DEFINITIONS: Record = { + superhigh: { + tier: 'superhigh', + segmentationWidth: 256, + segmentationHeight: 256, + segmentationCadence: 1, + maskRefineScale: 0.5, + blurDownsampleScale: 0.5, + blurRadius: 4, + bilateralRadius: 5, + bilateralSpatialSigma: 3.5, + bilateralRangeSigma: 0.1, + bypass: false, + modelPath: '/assets/mediapipe-models/selfie_multiclass_256x256.tflite', + }, + high: { + tier: 'high', + segmentationWidth: 256, + segmentationHeight: 256, + segmentationCadence: 1, + maskRefineScale: 0.5, + blurDownsampleScale: 0.5, + blurRadius: 4, + bilateralRadius: 5, + bilateralSpatialSigma: 3.5, + bilateralRangeSigma: 0.1, + bypass: false, + modelPath: '/assets/mediapipe-models/selfie_segmenter_landscape.tflite', + }, + medium: { + tier: 'medium', + segmentationWidth: 256, + segmentationHeight: 144, + segmentationCadence: 2, + maskRefineScale: 0.5, + blurDownsampleScale: 0.5, + blurRadius: 3, + bilateralRadius: 5, + bilateralSpatialSigma: 3.5, + bilateralRangeSigma: 0.1, + bypass: false, + modelPath: '/assets/mediapipe-models/selfie_segmenter_landscape.tflite', + }, + low: { + tier: 'low', + segmentationWidth: 160, + segmentationHeight: 96, + segmentationCadence: 3, + maskRefineScale: 0.4, + blurDownsampleScale: 0.25, + blurRadius: 2, + bilateralRadius: 3, + bilateralSpatialSigma: 2.5, + bilateralRangeSigma: 0.12, + bypass: false, + modelPath: '/assets/mediapipe-models/selfie_segmenter_landscape.tflite', + }, + bypass: { + tier: 'bypass', + segmentationWidth: 0, + segmentationHeight: 0, + segmentationCadence: 0, + maskRefineScale: 1, + blurDownsampleScale: 1, + blurRadius: 0, + bilateralRadius: 0, + bilateralSpatialSigma: 0, + bilateralRangeSigma: 0, + bypass: true, + modelPath: '/assets/mediapipe-models/selfie_segmenter_landscape.tflite', + }, +}; + +/** + * Default overlay parameters for each effect mode. + */ +export const MODE_DEFAULTS: Record = { + blur: { + temporalAlpha: 0.78, + softLow: 0.3, + softHigh: 0.65, + matteLow: 0.45, + matteHigh: 0.6, + matteHysteresis: 0.04, + }, + virtual: { + temporalAlpha: 0.62, + softLow: 0.3, + softHigh: 0.65, + matteLow: 0.45, + matteHigh: 0.6, + matteHysteresis: 0.04, + }, +}; + +/** + * Gets mode-specific overlay parameters for a given mode and tier. + * + * Applies tier-specific adjustments to the base mode defaults: + * - Tier D: Disables temporal smoothing (temporalAlpha = 0) since bypass mode doesn't need it + * - Virtual mode at Tier C: Expands matte thresholds slightly to improve edge quality at lower resolution + * + * @param mode - The effect mode ('blur' or 'virtual'). + * @param tier - The quality tier ('A', 'B', 'C', or 'D'). + * @returns Mode overlay parameters with tier-specific adjustments applied. + */ +export function getModeOverlay(mode: Mode, tier: QualityTier): ModeOverlay { + const base = MODE_DEFAULTS[mode]; + if (tier === 'bypass') { + return {...base, temporalAlpha: 0}; + } + if (mode === 'virtual' && tier === 'low') { + return {...base, matteLow: base.matteLow - 0.02, matteHigh: base.matteHigh + 0.02}; + } + return base; +} + +/** + * Applies mode-specific overlay parameters to tier parameters. + * + * Merges the base tier performance parameters with mode-specific overlay values + * to produce the final quality tier parameters. The overlay values override + * the tier defaults to optimize rendering for each effect mode. + * + * @param tier - Base performance tier parameters. + * @param mode - The effect mode ('blur' or 'virtual'). + * @returns Complete quality tier parameters with mode-specific adjustments. + */ +export function applyModeOverlay(tier: PerfTierParams, mode: Mode): QualityTierParams { + const overlay = getModeOverlay(mode, tier.tier); + return { + ...tier, + softLow: overlay.softLow, + softHigh: overlay.softHigh, + matteLow: overlay.matteLow, + matteHigh: overlay.matteHigh, + matteHysteresis: overlay.matteHysteresis, + temporalAlpha: overlay.temporalAlpha, + }; +} + +/** + * Resolves quality tier parameters for a given tier and mode. + * + * Looks up the tier definition and applies mode-specific overlays to produce + * the final quality parameters. This is a convenience function that combines + * tier lookup and mode overlay application. + * + * @param tier - The quality tier. + * @param mode - The effect mode ('blur' or 'virtual'). + * @returns Complete quality tier parameters for the specified tier and mode. + */ +export function resolveTierParams(tier: QualityTier, mode: Mode): QualityTierParams { + return applyModeOverlay(TIER_DEFINITIONS[tier], mode); +} + +/** + * Resolves the segmentation model path for a given tier. + * + * Resolution priority: + * 1. Tier-specific override from `overrides` map + * 2. Global `fallback` path + * 3. Default model path from tier definition + * + * @param tier - The quality tier. + * @param overrides - Optional map of tier-specific model path overrides. + * @param fallback - Optional fallback model path if no tier override exists. + * @returns The resolved segmentation model path for the tier. + */ +export function resolveSegmentationModelPath( + tier: QualityTier, + overrides: SegmentationModelByTier | undefined, + fallback: string | undefined, +): string { + return overrides?.[tier] ?? fallback ?? TIER_DEFINITIONS[tier].modelPath; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/index.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/index.ts new file mode 100644 index 00000000000..8b032f59ffb --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/index.ts @@ -0,0 +1,60 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Public API exports for the quality control module. + * + * This module provides: + * - QualityController: Adaptive quality tier management + * - Tier definitions and resolution utilities + * - Quality policy resolution based on browser capabilities + * - Performance metrics collection and aggregation + * - Blur radius computation utilities + */ + +export {QualityController} from './qualityController'; +export {resolveQualityPolicy, baselineTierForCapabilities, applyPolicyMode} from './capabilityPolicy'; +export {DEFAULT_TUNING, type QualityTuning} from './tuning'; +export type {PerformanceSample} from './samples'; +export { + TIER_DEFINITIONS, + resolveSegmentationModelPath, + resolveTierParams, + applyModeOverlay, + type TierDefinition, + type PerfTierParams, + type ModeOverlay, +} from './definitions'; + +export { + effectModeToProcessingMode, + getBypassTier, + resolveQualityTier, + resolveQualityTierForEffectMode, + isProcessingMode, +} from './resolve'; +export { + buildMetrics, + createMetricsWindow, + pushMetricsSample, + resetMetricsWindow, + type MetricsSample, + type MetricsWindow, +} from './metrics'; +export {computeBlurRadius} from './blur'; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/metrics.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/metrics.ts new file mode 100644 index 00000000000..8a4283ecaca --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/metrics.ts @@ -0,0 +1,140 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {PerformanceSample} from './samples'; + +import type {Metrics} from '../backgroundEffectsWorkerTypes'; + +/** + * Performance sample type alias for metrics collection. + * Represents timing measurements for a single frame. + */ +export type MetricsSample = PerformanceSample; + +/** + * Rolling window for performance metrics collection. + * + * Uses a ring buffer to maintain a fixed-size window of recent performance samples. + * Tracks running totals for efficient average calculation without re-scanning all samples. + */ +export interface MetricsWindow { + /** Ring buffer of performance samples (null slots indicate unused positions). */ + samples: Array; + /** Maximum number of samples in the window. */ + maxSamples: number; + /** Next write index into the ring buffer (wraps around). */ + index: number; + /** Number of valid samples currently in the window. */ + count: number; + /** Running totals for all samples in the window (for efficient averaging). */ + totals: {totalMs: number; segmentationMs: number; gpuMs: number}; +} + +/** + * Creates a new metrics window with the specified maximum sample count. + * + * Initializes a ring buffer structure for collecting performance samples. + * The window starts empty and fills as samples are added. + * + * @param maxSamples - Maximum number of samples to retain in the window. + * Must be at least 1 (will be clamped if less). + * @returns A new, empty metrics window ready for sample collection. + */ +export const createMetricsWindow = (maxSamples: number): MetricsWindow => { + const safeMaxSamples = Math.max(1, maxSamples); + return { + samples: new Array(safeMaxSamples).fill(null), + maxSamples: safeMaxSamples, + index: 0, + count: 0, + totals: {totalMs: 0, segmentationMs: 0, gpuMs: 0}, + }; +}; + +/** + * Resets a metrics window to its initial empty state. + * + * Clears all samples, resets counters, and zeros running totals. + * The window can be reused after reset without creating a new instance. + * + * @param window - The metrics window to reset. + */ +export const resetMetricsWindow = (window: MetricsWindow): void => { + window.samples.fill(null); + window.index = 0; + window.count = 0; + window.totals.totalMs = 0; + window.totals.segmentationMs = 0; + window.totals.gpuMs = 0; +}; + +/** + * Adds a new performance sample to the metrics window. + * + * Implements a ring buffer: when the window is full, the oldest sample is + * overwritten. Running totals are updated incrementally for efficient averaging. + * + * @param window - The metrics window to add the sample to. + * @param sample - The performance sample to add (timing measurements for one frame). + */ +export const pushMetricsSample = (window: MetricsWindow, sample: MetricsSample): void => { + const outgoing = window.samples[window.index]; + if (outgoing) { + window.totals.totalMs -= outgoing.totalMs; + window.totals.segmentationMs -= outgoing.segmentationMs; + window.totals.gpuMs -= outgoing.gpuMs; + } else { + window.count += 1; + } + + window.samples[window.index] = sample; + window.totals.totalMs += sample.totalMs; + window.totals.segmentationMs += sample.segmentationMs; + window.totals.gpuMs += sample.gpuMs; + window.index = (window.index + 1) % window.maxSamples; +}; + +/** + * Builds a Metrics object from a metrics window and additional context. + * + * Computes average timings from the window's running totals and combines + * with tier information, dropped frame count, and segmentation delegate type. + * + * @param window - The metrics window containing performance samples. + * @param droppedFrames - Number of frames dropped due to backpressure or performance issues. + * @param tier - Current quality tier ('A', 'B', 'C', or 'D'). + * @param segmentationDelegate - Segmentation execution context ('CPU', 'GPU', or null if not applicable). + * @returns Complete metrics object with averaged performance data. + */ +export const buildMetrics = ( + window: MetricsWindow, + droppedFrames: number, + tier: Metrics['tier'], + segmentationDelegate: 'CPU' | 'GPU' | null = null, +): Metrics => { + const count = window.count || 1; + return { + avgTotalMs: window.totals.totalMs / count, + avgSegmentationMs: window.totals.segmentationMs / count, + avgGpuMs: window.totals.gpuMs / count, + segmentationDelegate, + droppedFrames, + tier, + }; +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/qualityController.test.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/qualityController.test.ts new file mode 100644 index 00000000000..60b7a5c3432 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/qualityController.test.ts @@ -0,0 +1,310 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {QualityController} from './qualityController'; +import {DEFAULT_TUNING} from './tuning'; + +const MODE_BLUR = 'blur' as const; +const MODE_VIRTUAL = 'virtual' as const; + +const TARGET_FPS = 30; +const BUDGET_MS = 1000 / TARGET_FPS; +const DOWNGRADE_THRESHOLD_MS = BUDGET_MS * DEFAULT_TUNING.downgradeThresholdRatio; +const HYSTERESIS_FRAMES = DEFAULT_TUNING.hysteresisFrames; +const DOWNGRADE_MIN_SAMPLES = DEFAULT_TUNING.maxSamples * DEFAULT_TUNING.downgradeWarmupWindows; +const OVER_BUDGET_DEBT_FRAMES = DEFAULT_TUNING.overBudgetDebtFrames; +const COOLDOWN_FRAMES = DEFAULT_TUNING.cooldownFramesAfterDowngrade; + +const cpuBoundSample = {totalMs: 40, segmentationMs: 30, gpuMs: 5}; +const gpuBoundSample = {totalMs: 40, segmentationMs: 5, gpuMs: 30}; +const fastSample = {totalMs: 10, segmentationMs: 3, gpuMs: 3}; +const slowSample = {totalMs: 40, segmentationMs: 10, gpuMs: 10}; +const balancedSample = {totalMs: 40, segmentationMs: 20, gpuMs: 18}; +const veryFastSample = {totalMs: 5, segmentationMs: 1, gpuMs: 1}; + +const OVER_BUDGET_DEBT_THRESHOLD_MS = BUDGET_MS * OVER_BUDGET_DEBT_FRAMES; +const OVER_BUDGET_DELTA_MS = Math.max(1, cpuBoundSample.totalMs - DOWNGRADE_THRESHOLD_MS); +const OVER_BUDGET_TRIGGER_FRAMES = Math.ceil(OVER_BUDGET_DEBT_THRESHOLD_MS / OVER_BUDGET_DELTA_MS); +const DOWNGRADE_TRIGGER_SAMPLES = Math.max(HYSTERESIS_FRAMES, DOWNGRADE_MIN_SAMPLES, OVER_BUDGET_TRIGGER_FRAMES) + 1; + +describe('QualityController', () => { + it('downgrades CPU/ML-bound workloads to the next tier', () => { + const controller = new QualityController(TARGET_FPS); + let params; + for (let i = 0; i < DOWNGRADE_TRIGGER_SAMPLES; i += 1) { + params = controller.update(cpuBoundSample, MODE_BLUR); + } + expect(params?.tier).toBe('high'); + }); + + it('downgrades GPU-bound workloads more aggressively', () => { + const controller = new QualityController(TARGET_FPS); + let params = controller.getTier(MODE_BLUR); + + // first downgrade + let guard = 0; + while (params.tier === 'superhigh' && guard++ < 1000) { + params = controller.update(gpuBoundSample, MODE_BLUR); + } + + expect(params.tier).toBe('high'); + + // next downgrade + guard = 0; + while (params.tier === 'high' && guard++ < 1000) { + params = controller.update(gpuBoundSample, MODE_BLUR); + } + + expect(params.tier).toBe('low'); + }); + + it('applies mode-specific overlays', () => { + const controller = new QualityController(TARGET_FPS); + const blur = controller.getTier(MODE_BLUR); + const virtual = controller.getTier(MODE_VIRTUAL); + expect(blur.temporalAlpha).not.toBe(virtual.temporalAlpha); + expect(virtual.matteLow).toBeGreaterThan(0); + }); + + it('resets hysteresis when mode changes', () => { + const controller = new QualityController(TARGET_FPS); + controller.setTier('low'); + for (let i = 0; i < HYSTERESIS_FRAMES - 1; i += 1) { + controller.update(fastSample, MODE_BLUR); + } + const params = controller.update(fastSample, MODE_VIRTUAL); + expect(params.tier).toBe('low'); + }); + + it('does not oscillate within a single hysteresis window', () => { + const controller = new QualityController(TARGET_FPS); + let params; + for (let i = 0; i < HYSTERESIS_FRAMES - 5; i += 1) { + params = controller.update(i % 2 === 0 ? slowSample : fastSample, MODE_BLUR); + } + expect(params?.tier).toBe('superhigh'); + }); + + it('starts at tier superhigh', () => { + const controller = new QualityController(TARGET_FPS); + const params = controller.getTier(MODE_BLUR); + expect(params.tier).toBe('superhigh'); + }); + + it('upgrades tier when performance improves', () => { + const controller = new QualityController(TARGET_FPS); + // Start at tier C + controller.setTier('low'); + expect(controller.getTier(MODE_BLUR).tier).toBe('low'); + + // Provide fast samples to trigger upgrade + let params; + for (let i = 0; i < HYSTERESIS_FRAMES + 1; i += 1) { + params = controller.update(veryFastSample, MODE_BLUR); + } + // Should upgrade from C to B + expect(params?.tier).toBe('medium'); + + // Continue with fast samples to upgrade further + for (let i = 0; i < HYSTERESIS_FRAMES + 1; i += 1) { + params = controller.update(veryFastSample, MODE_BLUR); + } + // Should upgrade from B to A + expect(params?.tier).toBe('high'); + }); + + it('prevents immediate upgrade after downgrade due to cooldown', () => { + const controller = new QualityController(TARGET_FPS); + + // Start at tier superhigh + let params = controller.getTier(MODE_BLUR); + expect(params.tier).toBe('superhigh'); + + // Trigger downgrade with CPU-bound samples (superhigh -> high) + let guard = 0; + while (params.tier === 'superhigh' && guard++ < 1000) { + params = controller.update(cpuBoundSample, MODE_BLUR); + } + expect(guard).toBeLessThan(1000); + expect(params.tier).toBe('high'); + + // Provide fast samples immediately after downgrade + // Cooldown decrements each frame, so after 60 frames it will be 0 + // But we need to check before it expires - check after just a few frames + for (let i = 0; i < 5; i++) { + params = controller.update(veryFastSample, MODE_BLUR); + } + expect(params.tier).toBe('high'); // stay on high because cooldown > 0 + + // Enough frames for cooldown + guard = 0; + while (guard++ < COOLDOWN_FRAMES) { + params = controller.update(veryFastSample, MODE_BLUR); + } + + // Now fulfill hysteresis + guard = 0; + while (guard++ < HYSTERESIS_FRAMES + 1) { + params = controller.update(veryFastSample, MODE_BLUR); + } + + // must NOT exceed 'high' due to maxTier cap + expect(params.tier).not.toBe('superhigh'); + expect(['high', 'medium']).toContain(params.tier); + }); + + it('applies bypass mode and zero temporal alpha for tier bypass', () => { + const controller = new QualityController(TARGET_FPS); + controller.setTier('bypass'); + const params = controller.getTier(MODE_BLUR); + expect(params.tier).toBe('bypass'); + expect(params.bypass).toBe(true); + expect(params.temporalAlpha).toBe(0); + }); + + it('applies tier low virtual mode adjustments', () => { + const controller = new QualityController(TARGET_FPS); + controller.setTier('low'); + const virtualParams = controller.getTier(MODE_VIRTUAL); + const blurParams = controller.getTier(MODE_BLUR); + + // Virtual mode should apply a different matte tuning than blur at the same tier. + expect(virtualParams.matteLow).not.toBe(blurParams.matteLow); + expect(virtualParams.matteHigh).not.toBe(blurParams.matteHigh); + expect(virtualParams.matteLow).toBeLessThan(virtualParams.matteHigh); + }); + + it('calculates correct averages over sample window', () => { + const controller = new QualityController(TARGET_FPS); + + // Add some samples + controller.update({totalMs: 10, segmentationMs: 5, gpuMs: 3}, MODE_BLUR); + controller.update({totalMs: 20, segmentationMs: 10, gpuMs: 8}, MODE_BLUR); + controller.update({totalMs: 30, segmentationMs: 15, gpuMs: 12}, MODE_BLUR); + + const averages = controller.getAverages(); + expect(averages.totalMs).toBe(20); + expect(averages.segmentationMs).toBe(10); + expect(averages.gpuMs).toBeCloseTo(7.67, 1); + }); + + it('handles balanced CPU/GPU workloads', () => { + const controller = new QualityController(TARGET_FPS); + // Balanced sample: neither CPU nor GPU dominates (>55%) + let params = controller.getTier(MODE_BLUR); + let guard = 0; + while (params.tier === 'superhigh' && guard++ < 1000) { + params = controller.update(balancedSample, MODE_BLUR); + } + // Should step down normally (superhigh -> high) for balanced workloads + expect(params?.tier).toBe('high'); + expect(guard).toBeLessThan(1000); + }); + + it('manually sets tier and resets counters', () => { + const controller = new QualityController(TARGET_FPS); + // Manually set to tier C + controller.setTier('low'); + expect(controller.getTier(MODE_BLUR).tier).toBe('low'); + + // Should allow immediate tier change after setTier + controller.setTier('high'); + expect(controller.getTier(MODE_BLUR).tier).toBe('high'); + }); + + it('handles multiple tier transitions', () => { + const controller = new QualityController(TARGET_FPS); + let params = controller.getTier(MODE_BLUR); + + // superhigh -> high + let guard = 0; + while (params.tier === 'superhigh' && guard++ < 1000) { + params = controller.update(cpuBoundSample, MODE_BLUR); + } + expect(params?.tier).toBe('high'); + + // high -> medium (CPU-bound downgrade) + guard = 0; + while (params.tier === 'high' && guard++ < 1000) { + params = controller.update(cpuBoundSample, MODE_BLUR); + } + expect(params.tier).toBe('medium'); + + // medium -> low (further downgrade) + guard = 0; + while (params.tier === 'medium' && guard++ < 1000) { + params = controller.update(slowSample, MODE_BLUR); + } + expect(params?.tier).toBe('low'); + + // low -> bypass (final downgrade) + guard = 0; + while (params.tier === 'low' && guard++ < 1000) { + params = controller.update(slowSample, MODE_BLUR); + } + expect(params.tier).toBe('bypass'); + + // bypass -> low (upgrade path) + guard = 0; + while (params.tier === 'bypass' && guard++ < 1000) { + params = controller.update(veryFastSample, MODE_BLUR); + } + expect(params.tier).toBe('low'); + }); + + it('maintains sample window size limit', () => { + const controller = new QualityController(TARGET_FPS); + const sampleCount = DEFAULT_TUNING.maxSamples + 20; + + // Add more samples than maxSamples + for (let i = 0; i < sampleCount; i += 1) { + controller.update({totalMs: 10 + i, segmentationMs: 5, gpuMs: 3}, MODE_BLUR); + } + + const averages = controller.getAverages(); + const first = 10 + (sampleCount - DEFAULT_TUNING.maxSamples); + const last = 10 + (sampleCount - 1); + const expectedAverage = (first + last) / 2; + expect(averages.totalMs).toBeCloseTo(expectedAverage, 1); + }); + + it('handles boundary conditions at thresholds', () => { + const controller = new QualityController(TARGET_FPS); + const budget = 1000 / TARGET_FPS; + const upgradeThreshold = budget * DEFAULT_TUNING.upgradeThresholdRatio; + const justAboveUpgrade = upgradeThreshold + 0.5; + const clearlyBelowUpgrade = upgradeThreshold - 5; + + const atUpgradeThreshold = {totalMs: justAboveUpgrade, segmentationMs: 10, gpuMs: 8}; + controller.setTier('medium'); + let params; + for (let i = 0; i < HYSTERESIS_FRAMES + 1; i += 1) { + params = controller.update(atUpgradeThreshold, MODE_BLUR); + } + // Just above threshold, should not upgrade. + expect(params?.tier).toBe('medium'); + + const belowUpgradeThreshold = {totalMs: clearlyBelowUpgrade, segmentationMs: 9, gpuMs: 8}; + for (let i = 0; i < HYSTERESIS_FRAMES + 1; i += 1) { + params = controller.update(belowUpgradeThreshold, MODE_BLUR); + } + // Well below threshold, should upgrade. + expect(params?.tier).toBe('superhigh'); + }); +}); diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/qualityController.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/qualityController.ts new file mode 100644 index 00000000000..19de5aaaf6b --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/qualityController.ts @@ -0,0 +1,542 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {applyModeOverlay, TIER_DEFINITIONS} from './definitions'; +import type {PerformanceSample} from './samples'; +import {DEFAULT_TUNING} from './tuning'; + +import type {Mode, QualityTier, QualityTierParams} from '../backgroundEffectsWorkerTypes'; + +const DEFAULT_TARGET_FPS = 30; + +/** + * Adaptive quality controller that dynamically adjusts rendering quality tiers + * based on real-time performance measurements. + * + * The controller uses a hysteresis-based system to prevent tier oscillation: + * - Monitors frame processing time (segmentation + GPU rendering) + * - Accumulates over-budget debt and downgrades once it exceeds a threshold + * - Upgrades when performance is below the configured budget ratio + * - Requires a stability window before tier changes to prevent thrashing + * - Implements cooldown periods after downgrades to ensure stability + * + * Tier selection considers dominant cost (CPU/ML vs GPU) to optimize + * the most impactful parameters for each bottleneck. + */ +export class QualityController { + /** Maximum number of samples to retain in the rolling window. */ + private readonly maxSamples = DEFAULT_TUNING.maxSamples; + /** Rolling window of performance samples for averaging (ring buffer). */ + private readonly samples: Array = new Array(this.maxSamples).fill( + null, + ); + /** Next write index into the sample ring buffer. */ + private sampleIndex = 0; + /** Number of samples currently stored in the window. */ + private sampleCount = 0; + /** Running totals for the current sample window. */ + private readonly sampleTotals = {totalMs: 0, segmentationMs: 0, gpuMs: 0}; + /** Total number of samples observed since the last reset. */ + private totalSamplesSeen = 0; + /** Current quality tier. Starts at (highest quality). */ + private tier: QualityTier = 'superhigh'; + /** Maximum tier allowed for upgrades once performance caps are applied. */ + private maxTier: QualityTier | null = null; + /** Frame time budget in milliseconds derived from target FPS. */ + private readonly budgetMs: number; + /** Threshold in milliseconds below which we can upgrade tier. */ + private readonly upgradeThresholdMs: number; + /** Threshold in milliseconds above which we must downgrade tier. */ + private readonly downgradeThresholdMs: number; + /** EWMA smoothing factor for adaptive decisions. */ + private readonly ewmaAlpha: number; + /** Over-budget debt threshold in milliseconds before downgrading. */ + private readonly overBudgetDebtThresholdMs: number; + /** Debt recovery ratio applied when under the downgrade threshold. */ + private readonly overBudgetDebtRecoveryRatio: number; + /** Accumulated over-budget debt in milliseconds. */ + private overBudgetDebtMs = 0; + /** Exponentially weighted moving average of total frame time. */ + private ewmaTotalMs: number | null = null; + /** Exponentially weighted moving average of segmentation time. */ + private ewmaSegmentationMs: number | null = null; + /** Exponentially weighted moving average of GPU time. */ + private ewmaGpuMs: number | null = null; + /** Number of frames required for stability before allowing tier changes. */ + private readonly hysteresisFrames: number; + /** Minimum total sample count before allowing downgrades (warm-up period). */ + private readonly downgradeMinSamples: number; + /** Ratio threshold for severe over-budget detection. */ + private readonly severeDowngradeRatio: number; + /** Consecutive severe over-budget frames required to downgrade quickly. */ + private readonly severeDowngradeConfirmFrames: number; + /** Minimum sample count before allowing a severe downgrade. */ + private readonly severeDowngradeMinSamples: number; + /** Cooldown frames after downgrade before considering an upgrade. */ + private readonly cooldownFramesAfterDowngrade: number; + /** Window after an upgrade to count a downgrade as a failed upgrade. */ + private readonly upgradeFailureWindowFrames: number; + /** Maximum failed upgrade attempts before capping upgrades. */ + private readonly upgradeFailureLimit: number; + /** Ratio threshold to decide dominant cost (segmentation vs GPU). */ + private readonly dominantRatioThreshold: number; + /** Counter tracking consecutive stable frames at current tier. */ + private stableFrames = 0; + /** Counter tracking consecutive severe over-budget frames. */ + private severeOverBudgetFrames = 0; + /** Cooldown counter preventing immediate upgrades after downgrades. */ + private cooldownFrames = 0; + /** Sample index when the last upgrade occurred. */ + private lastUpgradeSample: number | null = null; + /** Count of recent failed upgrade attempts. */ + private upgradeFailureCount = 0; + /** Current effect mode to apply mode-specific overlays. */ + private currentMode: Mode | null = null; + + /** + * Creates a new quality controller. + * + * @param targetFps - Target frames per second. Used to calculate frame budget + * and performance thresholds (budget = 1000ms / targetFps). + */ + constructor(targetFps: number, maxTier: QualityTier | null = null) { + const resolvedTargetFps = Number.isFinite(targetFps) && targetFps > 0 ? targetFps : DEFAULT_TARGET_FPS; + if (process.env.NODE_ENV !== 'production') { + if (resolvedTargetFps !== targetFps) { + console.warn('[QualityController] invalid targetFps, falling back', { + targetFps, + resolvedTargetFps, + }); + } + console.info('[QualityController] init', {targetFps: resolvedTargetFps}); + } + this.maxTier = maxTier; + // Calculate frame budget: time available per frame to maintain target FPS + const budget = 1000 / resolvedTargetFps; + this.budgetMs = budget; + // Upgrade threshold: require more headroom to increase quality + this.upgradeThresholdMs = budget * DEFAULT_TUNING.upgradeThresholdRatio; + // Downgrade threshold: approaching or exceeding the frame budget + this.downgradeThresholdMs = budget * DEFAULT_TUNING.downgradeThresholdRatio; + this.ewmaAlpha = DEFAULT_TUNING.ewmaAlpha; + this.overBudgetDebtThresholdMs = budget * DEFAULT_TUNING.overBudgetDebtFrames; + this.overBudgetDebtRecoveryRatio = DEFAULT_TUNING.overBudgetDebtRecoveryRatio; + this.hysteresisFrames = DEFAULT_TUNING.hysteresisFrames; + // Require full windows before downgrading to avoid cold-start spikes. + this.downgradeMinSamples = this.maxSamples * DEFAULT_TUNING.downgradeWarmupWindows; + this.severeDowngradeRatio = DEFAULT_TUNING.severeDowngradeRatio; + this.severeDowngradeConfirmFrames = DEFAULT_TUNING.severeDowngradeConfirmFrames; + this.severeDowngradeMinSamples = DEFAULT_TUNING.severeDowngradeWarmupFrames; + this.cooldownFramesAfterDowngrade = DEFAULT_TUNING.cooldownFramesAfterDowngrade; + this.upgradeFailureWindowFrames = DEFAULT_TUNING.upgradeFailureWindowFrames; + this.upgradeFailureLimit = DEFAULT_TUNING.upgradeFailureLimit; + this.dominantRatioThreshold = DEFAULT_TUNING.dominantRatioThreshold; + } + + /** + * Gets the current quality tier parameters for the specified mode. + * Applies mode-specific overlays to the base tier parameters. + * + * @param mode - The effect mode ('blur' or 'virtual'). + * @returns Quality tier parameters with mode-specific adjustments applied. + */ + public getTier(mode: Mode): QualityTierParams { + this.handleModeChange(mode); + return applyModeOverlay(TIER_DEFINITIONS[this.tier], mode); + } + + /** + * Manually sets the quality tier, bypassing adaptive logic. + * Resets stability counters to allow immediate tier changes. + * + * @param tier - The quality tier to set ('A', 'B', 'C', or 'D'). + */ + public setTier(tier: QualityTier): void { + this.tier = tier; + this.stableFrames = 0; + this.cooldownFrames = 0; + this.resetSamples(); + } + + /** + * Updates the quality controller with a new performance sample and returns + * the current quality tier parameters. + * + * This method implements the adaptive quality algorithm: + * 1. Adds the sample to the rolling window + * 2. Calculates average performance over the window + * 3. Evaluates tier changes based on thresholds and stability requirements + * 4. Applies mode-specific overlays to the tier parameters + * + * @param sample - Performance measurements for the last processed frame. + * @param mode - The current effect mode ('blur' or 'virtual'). + * @returns Quality tier parameters with mode-specific adjustments applied. + */ + public update(sample: PerformanceSample, mode: Mode): QualityTierParams { + this.handleModeChange(mode); + + // Maintain rolling window of samples for averaging + this.addSample(sample); + this.totalSamplesSeen += 1; + this.updateEwma(sample); + const ewmaTotalMs = this.ewmaTotalMs ?? sample.totalMs; + const ewmaSegmentationMs = this.ewmaSegmentationMs ?? sample.segmentationMs; + const ewmaGpuMs = this.ewmaGpuMs ?? sample.gpuMs; + const overBudgetRatio = ewmaTotalMs / this.budgetMs; + + // Increment stability counter and decrement cooldown + this.stableFrames += 1; + if (this.cooldownFrames > 0) { + this.cooldownFrames -= 1; + } + const debtDelta = ewmaTotalMs - this.downgradeThresholdMs; + if (debtDelta > 0) { + this.overBudgetDebtMs += debtDelta; + } else { + this.overBudgetDebtMs = Math.max(0, this.overBudgetDebtMs + debtDelta * this.overBudgetDebtRecoveryRatio); + } + if (overBudgetRatio >= this.severeDowngradeRatio) { + this.severeOverBudgetFrames += 1; + } else { + this.severeOverBudgetFrames = 0; + } + + const isOverBudget = debtDelta > 0; + const canDowngradeSevere = + this.severeOverBudgetFrames >= this.severeDowngradeConfirmFrames && + this.totalSamplesSeen >= this.severeDowngradeMinSamples; + const canDowngradeNormal = + this.stableFrames >= this.hysteresisFrames && + this.overBudgetDebtMs >= this.overBudgetDebtThresholdMs && + this.totalSamplesSeen >= this.downgradeMinSamples && + isOverBudget; + + // Downgrade if performance exceeds threshold (severe path can bypass hysteresis) + if (canDowngradeSevere || canDowngradeNormal) { + // Determine dominant cost to optimize downgrade strategy + const dominant = this.getDominantCost(ewmaTotalMs, ewmaSegmentationMs, ewmaGpuMs); + const nextTier = this.downgradeTier(dominant); + if (nextTier !== this.tier) { + if (this.tier === 'superhigh') { + this.applyMaxTierCap('high'); + } + this.registerUpgradeFailure(nextTier); + if (process.env.NODE_ENV !== 'production') { + console.info('[QualityController] downgrade', { + from: this.tier, + to: nextTier, + ewmaTotalMs, + ewmaSegmentationMs, + ewmaGpuMs, + overBudgetDebtMs: this.overBudgetDebtMs, + overBudgetDebtThresholdMs: this.overBudgetDebtThresholdMs, + severeOverBudgetFrames: this.severeOverBudgetFrames, + overBudgetRatio, + sampleCount: this.sampleCount, + maxTier: this.maxTier, + }); + } + this.tier = nextTier; + this.stableFrames = 0; + this.severeOverBudgetFrames = 0; + this.overBudgetDebtMs = 0; + // Apply cooldown to prevent immediate re-upgrade + this.cooldownFrames = this.cooldownFramesAfterDowngrade; + } + } + // Upgrade if performance is well below threshold and cooldown expired + else if ( + this.stableFrames >= this.hysteresisFrames && + ewmaTotalMs < this.upgradeThresholdMs && + this.cooldownFrames === 0 + ) { + const nextTier = this.upgradeTier(); + if (nextTier !== this.tier) { + if (process.env.NODE_ENV !== 'production') { + console.info('[QualityController] upgrade', { + from: this.tier, + to: nextTier, + ewmaTotalMs, + ewmaSegmentationMs, + ewmaGpuMs, + sampleCount: this.sampleCount, + maxTier: this.maxTier, + }); + } + this.tier = nextTier; + this.stableFrames = 0; + this.severeOverBudgetFrames = 0; + this.lastUpgradeSample = this.totalSamplesSeen; + } + } + + return applyModeOverlay(TIER_DEFINITIONS[this.tier], mode); + } + + /** + * Gets the average performance metrics over the current sample window. + * Useful for monitoring and debugging performance characteristics. + * + * @returns Average timing measurements across all samples in the window. + */ + public getAverages(): PerformanceSample { + const denom = Math.max(1, this.sampleCount); + return { + totalMs: this.sampleTotals.totalMs / denom, + segmentationMs: this.sampleTotals.segmentationMs / denom, + gpuMs: this.sampleTotals.gpuMs / denom, + }; + } + + /** + * Updates exponentially weighted moving averages for decision-making. + * + * Uses EWMA to smooth transient spikes while remaining responsive to trends. + * + * @param sample - Performance measurements for the frame to add. + */ + private updateEwma(sample: PerformanceSample): void { + if (this.ewmaTotalMs === null || this.ewmaSegmentationMs === null || this.ewmaGpuMs === null) { + this.ewmaTotalMs = sample.totalMs; + this.ewmaSegmentationMs = sample.segmentationMs; + this.ewmaGpuMs = sample.gpuMs; + return; + } + const alpha = this.ewmaAlpha; + const invAlpha = 1 - alpha; + this.ewmaTotalMs = alpha * sample.totalMs + invAlpha * this.ewmaTotalMs; + this.ewmaSegmentationMs = alpha * sample.segmentationMs + invAlpha * this.ewmaSegmentationMs; + this.ewmaGpuMs = alpha * sample.gpuMs + invAlpha * this.ewmaGpuMs; + } + + /** + * Adds a performance sample to the rolling window using a ring buffer. + * + * Implements efficient O(1) insertion by: + * - Removing the outgoing sample's contribution from running totals (if window is full) + * - Adding the new sample's contribution to running totals + * - Updating the write index with wraparound + * - Tracking sample count (increments only when window is not yet full) + * + * @param sample - Performance measurements for the frame to add. + */ + private addSample(sample: PerformanceSample): void { + const outgoing = this.samples[this.sampleIndex]; + if (outgoing) { + this.sampleTotals.totalMs -= outgoing.totalMs; + this.sampleTotals.segmentationMs -= outgoing.segmentationMs; + this.sampleTotals.gpuMs -= outgoing.gpuMs; + } else { + this.sampleCount += 1; + } + + this.samples[this.sampleIndex] = sample; + this.sampleTotals.totalMs += sample.totalMs; + this.sampleTotals.segmentationMs += sample.segmentationMs; + this.sampleTotals.gpuMs += sample.gpuMs; + this.sampleIndex = (this.sampleIndex + 1) % this.maxSamples; + } + + /** + * Resets the sample window to an empty state. + * + * Clears all samples, resets counters, and zeros running totals. + * Used when manually setting tier or when mode changes to allow immediate + * re-evaluation without stale data. + */ + private resetSamples(): void { + this.samples.fill(null); + this.sampleIndex = 0; + this.sampleCount = 0; + this.sampleTotals.totalMs = 0; + this.sampleTotals.segmentationMs = 0; + this.sampleTotals.gpuMs = 0; + this.totalSamplesSeen = 0; + this.ewmaTotalMs = null; + this.ewmaSegmentationMs = null; + this.ewmaGpuMs = null; + this.severeOverBudgetFrames = 0; + this.overBudgetDebtMs = 0; + this.lastUpgradeSample = null; + this.upgradeFailureCount = 0; + } + + /** + * Handles mode changes by resetting stability counters. + * Mode changes can affect performance, so we reset hysteresis to allow + * immediate tier re-evaluation. + * + * @param mode - The new effect mode. + */ + private handleModeChange(mode: Mode): void { + if (this.currentMode !== mode) { + this.currentMode = mode; + this.stableFrames = 0; + this.cooldownFrames = 0; + this.resetSamples(); + } + } + + /** + * Determines the dominant performance bottleneck from timing measurements. + * Used to optimize tier downgrade strategy (e.g., skip tiers for GPU-bound cases). + * + * @param avgTotalMs - Average total processing time in milliseconds. + * @param avgSegmentationMs - Average segmentation time in milliseconds. + * @param avgGpuMs - Average GPU rendering time in milliseconds. + * @returns 'cpu' if segmentation dominates (>55%), 'gpu' if GPU dominates (>55%), + * otherwise 'balanced'. + */ + private getDominantCost(avgTotalMs: number, avgSegmentationMs: number, avgGpuMs: number): 'cpu' | 'gpu' | 'balanced' { + if (avgTotalMs <= 0) { + return 'balanced'; + } + const segRatio = avgSegmentationMs / avgTotalMs; + const gpuRatio = avgGpuMs / avgTotalMs; + // Threshold indicates dominant cost component + if (segRatio > this.dominantRatioThreshold) { + return 'cpu'; + } + if (gpuRatio > this.dominantRatioThreshold) { + return 'gpu'; + } + return 'balanced'; + } + + /** + * Determines the next lower quality tier based on current tier and dominant cost. + * + * Downgrade policy: + * - CPU/ML bound: Step down normally (A→B→C→D) to reduce segmentation cadence/resolution. + * - GPU bound: Skip from A to C for a stronger GPU cost reduction (blur/bilateral parameters). + * - Balanced: Step down normally. + * + * @param dominant - The dominant performance bottleneck ('cpu', 'gpu', or 'balanced'). + * @returns The next lower quality tier. + */ + private downgradeTier(dominant: 'cpu' | 'gpu' | 'balanced'): QualityTier { + if (this.tier === 'superhigh') { + return 'high'; + } + + if (this.tier === 'high') { + // GPU-bound: skip medium tier for stronger GPU cost reduction + // CPU-bound: normal step down to B + return dominant === 'gpu' ? 'low' : 'medium'; + } + if (this.tier === 'medium') { + return 'low'; + } + if (this.tier === 'low') { + return 'bypass'; + } + + // last case + return 'bypass'; + } + + /** + * Determines the next higher quality tier based on current tier. + * Always steps up one tier at a time (D→C→B→A) for gradual quality improvement. + * + * @returns The next higher quality tier, or current tier if already at maximum. + */ + private upgradeTier(): QualityTier { + if (this.tier === 'superhigh') { + return 'superhigh'; + } + if (this.tier === 'bypass') { + return this.canUpgradeTo('low') ? 'low' : 'bypass'; + } + if (this.tier === 'low') { + return this.canUpgradeTo('medium') ? 'medium' : 'low'; + } + + if (this.tier === 'medium') { + return this.canUpgradeTo('high') ? 'high' : 'medium'; + } + // Tier can only go to superhigh (maximum quality) + return this.tier === 'high' && this.canUpgradeTo('superhigh') ? 'superhigh' : 'high'; + } + + /** + * Tracks failed upgrade attempts and caps future upgrades when necessary. + * + * If a downgrade happens soon after an upgrade, it is treated as a failed + * upgrade attempt. After a configurable number of failures, upgrades are + * capped at the downgraded tier to avoid repeated oscillation. + * + * @param nextTier - The tier we are downgrading to. + */ + private registerUpgradeFailure(nextTier: QualityTier): void { + if (this.lastUpgradeSample === null) { + return; + } + const framesSinceUpgrade = this.totalSamplesSeen - this.lastUpgradeSample; + if (framesSinceUpgrade <= this.upgradeFailureWindowFrames) { + this.upgradeFailureCount += 1; + if (this.upgradeFailureCount >= this.upgradeFailureLimit) { + this.applyMaxTierCap(nextTier); + } + } + this.lastUpgradeSample = null; + } + + /** + * Applies a maximum tier cap, keeping the most restrictive cap. + * + * @param cap - The highest tier allowed for future upgrades. + */ + private applyMaxTierCap(cap: QualityTier): void { + if (!this.maxTier || this.getTierRank(cap) < this.getTierRank(this.maxTier)) { + this.maxTier = cap; + if (process.env.NODE_ENV !== 'production') { + console.info('[QualityController] maxTier cap set', {maxTier: this.maxTier}); + } + } + } + + /** + * Checks if an upgrade to the specified tier is allowed. + * + * When performance has been capped (maxTier is set), upgrades are restricted + * to prevent exceeding the maximum tier that was reached before the cap. + * This prevents oscillation after a performance-based downgrade from tier A. + * + * @param tier - The tier to check upgrade eligibility for. + * @returns True if upgrade to the tier is allowed, false if it exceeds maxTier. + */ + private canUpgradeTo(tier: QualityTier): boolean { + if (!this.maxTier) { + return true; + } + return this.getTierRank(tier) <= this.getTierRank(this.maxTier); + } + + private getTierRank(tier: QualityTier): number { + const rank: Record = { + bypass: 0, + low: 1, + medium: 2, + high: 3, + superhigh: 4, + }; + return rank[tier]; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/resolve.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/resolve.ts new file mode 100644 index 00000000000..54ccc77970a --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/resolve.ts @@ -0,0 +1,124 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {resolveTierParams} from './definitions'; + +import type {EffectMode, Mode, QualityMode, QualityTierParams} from '../backgroundEffectsWorkerTypes'; + +/** + * Interface for objects that can provide quality tier parameters. + * Used to abstract over QualityController instances for testing and flexibility. + */ +export interface QualityControllerLike { + /** + * Gets the current quality tier parameters for the specified mode. + * + * @param mode - The effect mode ('blur' or 'virtual'). + * @returns Quality tier parameters with mode-specific adjustments applied. + */ + getTier(mode: Mode): QualityTierParams; +} + +/** + * Converts an effect mode to its corresponding processing mode. + * + * Maps 'virtual' to 'virtual' and all other modes (including 'blur') to 'blur'. + * The 'passthrough' mode is not a processing mode and should be handled separately. + * + * @param mode - The effect mode to convert. + * @returns The corresponding processing mode ('blur' or 'virtual'). + */ +export const effectModeToProcessingMode = (mode: EffectMode): Mode => (mode === 'virtual' ? 'virtual' : 'blur'); + +/** + * Type guard that checks if an effect mode is a processing mode. + * + * Processing modes require actual background effect processing, while 'passthrough' + * bypasses all processing and returns the original video stream. + * + * @param mode - The effect mode to check. + * @returns True if the mode is a processing mode ('blur' or 'virtual'), false if 'passthrough'. + */ +export const isProcessingMode = (mode: EffectMode): mode is Mode => mode !== 'passthrough'; + +/** + * Gets the bypass tier (tier D) parameters for a given mode. + * + * Tier D bypasses all processing and passes through original frames, but still + * needs mode-specific parameters for consistency. + * + * @param mode - The effect mode ('blur' or 'virtual'). + * @returns Bypass tier parameters with mode-specific adjustments. + */ +export const getBypassTier = (mode: Mode): QualityTierParams => resolveTierParams('bypass', mode); + +/** Default mode to use when resolving bypass tier for non-processing modes. */ +const DEFAULT_BYPASS_MODE: Mode = 'blur'; + +/** + * Resolves quality tier parameters for an effect mode. + * + * Handles both processing modes ('blur', 'virtual') and non-processing modes ('passthrough'): + * - For 'passthrough': Returns bypass tier (tier D) with default mode + * - For processing modes: Delegates to resolveQualityTier + * + * @param qualityController - Optional quality controller for adaptive quality ('auto' mode). + * @param quality - Quality mode ('auto' or fixed tier 'A'/'B'/'C'/'D'). + * @param mode - The effect mode ('blur', 'virtual', or 'passthrough'). + * @returns Quality tier parameters for the specified quality and effect mode. + */ +export const resolveQualityTierForEffectMode = ( + qualityController: QualityControllerLike | null, + quality: QualityMode, + mode: EffectMode, +): QualityTierParams => { + if (!isProcessingMode(mode)) { + return getBypassTier(DEFAULT_BYPASS_MODE); + } + return resolveQualityTier(qualityController, quality, mode); +}; + +/** + * Resolves quality tier parameters for a processing mode. + * + * Resolution logic: + * - Fixed quality ('A'/'B'/'C'/'D'): Returns tier parameters directly + * - Adaptive quality ('auto'): + * - If quality controller available: Gets current tier from controller + * - If no controller: Falls back to bypass tier (tier D) + * + * @param qualityController - Optional quality controller for adaptive quality ('auto' mode). + * @param quality - Quality mode ('auto' or fixed tier 'A'/'B'/'C'/'D'). + * @param mode - The processing mode ('blur' or 'virtual'). + * @returns Quality tier parameters for the specified quality and mode. + */ +export const resolveQualityTier = ( + qualityController: QualityControllerLike | null, + quality: QualityMode, + mode: Mode, +): QualityTierParams => { + if (quality !== 'auto') { + return resolveTierParams(quality, mode); + } + if (!qualityController) { + return getBypassTier(mode); + } + + return qualityController.getTier(mode); +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/samples.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/samples.ts new file mode 100644 index 00000000000..d5446dc52fc --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/samples.ts @@ -0,0 +1,30 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Performance sample containing timing measurements for a single frame. + */ +export interface PerformanceSample { + /** Total processing time in milliseconds (segmentation + GPU rendering). */ + totalMs: number; + /** Time spent on ML segmentation in milliseconds. */ + segmentationMs: number; + /** Time spent on GPU rendering operations in milliseconds. */ + gpuMs: number; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/quality/tuning.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/tuning.ts new file mode 100644 index 00000000000..9845fe5bc7c --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/quality/tuning.ts @@ -0,0 +1,53 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +export const DEFAULT_TUNING = { + // Rolling window size for averaging performance samples. + maxSamples: 30, + // Upgrade when we are comfortably under budget (ratio of frame budget). + upgradeThresholdRatio: 0.5, + // Downgrade when we approach or exceed budget (ratio of frame budget). + downgradeThresholdRatio: 1.0, + // EWMA smoothing factor for adaptive decisions (0-1). Higher = more responsive. + ewmaAlpha: 0.2, + // Over-budget debt threshold expressed as N frame budgets before downgrade. + overBudgetDebtFrames: 10, + // How quickly debt is repaid when under the downgrade threshold (0-1). + overBudgetDebtRecoveryRatio: 1, + // Ratio threshold for triggering fast downgrades when severely over budget. + severeDowngradeRatio: 2.0, + // Consecutive severe over-budget frames required to downgrade quickly. + severeDowngradeConfirmFrames: 4, + // Minimum frames before allowing severe downgrade (settling period). + severeDowngradeWarmupFrames: 30, + // Frames required before considering any tier change. + hysteresisFrames: 60, + // Require N full sample windows before allowing downgrade. + downgradeWarmupWindows: 3, + // Cooldown frames after downgrade before considering upgrade. + cooldownFramesAfterDowngrade: 60, + // Window after an upgrade to consider a downgrade as a failed upgrade. + upgradeFailureWindowFrames: 120, + // Maximum failed upgrade attempts before capping upgrades. + upgradeFailureLimit: 2, + // Ratio threshold to decide dominant cost (segmentation vs GPU). + dominantRatioThreshold: 0.55, +} as const; + +export type QualityTuning = typeof DEFAULT_TUNING; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/backgroundRenderer.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/backgroundRenderer.ts new file mode 100644 index 00000000000..a05b9309c72 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/backgroundRenderer.ts @@ -0,0 +1,125 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {Size} from './webGlResources'; + +/** + * Manages background image texture for virtual background rendering. + * + * Handles WebGL texture creation, upload, and lifecycle for background images + * used in virtual background mode. Provides utilities for calculating cover + * scale to ensure background images fill the output canvas appropriately. + */ +export class BackgroundRenderer { + private texture: WebGLTexture | null = null; + private size: Size | null = null; + + constructor(private readonly gl: WebGL2RenderingContext) {} + + /** + * Sets or clears the background image for virtual background mode. + * + * Uploads the image to a WebGL texture and stores its dimensions. If null + * is provided, releases the existing texture. Reuses existing texture if + * available to avoid unnecessary allocations. + * + * @param image - Background image as ImageBitmap, or null to clear. + * @param width - Image width in pixels. + * @param height - Image height in pixels. + * @returns Nothing. + */ + public setBackground(image: ImageBitmap | null, width: number, height: number): void { + if (!image) { + if (this.texture) { + this.gl.deleteTexture(this.texture); + } + this.texture = null; + this.size = null; + return; + } + + const texture = this.texture ?? this.gl.createTexture(); + if (!texture) { + return; + } + + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE); + this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, 0); + this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, image); + + this.texture = texture; + this.size = {width, height}; + } + + /** + * Returns the WebGL texture containing the background image. + * + * @returns WebGL texture, or null if no background is set. + */ + public getTexture(): WebGLTexture | null { + return this.texture; + } + + /** + * Returns the dimensions of the background image. + * + * @returns Size object with width and height, or null if no background is set. + */ + public getSize(): Size | null { + return this.size; + } + + /** + * Calculates scale factors for covering target size with background image. + * + * Computes scale factors that ensure the background image covers the entire + * target area while maintaining aspect ratio. Uses the larger of width/height + * ratios to ensure complete coverage. + * + * @param target - Target size to cover. + * @returns Tuple [scaleX, scaleY] for scaling the background texture coordinates. + */ + public getCoverScale(target: Size): [number, number] { + if (!this.size || this.size.width === 0 || this.size.height === 0) { + return [1, 1]; + } + const scale = Math.max(target.width / this.size.width, target.height / this.size.height); + return [(this.size.width * scale) / target.width, (this.size.height * scale) / target.height]; + } + + /** + * Destroys the background renderer and releases WebGL resources. + * + * Deletes the WebGL texture and clears all references. Should be called + * when the renderer is no longer needed to prevent memory leaks. + * + * @returns Nothing. + */ + public destroy(): void { + if (this.texture) { + this.gl.deleteTexture(this.texture); + } + this.texture = null; + this.size = null; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/renderPasses.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/renderPasses.ts new file mode 100644 index 00000000000..5cc343915ae --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/renderPasses.ts @@ -0,0 +1,87 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {ShaderPrograms} from './shaderPrograms'; +import type {Size, WebGlResources} from './webGlResources'; + +/** + * Helper class for executing WebGL render passes. + * + * Provides utilities for drawing to framebuffers and the default framebuffer + * using shader programs. Handles viewport setup, program binding, and uniform + * configuration. + */ +export class RenderPasses { + constructor( + private readonly gl: WebGL2RenderingContext, + private readonly programs: ShaderPrograms, + private readonly resources: WebGlResources, + ) {} + + /** + * Draws to a texture using a framebuffer. + * + * Binds the specified framebuffer, sets viewport, and draws a fullscreen + * quad using the specified shader program. Sets uFlipY to 0 for texture rendering. + * + * @param programKey - Key identifying the shader program to use. + * @param targetKey - Key identifying the target framebuffer/texture. + * @param size - Viewport size (width and height). + * @param uniforms - Uniform values to pass to the shader program. + * @returns Nothing. + */ + public drawToTexture(programKey: string, targetKey: string, size: Size, uniforms: Record): void { + const fbo = this.resources.getFramebuffer(targetKey); + if (!fbo) { + return; + } + this.gl.bindFramebuffer(this.gl.FRAMEBUFFER, fbo); + this.gl.viewport(0, 0, size.width, size.height); + this.programs.use(programKey, {...uniforms, uFlipY: 0.0}); + this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + } + + /** + * Draws to a framebuffer or the default framebuffer (screen). + * + * Binds the specified framebuffer (or null for default), sets viewport, + * and draws a fullscreen quad. Automatically sets uFlipY based on whether + * rendering to texture (0) or screen (1). + * + * @param programKey - Key identifying the shader program to use. + * @param fbo - Framebuffer to render to, or null for default framebuffer. + * @param width - Viewport width in pixels. + * @param height - Viewport height in pixels. + * @param uniforms - Uniform values to pass to the shader program. + * @returns Nothing. + */ + public drawSimple( + programKey: string, + fbo: WebGLFramebuffer | null, + width: number, + height: number, + uniforms: Record, + ): void { + this.gl.bindFramebuffer(this.gl.FRAMEBUFFER, fbo); + this.gl.viewport(0, 0, width, height); + const flipY = fbo === null ? 1.0 : 0.0; + this.programs.use(programKey, {...uniforms, uFlipY: flipY}); + this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/shaderPrograms.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/shaderPrograms.ts new file mode 100644 index 00000000000..35df10724b3 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/shaderPrograms.ts @@ -0,0 +1,259 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +// @ts-ignore +import compositeBlurFrag from '../shaders/compositeBlur.frag'; +// @ts-ignore +import compositeVirtualFrag from '../shaders/compositeVirtual.frag'; +// @ts-ignore +import debugOverlayFrag from '../shaders/debugOverlay.frag'; +// @ts-ignore +import downsampleFrag from '../shaders/downsample.frag'; +// @ts-ignore +import fullscreenVert from '../shaders/fullscreen.vert'; +// @ts-ignore +import gaussianBlurHFrag from '../shaders/gaussianBlurH.frag'; +// @ts-ignore +import gaussianBlurVFrag from '../shaders/gaussianBlurV.frag'; +// @ts-ignore +import jointBilateralFrag from '../shaders/jointBilateralMask.frag'; +// @ts-ignore +import maskUpsampleFrag from '../shaders/maskUpsample.frag'; +// @ts-ignore +import temporalMaskFrag from '../shaders/temporalMask.frag'; + +/** + * Information about a compiled WebGL shader program. + */ +export interface ProgramInfo { + /** Compiled and linked WebGL program. */ + program: WebGLProgram; + /** Map of uniform names to their locations in the program. */ + uniforms: Record; +} + +/** + * Manages WebGL shader programs for background effects rendering. + * + * Compiles and links vertex/fragment shader pairs, manages uniform locations, + * and provides a unified interface for setting uniforms and binding programs. + * Supports various uniform types: textures, 2D vectors, floats, and debug mode strings. + */ +export class ShaderPrograms { + private readonly programs: Record; + + constructor(private readonly gl: WebGL2RenderingContext) { + this.programs = this.createPrograms(); + } + + /** + * Activates a shader program and sets its uniforms. + * + * Binds the specified program and configures all provided uniforms. + * Automatically handles different uniform types: + * - WebGLTexture: Binds to texture units sequentially + * - Arrays: Sets as vec2 uniforms + * - Numbers: Sets as float uniforms + * - Strings: Maps debug mode strings to integers + * + * @param programKey - Key identifying the shader program to use. + * @param uniforms - Map of uniform names to values. + * @returns Nothing. + */ + public use(programKey: string, uniforms: Record): void { + const programInfo = this.programs[programKey]; + if (!programInfo) { + return; + } + + const gl = this.gl; + gl.useProgram(programInfo.program); + + let textureUnit = 0; + Object.entries(uniforms).forEach(([name, value]) => { + const location = programInfo.uniforms[name]; + if (!location) { + return; + } + if (value instanceof WebGLTexture) { + gl.activeTexture(gl.TEXTURE0 + textureUnit); + gl.bindTexture(gl.TEXTURE_2D, value); + gl.uniform1i(location, textureUnit); + textureUnit += 1; + return; + } + if (Array.isArray(value)) { + gl.uniform2f(location, value[0], value[1]); + return; + } + if (typeof value === 'number') { + gl.uniform1f(location, value); + return; + } + if (typeof value === 'string') { + const modeMap: Record = { + off: 0, + maskOverlay: 1, + maskOnly: 2, + edgeOnly: 3, + classOverlay: 4, + classOnly: 5, + }; + gl.uniform1i(location, modeMap[value] ?? 0); + } + }); + } + + /** + * Destroys all shader programs and releases WebGL resources. + * + * Deletes all compiled programs. Should be called when the renderer is + * no longer needed to prevent memory leaks. + * + * @returns Nothing. + */ + public destroy(): void { + Object.values(this.programs).forEach(({program}) => this.gl.deleteProgram(program)); + } + + /** + * Creates all shader programs used by the renderer. + * + * Compiles and links all vertex/fragment shader pairs for the rendering + * pipeline: downsample, mask upsampling, joint bilateral filtering, temporal + * smoothing, blur passes, compositing, and debug overlay. + * + * @returns Map of program keys to ProgramInfo objects. + */ + private createPrograms(): Record { + return { + downsample: this.createProgram(fullscreenVert, downsampleFrag, ['uSrc', 'uTexelSize', 'uFlipY']), + maskUpsample: this.createProgram(fullscreenVert, maskUpsampleFrag, ['uSrc', 'uTexelSize', 'uFlipY']), + jointBilateral: this.createProgram(fullscreenVert, jointBilateralFrag, [ + 'uMask', + 'uVideo', + 'uTexelSize', + 'uSpatialSigma', + 'uRangeSigma', + 'uRadius', + 'uFlipY', + ]), + temporalMask: this.createProgram(fullscreenVert, temporalMaskFrag, [ + 'uMask', + 'uPrevMask', + 'uTexelSize', + 'uAlpha', + 'uFlipY', + ]), + blurH: this.createProgram(fullscreenVert, gaussianBlurHFrag, ['uSrc', 'uTexelSize', 'uRadius', 'uFlipY']), + blurV: this.createProgram(fullscreenVert, gaussianBlurVFrag, ['uSrc', 'uTexelSize', 'uRadius', 'uFlipY']), + compositeBlur: this.createProgram(fullscreenVert, compositeBlurFrag, [ + 'uVideo', + 'uBlur', + 'uMask', + 'uTexelSize', + 'uSoftLow', + 'uSoftHigh', + 'uBlurStrength', + 'uFlipY', + ]), + compositeVirtual: this.createProgram(fullscreenVert, compositeVirtualFrag, [ + 'uVideo', + 'uMask', + 'uBg', + 'uTexelSize', + 'uMatteLow', + 'uMatteHigh', + 'uBgScale', + 'uFlipY', + ]), + debugOverlay: this.createProgram(fullscreenVert, debugOverlayFrag, ['uVideo', 'uMask', 'uMode', 'uFlipY']), + compositePassthrough: this.createProgram(fullscreenVert, downsampleFrag, ['uSrc', 'uTexelSize', 'uFlipY']), + }; + } + + /** + * Creates a single shader program from vertex and fragment shader sources. + * + * Compiles both shaders, links them into a program, and extracts uniform + * locations. Throws an error if compilation or linking fails. + * + * @param vertexSource - Vertex shader source code. + * @param fragmentSource - Fragment shader source code. + * @param uniforms - Array of uniform names to extract locations for. + * @returns ProgramInfo containing the program and uniform locations. + * @throws Error if shader compilation or program linking fails. + */ + private createProgram(vertexSource: string, fragmentSource: string, uniforms: string[]): ProgramInfo { + const gl = this.gl; + const vertexShader = this.compileShader(gl.VERTEX_SHADER, vertexSource); + const fragmentShader = this.compileShader(gl.FRAGMENT_SHADER, fragmentSource); + + const program = gl.createProgram(); + if (!program) { + throw new Error('Failed to create WebGL program'); + } + + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.bindAttribLocation(program, 0, 'aPosition'); + gl.bindAttribLocation(program, 1, 'aTexCoord'); + gl.linkProgram(program); + + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error(gl.getProgramInfoLog(program) || 'Failed to link program'); + } + + const uniformLocations: Record = {}; + uniforms.forEach(name => { + uniformLocations[name] = gl.getUniformLocation(program, name); + }); + + return {program, uniforms: uniformLocations}; + } + + /** + * Compiles a shader from source code. + * + * Creates a shader of the specified type, compiles it, and returns the + * compiled shader. Throws an error if compilation fails. + * + * @param type - Shader type (gl.VERTEX_SHADER or gl.FRAGMENT_SHADER). + * @param source - Shader source code. + * @returns Compiled WebGL shader. + * @throws Error if shader compilation fails. + */ + private compileShader(type: number, source: string): WebGLShader { + const gl = this.gl; + const shader = gl.createShader(type); + if (!shader) { + throw new Error('Failed to create shader'); + } + gl.shaderSource(shader, source); + gl.compileShader(shader); + + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + const info = gl.getShaderInfoLog(shader); + gl.deleteShader(shader); + throw new Error(info || 'Failed to compile shader'); + } + + return shader; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/webGlRenderer.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/webGlRenderer.ts new file mode 100644 index 00000000000..b1fbce12cfb --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/webGlRenderer.ts @@ -0,0 +1,484 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * WebGL2 renderer for background effects processing. + * + * This module provides GPU-accelerated rendering for background blur and virtual + * background effects. It implements a multi-pass rendering pipeline: + * - Mask refinement (upsample, joint bilateral filtering) + * - Temporal stabilization + * - Background blur (downsample + Gaussian blur) + * - Final compositing (blur or virtual background) + * + * The renderer works with both HTMLCanvasElement (main thread) and OffscreenCanvas + * (Web Worker), enabling background thread processing for better performance. + */ + +import {BackgroundRenderer} from './backgroundRenderer'; +import {RenderPasses} from './renderPasses'; +import {ShaderPrograms} from './shaderPrograms'; +import {type Size, WebGlResources} from './webGlResources'; + +import type {DebugMode, EffectMode, QualityTierParams} from '../backgroundEffectsWorkerTypes'; +import {computeBlurRadius} from '../quality'; + +/** + * Renderer configuration state. + */ +interface RendererConfig { + /** Output canvas width in pixels. */ + width: number; + /** Output canvas height in pixels. */ + height: number; + /** Quality tier parameters controlling rendering performance. */ + quality: QualityTierParams; + /** Effect mode ('blur', 'virtual', or 'passthrough'). */ + mode: EffectMode; + /** Debug visualization mode. */ + debugMode: DebugMode; + /** Blur strength (0-1) for blur effect mode. */ + blurStrength: number; +} + +interface MaskInputBitmap { + type: 'bitmap'; + bitmap: ImageBitmap; + width: number; + height: number; +} + +interface MaskInputTexture { + type: 'texture'; + texture: WebGLTexture; + width: number; + height: number; +} + +type MaskInput = MaskInputBitmap | MaskInputTexture; + +/** + * WebGL2 renderer for background effects processing. + * + * This class implements a multi-pass GPU rendering pipeline for background blur + * and virtual background effects. It manages WebGL resources (textures, framebuffers, + * shader programs) and performs the following rendering passes: + * + * 1. **Mask refinement**: Upsample low-res mask, apply joint bilateral filtering + * 2. **Temporal stabilization**: Smooth mask across frames using exponential moving average + * 3. **Background blur**: Downsample video, apply separable Gaussian blur + * 4. **Compositing**: Blend foreground and blurred/virtual background using refined mask + * + * The renderer supports both main thread (HTMLCanvasElement) and worker thread + * (OffscreenCanvas) operation for optimal performance. + */ +export class WebGlRenderer { + /** WebGL2 rendering context. */ + private readonly gl: WebGL2RenderingContext; + /** Vertex array object for fullscreen quad rendering. */ + private readonly vao: WebGLVertexArrayObject; + /** Shader program manager. */ + private readonly programs: ShaderPrograms; + /** WebGL resource manager for textures/framebuffers. */ + private readonly resources: WebGlResources; + /** Render pass helpers. */ + private readonly passes: RenderPasses; + /** Current renderer configuration. */ + private config: RendererConfig; + /** Background renderer for virtual background mode. */ + private readonly background: BackgroundRenderer; + /** Flag indicating if maskPrev has been initialized (for temporal stabilization). */ + private maskPrevInitialized = false; + + /** + * Creates a new WebGL renderer. + * + * Initializes WebGL2 context, creates fullscreen quad VAO, compiles all shader + * programs, and sets up initial configuration. The context is created with + * premultipliedAlpha: false and desynchronized: true for optimal performance. + * + * @param canvas - Canvas element (HTMLCanvasElement or OffscreenCanvas). + * @param width - Initial canvas width in pixels. + * @param height - Initial canvas height in pixels. + * @throws Error if WebGL2 is not supported. + */ + constructor(canvas: HTMLCanvasElement | OffscreenCanvas, width: number, height: number) { + const isOffscreen = typeof OffscreenCanvas !== 'undefined' && canvas instanceof OffscreenCanvas; + // Create WebGL2 context with performance optimizations + const gl = canvas.getContext('webgl2', { + premultipliedAlpha: false, + desynchronized: isOffscreen, + }); + if (!gl) { + throw new Error('WebGL2 not supported'); + } + this.gl = gl; + // Create fullscreen quad for rendering + this.vao = this.createQuad(); + // Compile all shader programs + this.programs = new ShaderPrograms(this.gl); + // Initialize resources and passes + this.resources = new WebGlResources(this.gl); + this.passes = new RenderPasses(this.gl, this.programs, this.resources); + this.background = new BackgroundRenderer(this.gl); + // Initialize default configuration + this.config = { + width, + height, + quality: { + tier: 'superhigh', + segmentationWidth: 256, + segmentationHeight: 144, + segmentationCadence: 1, + maskRefineScale: 0.5, + blurDownsampleScale: 0.25, + blurRadius: 4, + bilateralRadius: 5, + bilateralSpatialSigma: 3.5, + bilateralRangeSigma: 0.1, + softLow: 0.3, + softHigh: 0.65, + matteLow: 0.45, + matteHigh: 0.6, + matteHysteresis: 0.04, + temporalAlpha: 0.8, + bypass: false, + }, + mode: 'blur', + debugMode: 'off', + blurStrength: 0.5, + }; + // Configure renderer with initial settings (creates textures/framebuffers) + this.configure( + width, + height, + this.config.quality, + this.config.mode, + this.config.debugMode, + this.config.blurStrength, + ); + } + + /** + * Configures the renderer with new settings. + * + * Updates configuration and ensures all textures and framebuffers are + * created/resized to match the new dimensions and quality settings. + * Should be called whenever dimensions or quality tier changes. + * + * @param width - New canvas width in pixels. + * @param height - New canvas height in pixels. + * @param quality - Quality tier parameters. + * @param mode - Effect mode ('blur', 'virtual', or 'passthrough'). + * @param debugMode - Debug visualization mode. + * @param blurStrength - Blur strength (0-1) for blur effect mode. + */ + public configure( + width: number, + height: number, + quality: QualityTierParams, + mode: EffectMode, + debugMode: DebugMode, + blurStrength: number, + ): void { + this.config = {width, height, quality, mode, debugMode, blurStrength}; + // Ensure all textures and framebuffers exist with correct sizes + const maskPrevWasNew = this.resources.ensureResources({width, height, quality}); + if (maskPrevWasNew) { + this.maskPrevInitialized = false; + } + } + + /** + * Sets the background image/video for virtual background mode. + * + * Uploads the image bitmap to a WebGL texture. If image is null, clears + * the background. The texture is reused if it already exists (updated + * in place for efficiency). + * + * @param image - Background image as ImageBitmap, or null to clear. + * @param width - Image width in pixels. + * @param height - Image height in pixels. + */ + public setBackground(image: ImageBitmap | null, width: number, height: number): void { + this.background.setBackground(image, width, height); + } + + /** + * Renders a frame with background effects applied. + * + * This is the main rendering method that implements the multi-pass pipeline: + * + * **Early exits:** + * - If no mask and no previous mask: passthrough + * - If bypass mode or passthrough mode: passthrough + * + * **Mask refinement pipeline:** + * 1. Downsample video for blur processing + * 2. Upsample low-res mask to refine resolution + * 3. Joint bilateral filter (edge-preserving smoothing using video as guide) + * 4. Temporal stabilization (exponential moving average with previous frame) + * + * **Blur pipeline:** + * 5. Horizontal Gaussian blur pass + * 6. Vertical Gaussian blur pass (separable blur for efficiency) + * + * **Final compositing:** + * - Debug mode: Overlay mask visualization + * - Virtual mode: Composite with background image/video + * - Blur mode: Composite with blurred background + * + * @param frame - Input video frame as ImageBitmap. + * @param maskLow - Low-resolution segmentation mask (bitmap or GPU texture), + * or null if not available (will use previous frame's mask). + */ + public render(frame: ImageBitmap, maskLow: MaskInput | null): void { + const gl = this.gl; + const {quality, width, height, mode, debugMode, blurStrength} = this.config; + const isClassDebug = debugMode === 'classOverlay' || debugMode === 'classOnly'; + + // Ensure all textures and framebuffers are ready + const maskPrevWasNew = this.resources.ensureResources({width, height, quality}); + if (maskPrevWasNew) { + this.maskPrevInitialized = false; + } + + gl.bindVertexArray(this.vao); + // Upload input frame to video texture + this.resources.uploadTexture('videoTex', frame, width, height, undefined, undefined, true); + // Upload low-res mask if available + let maskTexture: WebGLTexture | null = null; + let maskSize: Size | null = null; + if (maskLow) { + if (maskLow.type === 'bitmap') { + this.resources.uploadTexture( + 'maskLowTex', + maskLow.bitmap, + maskLow.width, + maskLow.height, + gl.RGBA8, + gl.RGBA, + true, + ); + maskTexture = this.resources.getTexture('maskLowTex') ?? null; + maskSize = {width: maskLow.width, height: maskLow.height}; + } else { + this.resources.ensureExternalMaskTexture(maskLow.texture); + maskTexture = maskLow.texture; + maskSize = {width: maskLow.width, height: maskLow.height}; + } + } else if (!this.resources.getTexture('maskPrev')) { + // No mask and no previous mask: passthrough + this.passes.drawSimple('compositePassthrough', null, width, height, { + uSrc: this.resources.getTexture('videoTex'), + uTexelSize: [1 / width, 1 / height], + }); + return; + } + + if (isClassDebug) { + if (!maskTexture) { + this.passes.drawSimple('compositePassthrough', null, width, height, { + uSrc: this.resources.getTexture('videoTex'), + uTexelSize: [1 / width, 1 / height], + }); + return; + } + this.passes.drawSimple('debugOverlay', null, width, height, { + uVideo: this.resources.getTexture('videoTex'), + uMask: maskTexture, + uMode: debugMode, + }); + return; + } + + // Bypass mode: passthrough without processing + if (quality.bypass || mode === 'passthrough') { + this.passes.drawSimple('compositePassthrough', null, width, height, { + uSrc: this.resources.getTexture('videoTex'), + uTexelSize: [1 / width, 1 / height], + }); + return; + } + + // Pass 1: Downsample video for blur (reduces blur processing cost) + this.passes.drawToTexture('downsample', 'videoSmallTex', this.resources.getSize('videoSmallTex')!, { + uSrc: this.resources.getTexture('videoTex'), + uTexelSize: [1 / width, 1 / height], + }); + + // Pass 2: Upsample mask to refine resolution + if (maskTexture && maskSize) { + // Use new low-res mask from segmentation + this.passes.drawToTexture('maskUpsample', 'maskRefineA', this.resources.getSize('maskRefineA')!, { + uSrc: maskTexture, + uTexelSize: [1 / maskSize.width, 1 / maskSize.height], + }); + } else { + // Reuse previous frame's mask (cadence > 1) + const maskPrevSize = this.resources.getSize('maskPrev')!; + this.passes.drawToTexture('maskUpsample', 'maskRefineA', this.resources.getSize('maskRefineA')!, { + uSrc: this.resources.getTexture('maskPrev'), + uTexelSize: [1 / maskPrevSize.width, 1 / maskPrevSize.height], + }); + } + + // Pass 3: Joint bilateral filter (edge-preserving smoothing using video as guide) + const maskRefineASize = this.resources.getSize('maskRefineA')!; + this.passes.drawToTexture('jointBilateral', 'maskRefineB', this.resources.getSize('maskRefineB')!, { + uMask: this.resources.getTexture('maskRefineA'), + uVideo: this.resources.getTexture('videoTex'), + uTexelSize: [1 / maskRefineASize.width, 1 / maskRefineASize.height], + uSpatialSigma: quality.bilateralSpatialSigma, + uRangeSigma: quality.bilateralRangeSigma, + uRadius: quality.bilateralRadius, + }); + + // Pass 4: Temporal stabilization (exponential moving average with previous frame) + if (this.maskPrevInitialized) { + // Blend current mask with previous frame's mask for temporal consistency + const maskStableSize = this.resources.getSize('maskStable')!; + this.passes.drawToTexture('temporalMask', 'maskStable', maskStableSize, { + uMask: this.resources.getTexture('maskRefineB'), + uPrevMask: this.resources.getTexture('maskPrev'), + uTexelSize: [1 / maskStableSize.width, 1 / maskStableSize.height], + uAlpha: quality.temporalAlpha, + }); + // Swap stable mask to maskPrev for next frame + this.resources.swapTextures('maskStable', 'maskPrev'); + } else { + // First frame: copy refined mask directly to maskPrev (no previous frame to blend) + this.resources.swapTextures('maskRefineB', 'maskPrev'); + this.maskPrevInitialized = true; + } + + // Pass 5: Horizontal Gaussian blur (separable blur for efficiency) + const dynamicBlurRadius = computeBlurRadius(quality, blurStrength, true); + + const videoSmallSize = this.resources.getSize('videoSmallTex')!; + this.passes.drawToTexture('blurH', 'blurHTex', this.resources.getSize('blurHTex')!, { + uSrc: this.resources.getTexture('videoSmallTex'), + uTexelSize: [1 / videoSmallSize.width, 1 / videoSmallSize.height], + uRadius: dynamicBlurRadius, + }); + + // Pass 6: Vertical Gaussian blur (completes separable blur) + const blurHSize = this.resources.getSize('blurHTex')!; + this.passes.drawToTexture('blurV', 'blurVTex', this.resources.getSize('blurVTex')!, { + uSrc: this.resources.getTexture('blurHTex'), + uTexelSize: [1 / blurHSize.width, 1 / blurHSize.height], + uRadius: dynamicBlurRadius, + }); + + // Debug mode: Overlay mask visualization + if (debugMode !== 'off') { + this.passes.drawSimple('debugOverlay', null, width, height, { + uVideo: this.resources.getTexture('videoTex'), + uMask: this.resources.getTexture('maskPrev'), + uMode: debugMode, + }); + return; + } + + // Virtual background mode: Composite with background image/video + if (mode === 'virtual') { + this.passes.drawSimple('compositeVirtual', null, width, height, { + uVideo: this.resources.getTexture('videoTex'), + uMask: this.resources.getTexture('maskPrev'), + uBg: this.background.getTexture() ?? this.resources.getTexture('videoTex'), + uTexelSize: [1 / width, 1 / height], + uMatteLow: quality.matteLow, + uMatteHigh: quality.matteHigh, + uBgScale: this.background.getCoverScale({width, height}), + }); + return; + } + + // Blur mode: Composite with blurred background + this.passes.drawSimple('compositeBlur', null, width, height, { + uVideo: this.resources.getTexture('videoTex'), + uBlur: this.resources.getTexture('blurVTex'), + uMask: this.resources.getTexture('maskPrev'), + uTexelSize: [1 / width, 1 / height], + uSoftLow: quality.softLow, + uSoftHigh: quality.softHigh, + uBlurStrength: blurStrength, + }); + } + + /** + * Destroys all WebGL resources and cleans up. + * + * Deletes all textures, framebuffers, shader programs, and the vertex array object. + * Should be called when the renderer is no longer needed to free GPU resources. + */ + public destroy(): void { + const gl = this.gl; + this.resources.destroy(); + this.background.destroy(); + this.programs.destroy(); + gl.deleteVertexArray(this.vao); + } + + /** + * Creates a fullscreen quad vertex array object. + * + * Creates a VAO with a quad covering the entire screen (-1 to 1 in NDC space) + * with texture coordinates (0,0) to (1,1). Used for all fullscreen rendering passes. + * + * Vertex format: [x, y, u, v] per vertex (4 vertices, 16 bytes per vertex) + * + * @returns Vertex array object for fullscreen quad rendering. + * @throws Error if VAO or buffer creation fails. + */ + private createQuad(): WebGLVertexArrayObject { + const gl = this.gl; + const vao = gl.createVertexArray(); + if (!vao) { + throw new Error('Failed to create VAO'); + } + gl.bindVertexArray(vao); + + const buffer = gl.createBuffer(); + if (!buffer) { + throw new Error('Failed to create quad buffer'); + } + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + // Quad vertices: bottom-left, bottom-right, top-left, top-right + // Format: [x, y, u, v] per vertex + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array([-1, -1, 0, 0, 1, -1, 1, 0, -1, 1, 0, 1, 1, 1, 1, 1]), + gl.STATIC_DRAW, + ); + + // Position attribute (location 0): 2 floats, stride 16, offset 0 + const positionLocation = 0; + // Texture coordinate attribute (location 1): 2 floats, stride 16, offset 8 + const texLocation = 1; + gl.enableVertexAttribArray(positionLocation); + gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 16, 0); + gl.enableVertexAttribArray(texLocation); + gl.vertexAttribPointer(texLocation, 2, gl.FLOAT, false, 16, 8); + + return vao; + } + + // Resource and program management moved to helper classes. +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/webGlResources.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/webGlResources.ts new file mode 100644 index 00000000000..e5755845f24 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/renderer/webGlResources.ts @@ -0,0 +1,336 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {QualityTierParams} from '../backgroundEffectsWorkerTypes'; + +/** + * Size dimensions for textures and framebuffers. + */ +export interface Size { + /** Width in pixels. */ + width: number; + /** Height in pixels. */ + height: number; +} + +/** + * Configuration for WebGL resource allocation. + */ +export interface RendererConfig { + /** Output canvas width in pixels. */ + width: number; + /** Output canvas height in pixels. */ + height: number; + /** Quality tier parameters controlling resource sizes. */ + quality: QualityTierParams; +} + +/** + * Manages WebGL resources (textures, framebuffers) for the rendering pipeline. + * + * Handles creation, sizing, and lifecycle of all WebGL resources needed for + * background effects rendering. Automatically creates textures and framebuffers + * based on quality tier parameters, and manages resource swapping for ping-pong + * operations. + */ +export class WebGlResources { + private readonly textures: Map = new Map(); + private readonly framebuffers: Map = new Map(); + private sizes: Record = {}; + private maskTextureConfigured = new WeakSet(); + private floatLinearSupported: boolean | null = null; + + constructor(private readonly gl: WebGL2RenderingContext) {} + + /** + * Ensures all required WebGL resources are created and properly sized. + * + * Creates or resizes textures and framebuffers based on the renderer configuration + * and quality tier. Allocates resources for: + * - Video input texture + * - Mask textures (low-res, refined, stable, previous) + * - Blur intermediate textures (downsampled, horizontal, vertical) + * + * Returns true if maskPrev texture was newly created (needs initialization). + * + * @param config - Renderer configuration with dimensions and quality parameters. + * @returns True if maskPrev texture was newly created, false otherwise. + */ + public ensureResources(config: RendererConfig): boolean { + const {width, height, quality} = config; + this.ensureTexture('videoTex', width, height, this.gl.RGBA8, this.gl.RGBA); + + const maskLow = { + width: Math.max(1, quality.segmentationWidth), + height: Math.max(1, quality.segmentationHeight), + }; + this.ensureTexture('maskLowTex', maskLow.width, maskLow.height, this.gl.RGBA8, this.gl.RGBA); + + const refineSize = { + width: Math.max(1, Math.floor(width * quality.maskRefineScale)), + height: Math.max(1, Math.floor(height * quality.maskRefineScale)), + }; + this.ensureTexture('maskRefineA', refineSize.width, refineSize.height, this.gl.RGBA8, this.gl.RGBA); + this.ensureTexture('maskRefineB', refineSize.width, refineSize.height, this.gl.RGBA8, this.gl.RGBA); + this.ensureTexture('maskStable', refineSize.width, refineSize.height, this.gl.RGBA8, this.gl.RGBA); + const maskPrevWasNew = this.ensureTexture( + 'maskPrev', + refineSize.width, + refineSize.height, + this.gl.RGBA8, + this.gl.RGBA, + ); + if (maskPrevWasNew) { + this.initializeMaskPrevWithWhite(refineSize.width, refineSize.height); + } + + const blurSize = { + width: Math.max(1, Math.floor(width * quality.blurDownsampleScale)), + height: Math.max(1, Math.floor(height * quality.blurDownsampleScale)), + }; + this.ensureTexture('videoSmallTex', blurSize.width, blurSize.height, this.gl.RGBA8, this.gl.RGBA); + this.ensureTexture('blurHTex', blurSize.width, blurSize.height, this.gl.RGBA8, this.gl.RGBA); + this.ensureTexture('blurVTex', blurSize.width, blurSize.height, this.gl.RGBA8, this.gl.RGBA); + + return maskPrevWasNew; + } + + /** + * Retrieves a texture by key. + * + * @param key - Texture key identifier. + * @returns WebGL texture, or undefined if not found. + */ + public getTexture(key: string): WebGLTexture | undefined { + return this.textures.get(key); + } + + /** + * Retrieves the size of a texture by key. + * + * @param key - Texture key identifier. + * @returns Size object, or undefined if not found. + */ + public getSize(key: string): Size | undefined { + return this.sizes[key]; + } + + /** + * Retrieves a framebuffer by key. + * + * @param key - Framebuffer key identifier. + * @returns WebGL framebuffer, or undefined if not found. + */ + public getFramebuffer(key: string): WebGLFramebuffer | undefined { + return this.framebuffers.get(key); + } + + /** + * Uploads image data to a texture. + * + * Binds the texture and uploads ImageBitmap data. Supports custom internal + * format and format parameters, and optional Y-flip for coordinate system conversion. + * + * @param key - Texture key identifier. + * @param source - ImageBitmap to upload. + * @param width - Image width in pixels. + * @param height - Image height in pixels. + * @param internalFormat - Optional internal texture format (defaults to RGBA). + * @param format - Optional texture format (defaults to RGBA). + * @param flipY - Whether to flip the image vertically (defaults to false). + * @returns Nothing. + */ + public uploadTexture( + key: string, + source: ImageBitmap, + width: number, + height: number, + internalFormat?: number, + format?: number, + flipY = false, + ): void { + const texture = this.textures.get(key); + if (!texture) { + return; + } + const gl = this.gl; + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flipY ? 1 : 0); + if (internalFormat && format) { + gl.texImage2D(gl.TEXTURE_2D, 0, internalFormat, width, height, 0, format, gl.UNSIGNED_BYTE, source); + } else { + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source); + } + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 0); + } + + /** + * Configures an external mask texture for use in the rendering pipeline. + * + * Sets texture parameters (filtering, wrapping) for a mask texture that + * was created outside this resource manager. Uses linear filtering if + * float linear extension is available, otherwise uses nearest filtering. + * Only configures once per texture (uses WeakSet to track). + * + * @param texture - External WebGL texture to configure. + * @returns Nothing. + */ + public ensureExternalMaskTexture(texture: WebGLTexture): void { + if (this.maskTextureConfigured.has(texture)) { + return; + } + const gl = this.gl; + const floatLinearSupported = + this.floatLinearSupported ?? + (this.floatLinearSupported = + !!gl.getExtension('OES_texture_float_linear') || !!gl.getExtension('OES_texture_half_float_linear')); + const filter = floatLinearSupported ? gl.LINEAR : gl.NEAREST; + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, filter); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, filter); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + this.maskTextureConfigured.add(texture); + } + + /** + * Swaps two textures and their associated framebuffers and sizes. + * + * Exchanges the textures, framebuffers, and size records for two keys. + * Used for ping-pong rendering operations where buffers are alternated + * between read and write. + * + * @param a - First texture key. + * @param b - Second texture key. + * @returns Nothing. + */ + public swapTextures(a: string, b: string): void { + const texA = this.textures.get(a); + const texB = this.textures.get(b); + if (!texA || !texB) { + return; + } + this.textures.set(a, texB); + this.textures.set(b, texA); + + const sizeA = this.sizes[a]; + const sizeB = this.sizes[b]; + this.sizes[a] = sizeB; + this.sizes[b] = sizeA; + + const fboA = this.framebuffers.get(a); + const fboB = this.framebuffers.get(b); + if (fboA && fboB) { + this.framebuffers.set(a, fboB); + this.framebuffers.set(b, fboA); + } + } + + /** + * Destroys all WebGL resources and clears all references. + * + * Deletes all textures and framebuffers, and resets internal state. + * Should be called when the renderer is no longer needed to prevent + * memory leaks. + * + * @returns Nothing. + */ + public destroy(): void { + this.textures.forEach(texture => this.gl.deleteTexture(texture)); + this.framebuffers.forEach(fbo => this.gl.deleteFramebuffer(fbo)); + this.textures.clear(); + this.framebuffers.clear(); + this.sizes = {}; + this.maskTextureConfigured = new WeakSet(); + this.floatLinearSupported = null; + } + + /** + * Ensures a texture exists and is properly sized. + * + * Creates a new texture if it doesn't exist, or resizes it if dimensions + * have changed. Creates an associated framebuffer for render-to-texture. + * Returns true if the texture was newly created. + * + * @param key - Texture key identifier. + * @param width - Required texture width in pixels. + * @param height - Required texture height in pixels. + * @param internalFormat - Internal texture format. + * @param format - Texture format. + * @returns True if texture was newly created, false if it already existed with correct size. + */ + private ensureTexture(key: string, width: number, height: number, internalFormat: number, format: number): boolean { + const gl = this.gl; + const existing = this.textures.get(key); + const size = this.sizes[key]; + + if (existing && size && size.width === width && size.height === height) { + return false; + } + + if (existing) { + gl.deleteTexture(existing); + } + + const texture = gl.createTexture(); + if (!texture) { + throw new Error('Failed to create texture'); + } + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texImage2D(gl.TEXTURE_2D, 0, internalFormat, width, height, 0, format, gl.UNSIGNED_BYTE, null); + + this.textures.set(key, texture); + this.sizes[key] = {width, height}; + + const fbo = this.framebuffers.get(key) ?? gl.createFramebuffer(); + if (fbo) { + gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0); + this.framebuffers.set(key, fbo); + } + + return true; + } + + /** + * Initializes the maskPrev texture with white (fully opaque). + * + * Clears the maskPrev framebuffer to white, providing a safe initial + * state for temporal mask smoothing. Called when maskPrev is first created. + * + * @param width - Texture width in pixels. + * @param height - Texture height in pixels. + * @returns Nothing. + */ + private initializeMaskPrevWithWhite(width: number, height: number): void { + const fbo = this.framebuffers.get('maskPrev'); + if (!fbo) { + return; + } + const gl = this.gl; + gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); + gl.viewport(0, 0, width, height); + gl.clearColor(1.0, 1.0, 1.0, 1.0); + gl.clear(gl.COLOR_BUFFER_BIT); + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/maskPostProcessor.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/maskPostProcessor.ts new file mode 100644 index 00000000000..900b2d5421f --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/maskPostProcessor.ts @@ -0,0 +1,85 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {SegmentationResult} from './segmenter'; + +import type {EffectMode, QualityTierParams} from '../backgroundEffectsWorkerTypes'; + +/** + * Context information for mask post-processing operations. + */ +export interface MaskPostProcessContext { + qualityTier: QualityTierParams; + mode: EffectMode; + timestampMs: number; + /** Frame dimensions in pixels. */ + frameSize: {width: number; height: number}; +} + +/** + * Interface for mask post-processing operations. + * + * Post-processors can modify segmentation results before rendering, + * such as applying temporal smoothing, edge refinement, or other + * enhancements to improve mask quality. + */ +export interface MaskPostProcessor { + process( + result: SegmentationResult, + context: MaskPostProcessContext, + ): Promise | SegmentationResult; + /** + * Resets post-processor state (e.g., clears temporal buffers). + * + * @returns Nothing. + */ + reset(): void; +} + +/** + * Factory interface for creating mask post-processor instances. + */ +export interface MaskPostProcessorFactory { + /** + * Creates a new mask post-processor instance. + * + * @returns A new MaskPostProcessor instance. + */ + create(): MaskPostProcessor; +} + +/** + * No-op mask post-processor that returns results unchanged. + * + * Used as a default when no post-processing is needed. Simply passes + * through segmentation results without modification. + */ +export class NoopMaskPostProcessor implements MaskPostProcessor { + /** + * Returns the segmentation result unchanged. + * + * @param result - Segmentation result to process. + * @returns The same result, unmodified. + */ + public process(result: SegmentationResult): SegmentationResult { + return result; + } + + public reset(): void {} +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/mediaPipeSegmenter.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/mediaPipeSegmenter.ts new file mode 100644 index 00000000000..f99ef4c1f81 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/mediaPipeSegmenter.ts @@ -0,0 +1,104 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {Segmenter} from './segmenter'; +import type {SegmentationResult} from './segmenter'; +import type { + SegmenterDelegate, + SegmenterFactory, + SegmenterInit, + SegmenterLike, + SegmenterOptions, +} from './segmenterTypes'; + +/** + * Adapter that wraps the MediaPipe Segmenter to implement SegmenterLike interface. + * + * Provides a unified interface for segmentation operations, delegating to + * the underlying MediaPipe Segmenter implementation. + */ +class MediaPipeSegmenterAdapter implements SegmenterLike { + private readonly segmenter: Segmenter; + private readonly delegate: SegmenterDelegate; + + constructor(init: SegmenterInit) { + this.segmenter = new Segmenter(init.modelPath, init.delegate, init.canvas); + this.delegate = init.delegate; + } + + /** + * Initializes the MediaPipe segmenter. + * + * @returns Promise that resolves when initialization is complete. + */ + public init(): Promise { + return this.segmenter.init(); + } + + /** + * Configures the segmenter for a specific input size. + * + * @param width - Input width in pixels. + * @param height - Input height in pixels. + * @returns Nothing. + */ + public configure(width: number, height: number): void { + this.segmenter.configure(width, height); + } + + /** + * Performs segmentation on a video frame. + * + * @param frame - Input video frame as ImageBitmap. + * @param timestampMs - Frame timestamp in milliseconds. + * @param options - Optional segmentation options (e.g., includeClassMask). + * @returns Promise resolving to segmentation result with mask and metadata. + */ + public segment(frame: ImageBitmap, timestampMs: number, options?: SegmenterOptions): Promise { + return this.segmenter.segment(frame, timestampMs, options); + } + + /** + * Closes the segmenter and releases resources. + * + * @returns Nothing. + */ + public close(): void { + this.segmenter.close(); + } + + /** + * Returns the delegate type used by this segmenter. + * + * @returns 'CPU' or 'GPU' depending on initialization. + */ + public getDelegate(): SegmenterDelegate { + return this.delegate; + } +} + +/** + * Factory for creating MediaPipe segmenter instances. + * + * Default segmenter factory used by pipelines. Creates MediaPipeSegmenterAdapter + * instances that wrap the underlying MediaPipe Segmenter. + */ +export const MediaPipeSegmenterFactory: SegmenterFactory = { + create: (init: SegmenterInit) => new MediaPipeSegmenterAdapter(init), +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenter.test.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenter.test.ts new file mode 100644 index 00000000000..1b8d2bb0309 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenter.test.ts @@ -0,0 +1,783 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {Segmenter} from './segmenter'; + +// Mock MediaPipe +jest.mock('@mediapipe/tasks-vision', () => { + const mockSegmenter = { + segmentForVideo: jest.fn(), + close: jest.fn(), + }; + + return { + FilesetResolver: { + forVisionTasks: jest.fn().mockResolvedValue({}), + }, + ImageSegmenter: { + createFromOptions: jest.fn().mockResolvedValue(mockSegmenter), + }, + }; +}); + +describe('Segmenter', () => { + const MODEL_PATH = '/test/model.tflite'; + const originalFetch = global.fetch; + let mockSegmenter: any; + let mockResizeCtx: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; + let mockMaskCtx: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; + let mockFrame: ImageBitmap; + let createImageBitmapSpy: jest.SpyInstance; + let performanceNowSpy: jest.SpyInstance; + let fetchSpy: jest.SpyInstance; + let consoleWarnSpy: jest.SpyInstance; + + beforeEach(() => { + // Define createImageBitmap if it doesn't exist + if (typeof global.createImageBitmap === 'undefined') { + (global as any).createImageBitmap = jest.fn(); + } + + // Define OffscreenCanvas if it doesn't exist + if (typeof global.OffscreenCanvas === 'undefined') { + (global as any).OffscreenCanvas = class OffscreenCanvas { + constructor( + public width: number, + public height: number, + ) { + } + + getContext(): any { + return null; + } + }; + } + + // Ensure ImageData is available (jsdom doesn't provide it by default) + if (typeof global.ImageData === 'undefined') { + (global as any).ImageData = class ImageData { + public readonly data: Uint8ClampedArray; + public readonly width: number; + public readonly height: number; + + constructor(width: number, height: number) { + this.width = width; + this.height = height; + this.data = new Uint8ClampedArray(width * height * 4); + } + }; + } + + // Mock MediaPipe ImageSegmenter + const {ImageSegmenter} = require('@mediapipe/tasks-vision'); + mockSegmenter = { + segmentForVideo: jest.fn(), + close: jest.fn(), + }; + ImageSegmenter.createFromOptions.mockResolvedValue(mockSegmenter); + + // Mock canvas contexts + mockResizeCtx = { + drawImage: jest.fn(), + } as any; + mockMaskCtx = { + clearRect: jest.fn(), + putImageData: jest.fn(), + } as any; + + // Mock ImageBitmap - needs to be compatible with jsdom's drawImage + // Create a real canvas and use it as ImageBitmap-like object + const frameCanvas = document.createElement('canvas'); + frameCanvas.width = 640; + frameCanvas.height = 480; + mockFrame = frameCanvas as any; // Use canvas as ImageBitmap for jsdom compatibility + + // Mock createImageBitmap + const mockImageBitmap = { + width: 256, + height: 144, + close: jest.fn(), + } as any; + createImageBitmapSpy = jest.spyOn(global, 'createImageBitmap' as any).mockResolvedValue(mockImageBitmap); + + // Mock performance.now() + let timeCounter = 0; + performanceNowSpy = jest.spyOn(performance, 'now').mockImplementation(() => { + timeCounter += 10; + return timeCounter; + }); + + // Mock fetch - ensure it exists first + if (typeof global.fetch === 'undefined') { + (global as any).fetch = jest.fn(); + } + fetchSpy = jest.spyOn(global, 'fetch' as any).mockResolvedValue({ + ok: true, + status: 200, + } as Response); + + consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + if (originalFetch !== undefined) { + global.fetch = originalFetch; + } else { + delete (global as any).fetch; + } + }); + + describe('constructor', () => { + it('creates segmenter with model path and CPU delegate', () => { + const segmenter = new Segmenter(MODEL_PATH, 'CPU'); + expect(segmenter).toBeDefined(); + }); + + it('creates segmenter with GPU delegate', () => { + const segmenter = new Segmenter(MODEL_PATH, 'GPU'); + expect(segmenter).toBeDefined(); + }); + + it('defaults to CPU delegate when not specified', () => { + const segmenter = new Segmenter(MODEL_PATH); + expect(segmenter).toBeDefined(); + }); + }); + + describe('init', () => { + it('initializes MediaPipe segmenter with correct options', async () => { + const segmenter = new Segmenter(MODEL_PATH, 'CPU'); + const {FilesetResolver, ImageSegmenter} = require('@mediapipe/tasks-vision'); + + await segmenter.init(); + + expect(FilesetResolver.forVisionTasks).toHaveBeenCalledWith('/min/mediapipe/wasm'); + expect(ImageSegmenter.createFromOptions).toHaveBeenCalledWith( + {}, + expect.objectContaining({ + baseOptions: { + modelAssetPath: MODEL_PATH, + delegate: 'CPU', + }, + runningMode: 'VIDEO', + outputCategoryMask: true, + outputConfidenceMasks: true, + }), + ); + }); + + it('initializes with GPU delegate when specified', async () => { + const segmenter = new Segmenter(MODEL_PATH, 'GPU'); + const {ImageSegmenter} = require('@mediapipe/tasks-vision'); + + await segmenter.init(); + + expect(ImageSegmenter.createFromOptions).toHaveBeenCalledWith( + {}, + expect.objectContaining({ + baseOptions: { + modelAssetPath: MODEL_PATH, + delegate: 'GPU', + }, + }), + ); + }); + + it('probes for WASM asset', async () => { + const segmenter = new Segmenter(MODEL_PATH); + await segmenter.init(); + + expect(fetchSpy).toHaveBeenCalledWith('/min/mediapipe/wasm/vision_wasm_internal.wasm', {method: 'HEAD'}); + }); + + it('probes for model asset', async () => { + const segmenter = new Segmenter(MODEL_PATH); + await segmenter.init(); + + expect(fetchSpy).toHaveBeenCalledWith(MODEL_PATH, {method: 'HEAD'}); + }); + + it('segmenter exists when assets are available', async () => { + fetchSpy.mockResolvedValue({ok: true, status: 200} as Response); + const segmenter = new Segmenter(MODEL_PATH); + + await segmenter.init(); + + expect(segmenter.getSegmenter()).not.toBeNull(); + }); + + it('probeAsset has no effect', async () => { + fetchSpy.mockResolvedValue({ok: false, status: 404} as Response); + const segmenter = new Segmenter(MODEL_PATH); + + await segmenter.init(); + + expect(segmenter.getSegmenter()).not.toBeNull(); + }); + + it('handles fetch errors gracefully', async () => { + fetchSpy.mockRejectedValue(new Error('Network error')); + const segmenter = new Segmenter(MODEL_PATH); + + await expect(segmenter.init()).resolves.not.toThrow(); + }); + + it('skips probe when fetch is unavailable', async () => { + delete (global as any).fetch; + const segmenter = new Segmenter(MODEL_PATH); + + await expect(segmenter.init()).resolves.not.toThrow(); + }); + }); + + describe('configure', () => { + it('creates OffscreenCanvas when available', () => { + const segmenter = new Segmenter(MODEL_PATH); + const OffscreenCanvasConstructor = global.OffscreenCanvas; + (global as any).OffscreenCanvas = jest.fn().mockImplementation((width: number, height: number) => { + return { + width, + height, + getContext: jest.fn().mockReturnValue(mockResizeCtx), + } as any; + }) as any; + + segmenter.configure(256, 144); + + expect(global.OffscreenCanvas).toHaveBeenCalledTimes(2); // resize and mask canvases + expect(global.OffscreenCanvas).toHaveBeenCalledWith(256, 144); + + // Restore + global.OffscreenCanvas = OffscreenCanvasConstructor; + }); + + it('creates HTMLCanvasElement when OffscreenCanvas unavailable', () => { + const segmenter = new Segmenter(MODEL_PATH); + const originalOffscreenCanvas = global.OffscreenCanvas; + delete (global as any).OffscreenCanvas; + + const createElementSpy = jest.spyOn(document, 'createElement').mockImplementation((tag: string) => { + if (tag === 'canvas') { + return { + width: 0, + height: 0, + getContext: jest.fn().mockReturnValue(mockResizeCtx), + } as any; + } + return document.createElement(tag); + }); + + segmenter.configure(256, 144); + + expect(createElementSpy).toHaveBeenCalledWith('canvas'); + expect(createElementSpy).toHaveBeenCalledTimes(2); // resize and mask canvases + + // Restore + global.OffscreenCanvas = originalOffscreenCanvas; + }); + + it('gets 2D context for resize canvas', () => { + const segmenter = new Segmenter(MODEL_PATH); + const mockCanvas = { + getContext: jest.fn().mockReturnValue(mockResizeCtx), + }; + const OffscreenCanvasConstructor = global.OffscreenCanvas; + (global as any).OffscreenCanvas = jest.fn().mockImplementation(() => mockCanvas as any); + + segmenter.configure(256, 144); + + expect(mockCanvas.getContext).toHaveBeenCalledWith('2d'); + + // Restore + global.OffscreenCanvas = OffscreenCanvasConstructor; + }); + + it('gets 2D context for mask canvas', () => { + const segmenter = new Segmenter(MODEL_PATH); + const mockCanvas = { + getContext: jest.fn().mockReturnValue(mockMaskCtx), + }; + const OffscreenCanvasConstructor = global.OffscreenCanvas; + (global as any).OffscreenCanvas = jest.fn().mockImplementation(() => mockCanvas as any); + + segmenter.configure(256, 144); + + expect(mockCanvas.getContext).toHaveBeenCalledWith('2d'); + + // Restore + global.OffscreenCanvas = OffscreenCanvasConstructor; + }); + + it('is idempotent - returns early if dimensions unchanged', () => { + const segmenter = new Segmenter(MODEL_PATH); + const OffscreenCanvasConstructor = global.OffscreenCanvas; + const offscreenCanvasMock = jest.fn().mockImplementation((width: number, height: number) => { + return { + width, + height, + getContext: jest.fn().mockReturnValue(mockResizeCtx), + } as any; + }); + (global as any).OffscreenCanvas = offscreenCanvasMock; + + segmenter.configure(256, 144); + const firstCallCount = offscreenCanvasMock.mock.calls.length; + + segmenter.configure(256, 144); + const secondCallCount = offscreenCanvasMock.mock.calls.length; + + expect(secondCallCount).toBe(firstCallCount); // No additional calls + + // Restore + global.OffscreenCanvas = OffscreenCanvasConstructor; + }); + + it('updates canvases when dimensions change', () => { + const segmenter = new Segmenter(MODEL_PATH); + const OffscreenCanvasConstructor = global.OffscreenCanvas; + const offscreenCanvasMock = jest.fn().mockImplementation((width: number, height: number) => { + return { + width, + height, + getContext: jest.fn().mockReturnValue(mockResizeCtx), + } as any; + }); + (global as any).OffscreenCanvas = offscreenCanvasMock; + + segmenter.configure(256, 144); + segmenter.configure(160, 96); + + expect(offscreenCanvasMock).toHaveBeenCalledWith(160, 96); + + // Restore + global.OffscreenCanvas = OffscreenCanvasConstructor; + }); + }); + + describe('segment', () => { + // Helper to setup segmenter with proper mocks + async function setupSegmenter(): Promise { + const segmenter = new Segmenter(MODEL_PATH); + await segmenter.init(); + + // Mock OffscreenCanvas to return canvases with our mocked contexts + const OffscreenCanvasConstructor = global.OffscreenCanvas; + let callCount = 0; + (global as any).OffscreenCanvas = jest.fn().mockImplementation((width: number, height: number) => { + callCount++; + return { + width, + height, + getContext: jest.fn().mockReturnValue(callCount === 1 ? mockResizeCtx : mockMaskCtx), + }; + }) as any; + + segmenter.configure(256, 144); + + // Store constructor for restoration + (segmenter as any)._offscreenCanvasConstructor = OffscreenCanvasConstructor; + + return segmenter; + } + + function restoreOffscreenCanvas(segmenter: Segmenter): void { + const constructor = (segmenter as any)._offscreenCanvasConstructor; + if (constructor) { + global.OffscreenCanvas = constructor; + } + } + + beforeEach(async () => { + // Setup segmenter with mocked MediaPipe + const {ImageSegmenter} = require('@mediapipe/tasks-vision'); + ImageSegmenter.createFromOptions.mockResolvedValue(mockSegmenter); + + // Ensure OffscreenCanvas is available + if (typeof global.OffscreenCanvas === 'undefined') { + (global as any).OffscreenCanvas = class OffscreenCanvas { + constructor( + public width: number, + public height: number, + ) {} + getContext(): any { + return null; + } + }; + } + }); + + it('returns null mask when segmenter not initialized', async () => { + const segmenter = new Segmenter(MODEL_PATH); + + const result = await segmenter.segment(mockFrame, 1000); + + expect(result.mask).toBeNull(); + expect(result.width).toBe(0); + expect(result.height).toBe(0); + expect(result.durationMs).toBe(0); + }); + + it('returns null mask when canvases not configured', async () => { + const segmenter = new Segmenter(MODEL_PATH); + await segmenter.init(); + + const result = await segmenter.segment(mockFrame, 1000); + + expect(result.mask).toBeNull(); + }); + + it('resizes input frame to segmentation resolution', async () => { + const segmenter = await setupSegmenter(); + + const mockImageData = new ImageData(256, 144); + const mockMask = { + getAsImageData: jest.fn().mockReturnValue(mockImageData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + await segmenter.segment(mockFrame, 1000); + + expect(mockResizeCtx.drawImage).toHaveBeenCalledWith(mockFrame, 0, 0, 256, 144); + + restoreOffscreenCanvas(segmenter); + }); + + it('calls segmentForVideo with canvas and timestamp', async () => { + const segmenter = await setupSegmenter(); + + const mockImageData = new ImageData(256, 144); + const mockMask = { + getAsImageData: jest.fn().mockReturnValue(mockImageData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + await segmenter.segment(mockFrame, 1234); + + expect(mockSegmenter.segmentForVideo).toHaveBeenCalledWith(expect.anything(), 1234); + + restoreOffscreenCanvas(segmenter); + }); + + it('handles ImageData mask format', async () => { + const segmenter = await setupSegmenter(); + + const mockImageData = new ImageData(256, 144); + const mockMask = { + getAsImageData: jest.fn().mockReturnValue(mockImageData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + await segmenter.segment(mockFrame, 1000); + + expect(mockMask.getAsImageData).toHaveBeenCalled(); + expect(mockMaskCtx.putImageData).toHaveBeenCalledWith(mockImageData, 0, 0); + + restoreOffscreenCanvas(segmenter); + }); + + it('handles Float32Array mask format', async () => { + const segmenter = await setupSegmenter(); + + const floatData = new Float32Array(256 * 144); + for (let i = 0; i < floatData.length; i++) { + floatData[i] = 0.5; // 50% confidence + } + const mockMask = { + getAsFloat32Array: jest.fn().mockReturnValue(floatData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + await segmenter.segment(mockFrame, 1000); + + expect(mockMask.getAsFloat32Array).toHaveBeenCalled(); + expect(mockMaskCtx.putImageData).toHaveBeenCalled(); + // Verify ImageData was created with correct values (0.5 * 255 = 127.5 ≈ 128) + const putImageDataCall = (mockMaskCtx.putImageData as jest.Mock).mock.calls[0][0] as ImageData; + expect(putImageDataCall.data[0]).toBe(128); // R channel + expect(putImageDataCall.data[1]).toBe(128); // G channel + expect(putImageDataCall.data[2]).toBe(128); // B channel + expect(putImageDataCall.data[3]).toBe(128); // A channel (Soft-Edge Rendering if Alpha mask not 100% viewable) + + restoreOffscreenCanvas(segmenter); + }); + + it('handles Uint8Array mask format', async () => { + const segmenter = await setupSegmenter(); + + const uint8Data = new Uint8Array(256 * 144); + for (let i = 0; i < uint8Data.length; i++) { + uint8Data[i] = 128; // 50% confidence (128/255) + } + const mockMask = { + getAsUint8Array: jest.fn().mockReturnValue(uint8Data), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + await segmenter.segment(mockFrame, 1000); + + expect(mockMask.getAsUint8Array).toHaveBeenCalled(); + expect(mockMaskCtx.putImageData).toHaveBeenCalled(); + + restoreOffscreenCanvas(segmenter); + }); + + it('handles null mask gracefully', async () => { + const segmenter = await setupSegmenter(); + + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [null], + }); + + const result = await segmenter.segment(mockFrame, 1000); + + expect(mockMaskCtx.clearRect).not.toHaveBeenCalled(); + expect(mockMaskCtx.putImageData).not.toHaveBeenCalled(); + expect(result.mask).toBeNull(); + + restoreOffscreenCanvas(segmenter); + }); + + it('handles empty confidence masks array', async () => { + const segmenter = await setupSegmenter(); + + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [], + }); + + const result = await segmenter.segment(mockFrame, 1000); + + expect(mockMaskCtx.clearRect).not.toHaveBeenCalled(); + expect(result.mask).toBeNull(); + + restoreOffscreenCanvas(segmenter); + }); + + it('creates ImageBitmap from mask canvas', async () => { + const segmenter = await setupSegmenter(); + + const mockImageData = new ImageData(256, 144); + const mockMask = { + getAsImageData: jest.fn().mockReturnValue(mockImageData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + await segmenter.segment(mockFrame, 1000); + + expect(createImageBitmapSpy).toHaveBeenCalled(); + + restoreOffscreenCanvas(segmenter); + }); + + it('calculates duration correctly', async () => { + const segmenter = await setupSegmenter(); + + performanceNowSpy.mockReturnValueOnce(100); + performanceNowSpy.mockReturnValueOnce(150); + + const mockImageData = new ImageData(256, 144); + const mockMask = { + getAsImageData: jest.fn().mockReturnValue(mockImageData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + const result = await segmenter.segment(mockFrame, 1000); + + expect(result.durationMs).toBe(50); + + restoreOffscreenCanvas(segmenter); + }); + + it('returns correct width and height', async () => { + const segmenter = await setupSegmenter(); + + const mockImageData = new ImageData(256, 144); + const mockMask = { + getAsImageData: jest.fn().mockReturnValue(mockImageData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + const result = await segmenter.segment(mockFrame, 1000); + + expect(result.width).toBe(256); + expect(result.height).toBe(144); + + restoreOffscreenCanvas(segmenter); + }); + + it('handles segmentation errors gracefully', async () => { + const segmenter = await setupSegmenter(); + + const error = new Error('Segmentation failed'); + mockSegmenter.segmentForVideo.mockImplementation(() => { + throw error; + }); + + const result = await segmenter.segment(mockFrame, 1000); + + expect(result.mask).toBeNull(); + expect(result.width).toBe(0); + expect(result.height).toBe(0); + expect(consoleWarnSpy).toHaveBeenCalledWith('[Segmenter] segment failed', error); + + restoreOffscreenCanvas(segmenter); + }); + + it('closes mask resources after released result segmentation', async () => { + const segmenter = await setupSegmenter(); + + const mockMask = { + getAsImageData: jest.fn().mockReturnValue(new ImageData(256, 144)), + close: jest.fn(), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + const result = await segmenter.segment(mockFrame, 1000); + + result.release(); + + expect(mockMask.close).toHaveBeenCalled(); + + restoreOffscreenCanvas(segmenter); + }); + + it('handles mask close errors gracefully', async () => { + const segmenter = await setupSegmenter(); + + const mockMask = { + getAsImageData: jest.fn().mockReturnValue(new ImageData(256, 144)), + close: jest.fn().mockImplementation(() => { + throw new Error('Already closed'); + }), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + // Should not throw + await expect(segmenter.segment(mockFrame, 1000)).resolves.toBeDefined(); + }); + + it('handles null mask in cleanup', async () => { + const segmenter = await setupSegmenter(); + + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [null], + }); + + // Should not throw + await expect(segmenter.segment(mockFrame, 1000)).resolves.toBeDefined(); + + restoreOffscreenCanvas(segmenter); + }); + + it('clamps Float32Array values to 0-1 range', async () => { + const segmenter = await setupSegmenter(); + + const floatData = new Float32Array(256 * 144); + floatData[0] = 1.5; // Above 1.0 + floatData[1] = -0.5; // Below 0.0 + floatData[2] = 0.5; // Valid + + const mockMask = { + getAsFloat32Array: jest.fn().mockReturnValue(floatData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + await segmenter.segment(mockFrame, 1000); + + expect(mockMaskCtx.putImageData).toHaveBeenCalled(); + const putImageDataCall = (mockMaskCtx.putImageData as jest.Mock).mock.calls[0][0] as ImageData; + expect(putImageDataCall.data[0]).toBe(255); // Clamped to 1.0 -> 255 + expect(putImageDataCall.data[4]).toBe(0); // Clamped to 0.0 -> 0 + expect(putImageDataCall.data[8]).toBe(128); // 0.5 -> 128 + + restoreOffscreenCanvas(segmenter); + }); + + it('handles data length mismatch (data shorter than width*height)', async () => { + const segmenter = await setupSegmenter(); + + const floatData = new Float32Array(100); // Shorter than 256*144 + const mockMask = { + getAsFloat32Array: jest.fn().mockReturnValue(floatData), + }; + mockSegmenter.segmentForVideo.mockReturnValue({ + confidenceMasks: [mockMask], + }); + + await segmenter.segment(mockFrame, 1000); + + expect(mockMaskCtx.putImageData).toHaveBeenCalled(); + const putImageDataCall = (mockMaskCtx.putImageData as jest.Mock).mock.calls[0][0] as ImageData; + expect(putImageDataCall.data.length).toBe(256 * 144 * 4); // Full RGBA size + + restoreOffscreenCanvas(segmenter); + }); + }); + + describe('close', () => { + it('closes MediaPipe segmenter', async () => { + const segmenter = new Segmenter(MODEL_PATH); + await segmenter.init(); + + segmenter.close(); + + expect(mockSegmenter.close).toHaveBeenCalled(); + }); + + it('sets segmenter to null', async () => { + const segmenter = new Segmenter(MODEL_PATH); + await segmenter.init(); + + segmenter.close(); + + // After close, segment should return null mask + const result = await segmenter.segment(mockFrame, 1000); + expect(result.mask).toBeNull(); + }); + + it('handles close when segmenter is null', () => { + const segmenter = new Segmenter(MODEL_PATH); + + // Should not throw + expect(() => segmenter.close()).not.toThrow(); + }); + }); +}); diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenter.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenter.ts new file mode 100644 index 00000000000..22e970e5fad --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenter.ts @@ -0,0 +1,607 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * MediaPipe-based person segmentation for background effects. + * + * This module provides a wrapper around MediaPipe's ImageSegmenter that: + * - Performs real-time person/background separation using ML models + * - Supports both CPU and GPU-accelerated inference + * - Handles frame resizing and mask format conversion + * - Works in both main thread and Web Worker contexts + */ + +import {FilesetResolver, ImageSegmenter} from '@mediapipe/tasks-vision'; + +import type {SegmenterOptions} from './segmenterTypes'; + +/** + * Result of a segmentation operation. + */ +export interface SegmentationResult { + /** Segmentation mask as ImageBitmap, or null if segmentation failed. */ + mask: ImageBitmap | null; + /** Multiclass mask with class indices encoded in RGB, or null if unavailable. */ + classMask: ImageBitmap | null; + /** Segmentation mask as WebGLTexture (zero-copy GPU path), or null if unavailable. */ + maskTexture: WebGLTexture | null; + /** Width of the mask in pixels. */ + width: number; + /** Height of the mask in pixels. */ + height: number; + /** Time taken for segmentation in milliseconds. */ + durationMs: number; + /** Releases MediaPipe mask resources after rendering. */ + release: () => void; +} + +/** + * Wrapper around MediaPipe ImageSegmenter for person/background separation. + * + * This class provides a simplified interface to MediaPipe's segmentation model, + * handling frame preprocessing, mask format conversion, and resource management. + * It supports both CPU and GPU inference delegates, and works in both main thread + * and Web Worker environments (using OffscreenCanvas when available). + * + * The segmenter uses MediaPipe segmentation models to generate confidence masks + * that separate foreground (person) from background, with optional multiclass output. + */ +export class Segmenter { + /** MediaPipe ImageSegmenter instance. */ + private segmenter: ImageSegmenter | null = null; + /** Canvas for resizing input frames to segmentation resolution. */ + private resizeCanvas: OffscreenCanvas | HTMLCanvasElement | null = null; + /** 2D context for resize canvas. */ + private resizeCtx: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D | null = null; + /** Canvas for rendering the final mask. */ + private maskCanvas: OffscreenCanvas | HTMLCanvasElement | null = null; + /** 2D context for mask canvas. */ + private maskCtx: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D | null = null; + /** Current segmentation resolution width in pixels. */ + private width = 0; + /** Current segmentation resolution height in pixels. */ + private height = 0; + /** Base path for MediaPipe WASM files. */ + private readonly wasmBasePath = '/min/mediapipe/wasm'; + /** + * Creates a new segmenter instance. + * + * @param modelPath - Path to the MediaPipe segmentation model file. + * @param delegate - Inference delegate ('CPU' or 'GPU'). GPU is faster but requires + * WebGL support. Defaults to 'CPU' for broader compatibility. + * @param gpuCanvas - Canvas bound to the WebGL context used by the GPU delegate. + */ + constructor( + private readonly modelPath: string, + private readonly delegate: 'CPU' | 'GPU' = 'CPU', + private readonly gpuCanvas?: HTMLCanvasElement | OffscreenCanvas, + ) {} + + /** + * Returns the delegate type used for segmentation ('CPU' or 'GPU'). + */ + public getDelegate(): 'CPU' | 'GPU' { + return this.delegate; + } + + /** + * Initializes the MediaPipe segmenter with the specified model. + * + * This method: + * 1. Probes for required assets (WASM files and model) + * 2. Resolves MediaPipe fileset + * 3. Creates the ImageSegmenter instance with video mode and confidence masks + * + * Must be called before using segment(). The segmenter is configured for + * video mode (temporal consistency) and outputs confidence masks (0-1 values). + * + * @throws May throw if model files are missing or initialization fails. + */ + public async init(): Promise { + // Probe for required assets (non-blocking) + await this.probeAsset(`${this.wasmBasePath}/vision_wasm_internal.wasm`); + await this.probeAsset(this.modelPath); + // Resolve MediaPipe fileset + const fileset = await FilesetResolver.forVisionTasks(this.wasmBasePath); + // Create segmenter with video mode and confidence masks + this.segmenter = await ImageSegmenter.createFromOptions(fileset, { + baseOptions: { + modelAssetPath: this.modelPath, + delegate: this.delegate, + }, + ...(this.gpuCanvas ? {canvas: this.gpuCanvas} : {}), + runningMode: 'VIDEO', // Video mode for temporal consistency + outputCategoryMask: true, // Needed for multiclass debug visualization + outputConfidenceMasks: true, // Confidence values (0-1) for smooth edges + }); + } + + /** + * Configures the segmenter for a specific resolution. + * + * Creates or updates the resize and mask canvases to match the specified + * dimensions. Uses OffscreenCanvas when available (Web Worker context), + * otherwise falls back to HTMLCanvasElement (main thread). + * + * This method is idempotent - if dimensions haven't changed, it returns early. + * Should be called whenever the segmentation resolution changes (e.g., when + * quality tier changes). + * + * @param width - Target segmentation width in pixels. + * @param height - Target segmentation height in pixels. + */ + public configure(width: number, height: number): void { + // Early return if dimensions unchanged + if (this.width === width && this.height === height) { + return; + } + this.width = width; + this.height = height; + + // Use OffscreenCanvas in Web Worker, HTMLCanvasElement on main thread + if (typeof OffscreenCanvas !== 'undefined') { + this.resizeCanvas = new OffscreenCanvas(width, height); + this.maskCanvas = new OffscreenCanvas(width, height); + } else { + const resize = document.createElement('canvas'); + const mask = document.createElement('canvas'); + resize.width = width; + resize.height = height; + mask.width = width; + mask.height = height; + this.resizeCanvas = resize; + this.maskCanvas = mask; + } + // Get 2D rendering contexts for both canvases + this.resizeCtx = this.resizeCanvas.getContext('2d') as + | OffscreenCanvasRenderingContext2D + | CanvasRenderingContext2D + | null; + this.maskCtx = this.maskCanvas.getContext('2d') as + | OffscreenCanvasRenderingContext2D + | CanvasRenderingContext2D + | null; + } + + /** + * Segments a video frame to extract person/background mask. + * + * Processing pipeline: + * 1. Resize input frame to segmentation resolution + * 2. Run MediaPipe segmentation (with temporal consistency in video mode) + * 3. Convert MediaPipe mask format to ImageData + * 4. Render mask to canvas and create ImageBitmap + * 5. Measure processing time + * + * The timestamp is used by MediaPipe for temporal smoothing between frames. + * All MediaPipe mask resources are automatically closed to prevent leaks. + * + * @param frame - Input video frame as ImageBitmap. + * @param timestampMs - Frame timestamp in milliseconds (monotonic, for temporal consistency). + * @returns Segmentation result with mask, dimensions, and processing time. + */ + public async segment( + frame: ImageBitmap, + timestampMs: number, + options: SegmenterOptions = {}, + ): Promise { + // Validate segmenter and canvases are initialized + if (!this.segmenter || !this.resizeCtx || !this.maskCtx || !this.resizeCanvas || !this.maskCanvas) { + return {mask: null, classMask: null, maskTexture: null, width: 0, height: 0, durationMs: 0, release: () => {}}; + } + + const start = performance.now(); + let masks: any[] = []; + let categoryMask: any | null = null; + + try { + // Step 1: Resize input frame to segmentation resolution + this.resizeCtx.drawImage(frame, 0, 0, this.width, this.height); + + // Step 2: Run MediaPipe segmentation (video mode uses timestamp for temporal smoothing) + const result = this.segmenter.segmentForVideo(this.resizeCanvas, timestampMs); + + masks = result.confidenceMasks ?? []; + categoryMask = result.categoryMask ?? null; + // Step 3: Convert MediaPipe mask format to ImageData + const primaryMask = masks[0] ?? null; + + const release = () => { + masks.forEach(mask => { + try { + mask?.close(); + } catch { + // Ignore errors when closing (mask may already be closed) + } + }); + try { + categoryMask?.close?.(); + } catch { + // Ignore errors when closing (mask may already be closed) + } + }; + + // if (!primaryMask) { + // const durationMs = performance.now() - start; + // return {mask: null, classMask: null, maskTexture: null, width: 0, height: 0, durationMs, release}; + // } + + const classMask = options.includeClassMask ? await this.buildClassMask(categoryMask) : null; + + if ( + masks.length === 1 && + primaryMask && + this.gpuCanvas && + typeof primaryMask.hasWebGLTexture === 'function' && + primaryMask.hasWebGLTexture() && + primaryMask.canvas === this.gpuCanvas && + typeof primaryMask.getAsWebGLTexture === 'function' + ) { + const maskTexture = primaryMask.getAsWebGLTexture(); + const durationMs = performance.now() - start; + return { + mask: null, + classMask, + maskTexture, + width: primaryMask.width ?? this.width, + height: primaryMask.height ?? this.height, + durationMs, + release, + }; + } + + let maskImage: ImageData | null = null; + + if (categoryMask) { + // multiclass model + maskImage = this.categoryMaskToBinaryMask(categoryMask); + } else { + // singleclass model + maskImage = this.combineConfidenceMasks(masks); + } + + if (!maskImage) { + const durationMs = performance.now() - start; + return {mask: null, classMask, maskTexture: null, width: 0, height: 0, durationMs, release}; + } + + // Step 4: Render mask to canvas + this.maskCtx.clearRect(0, 0, this.width, this.height); + this.maskCtx.putImageData(maskImage, 0, 0); + + // Step 5: Create ImageBitmap from mask canvas + const mask = await createImageBitmap(this.maskCanvas); + const durationMs = performance.now() - start; + return { + mask, + classMask, + maskTexture: null, + width: this.width, + height: this.height, + durationMs, + release, + }; + } catch (error) { + console.warn('[Segmenter] segment failed', error); + return { + mask: null, + classMask: null, + maskTexture: null, + width: 0, + height: 0, + durationMs: performance.now() - start, + release: () => { + masks.forEach(mask => { + try { + mask?.close(); + } catch { + // Ignore errors when closing (mask may already be closed) + } + }); + try { + categoryMask?.close?.(); + } catch { + // Ignore errors when closing (mask may already be closed) + } + }, + }; + } + } + + /** + * Closes the segmenter and releases all resources. + * + * Should be called when the segmenter is no longer needed to free GPU/CPU + * resources and prevent memory leaks. After calling close(), the segmenter + * cannot be used again (init() must be called on a new instance). + */ + public close(): void { + this.segmenter?.close(); + this.segmenter = null; + } + + /** + * Converts MediaPipe mask format to ImageData. + * + * MediaPipe masks can be provided in different formats: + * - ImageData (direct conversion) + * - Float32Array (0-1 range, normalized) + * - Uint8Array (0-255 range, already scaled) + * + * This method handles all formats and normalizes them to ImageData with + * RGBA channels (grayscale mask in RGB, alpha = 255). + * + * @param mask - MediaPipe mask object (format varies by MediaPipe version). + * @returns ImageData representation of the mask, or null if mask is invalid. + */ + private maskToImageData(mask: any): ImageData | null { + if (!mask) { + return null; + } + // Direct ImageData format (preferred, no conversion needed) + if (typeof mask.getAsImageData === 'function') { + return this.normalizeMaskImageData(mask.getAsImageData()); + } + // Float32Array format (0-1 range, needs normalization) + if (typeof mask.getAsFloat32Array === 'function') { + const data = mask.getAsFloat32Array() as Float32Array; + return this.buildMaskImageData(data, 1); + } + // Uint8Array format (0-255 range, already scaled) + if (typeof mask.getAsUint8Array === 'function') { + const data = mask.getAsUint8Array() as Uint8Array; + return this.buildMaskImageData(data, 255); + } + return null; + } + + private combineConfidenceMasks(masks: any[]): ImageData | null { + if (!masks.length) { + return null; + } + if (masks.length === 1) { + return this.maskToImageData(masks[0]); + } + const arrays = masks.map(mask => this.maskToFloatArray(mask)); + if (arrays.some(array => !array)) { + return this.maskToImageData(masks[0]); + } + const width = masks[0]?.width ?? this.width; + const height = masks[0]?.height ?? this.height; + const imageData = new ImageData(width, height); + const out = imageData.data; + const count = Math.min(arrays[0]!.length, width * height); + for (let i = 0; i < count; i += 1) { + let maxValue = 0; + for (let maskIndex = 1; maskIndex < arrays.length; maskIndex += 1) { + const value = arrays[maskIndex]![i]; + if (value > maxValue) { + maxValue = value; + } + } + const clamped = Math.max(0, Math.min(1, maxValue)); + const value = Math.round(clamped * 255); + const idx = i * 4; + out[idx] = value; + out[idx + 1] = value; + out[idx + 2] = value; + out[idx + 3] = 255; + } + return imageData; + } + + private categoryMaskToBinaryMask(mask: any): ImageData | null { + if (!mask) { + return null; + } + + const width = mask.width ?? this.width; + const height = mask.height ?? this.height; + + const imageData = new ImageData(width, height); + const out = imageData.data; + + if (typeof mask.getAsUint8Array === 'function') { + const data = mask.getAsUint8Array() as Uint8Array; + + for (let i = 0; i < data.length; i++) { + // 0 = background + // alles andere = Person + const isPerson = data[i] !== 0; + const value = isPerson ? 255 : 0; + + const idx = i * 4; + out[idx] = value; + out[idx + 1] = value; + out[idx + 2] = value; + out[idx + 3] = 255; + } + + return imageData; + } + + return null; + } + + private maskToFloatArray(mask: any): Float32Array | null { + if (!mask) { + return null; + } + if (typeof mask.getAsFloat32Array === 'function') { + return mask.getAsFloat32Array() as Float32Array; + } + if (typeof mask.getAsUint8Array === 'function') { + const data = mask.getAsUint8Array() as Uint8Array; + const out = new Float32Array(data.length); + for (let i = 0; i < data.length; i += 1) { + out[i] = data[i] / 255; + } + return out; + } + if (typeof mask.getAsImageData === 'function') { + const imageData = mask.getAsImageData() as ImageData; + const src = imageData.data; + const out = new Float32Array(imageData.width * imageData.height); + for (let i = 0, j = 0; i < src.length; i += 4, j += 1) { + out[j] = src[i] / 255; + } + return out; + } + return null; + } + + private async buildClassMask(categoryMask: any | null): Promise { + if (!categoryMask) { + return null; + } + const imageData = this.categoryMaskToImageData(categoryMask); + if (!imageData) { + return null; + } + try { + return await createImageBitmap(imageData); + } catch { + return null; + } + } + + private categoryMaskToImageData(mask: any): ImageData | null { + if (!mask) { + return null; + } + const width = mask.width ?? this.width; + const height = mask.height ?? this.height; + const imageData = new ImageData(width, height); + const out = imageData.data; + const write = (value: number, idx: number) => { + out[idx] = value; + out[idx + 1] = value; + out[idx + 2] = value; + out[idx + 3] = 255; + }; + if (typeof mask.getAsUint8Array === 'function') { + const data = mask.getAsUint8Array() as Uint8Array; + const count = Math.min(data.length, width * height); + for (let i = 0; i < count; i += 1) { + write(data[i], i * 4); + } + return imageData; + } + if (typeof mask.getAsFloat32Array === 'function') { + const data = mask.getAsFloat32Array() as Float32Array; + const count = Math.min(data.length, width * height); + for (let i = 0; i < count; i += 1) { + const value = Math.round(Math.max(0, Math.min(255, data[i]))); + write(value, i * 4); + } + return imageData; + } + if (typeof mask.getAsImageData === 'function') { + const data = mask.getAsImageData() as ImageData; + const src = data.data; + for (let i = 0; i < src.length; i += 4) { + write(src[i], i); + } + return imageData; + } + return null; + } + + /** + * Builds ImageData from raw mask array data. + * + * Converts a 1D array of mask values to RGBA ImageData format: + * - Normalizes values to 0-1 range based on scale parameter + * - Clamps values to valid range + * - Converts to 0-255 grayscale values + * - Creates RGBA format (grayscale in RGB channels, alpha = 255) + * + * @param data - Raw mask data array (Float32Array or Uint8Array). + * @param scale - Scale factor to normalize data (1 for Float32Array, 255 for Uint8Array). + * @returns ImageData with grayscale mask in RGBA format. + */ + private buildMaskImageData(data: ArrayLike, scale: number): ImageData { + const width = this.width; + const height = this.height; + const imageData = new ImageData(width, height); + const out = imageData.data; + // Handle cases where data length doesn't match dimensions + const count = Math.min(data.length, width * height); + for (let i = 0; i < count; i += 1) { + // Normalize to 0-1 range + const raw = data[i] / scale; + // Clamp to valid range + const clamped = Math.max(0, Math.min(1, raw)); + // Convert to 0-255 grayscale + const value = Math.round(clamped * 255); + // Write to RGBA channels (grayscale mask) + const idx = i * 4; + out[idx] = value; // R + out[idx + 1] = value; // G + out[idx + 2] = value; // B + out[idx + 3] = value; // A (mask alpha) + } + return imageData; + } + + private normalizeMaskImageData(imageData: ImageData): ImageData { + const out = new ImageData(imageData.width, imageData.height); + const src = imageData.data; + const dest = out.data; + for (let i = 0; i < src.length; i += 4) { + const value = src[i]; + dest[i] = value; + dest[i + 1] = value; + dest[i + 2] = value; + dest[i + 3] = value; + } + return out; + } + + /** + * Probes for asset availability (non-blocking check). + * + * Performs a HEAD request to verify that required assets (WASM files, + * model files) are available. This is a diagnostic check that logs + * warnings if assets are missing, but doesn't fail initialization. + * + * Useful for debugging missing files in development environments. + * + * @param url - URL of the asset to check. + */ + private async probeAsset(url: string): Promise { + // Skip probe if fetch is unavailable (e.g., Node.js environment) + if (typeof fetch !== 'function') { + return; + } + try { + // Use HEAD request to check availability without downloading + const response = await fetch(url, {method: 'HEAD'}); + if (!response.ok) { + // Intentionally ignore missing assets; init will surface issues if needed. + } + } catch (error) { + // Swallow probe errors to avoid blocking initialization. + } + } + + getSegmenter(): ImageSegmenter | null { + return this.segmenter; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenterTypes.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenterTypes.ts new file mode 100644 index 00000000000..544df2c4a5b --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/segmentation/segmenterTypes.ts @@ -0,0 +1,75 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {SegmentationResult} from './segmenter'; + +/** + * Delegate type for ML inference execution. + * + * - 'CPU': Runs inference on CPU (slower but widely supported) + * - 'GPU': Runs inference on GPU (faster but requires WebGL support) + */ +export type SegmenterDelegate = 'CPU' | 'GPU'; + +/** + * Initialization parameters for a segmenter instance. + */ +export interface SegmenterInit { + modelPath: string; + delegate: SegmenterDelegate; + /** Optional canvas for GPU delegate (required for GPU, unused for CPU). */ + canvas?: HTMLCanvasElement | OffscreenCanvas; +} + +/** + * Options for segmentation operations. + */ +export interface SegmenterOptions { + /** Whether to include class mask in the result (for debug visualization). */ + includeClassMask?: boolean; +} + +/** + * Interface for person segmentation operations. + * + * Provides a unified interface for different segmentation implementations + * (e.g., MediaPipe, TensorFlow.js). All segmenters must implement this interface. + */ +export interface SegmenterLike { + init(): Promise; + configure(width: number, height: number): void; + segment(frame: ImageBitmap, timestampMs: number, options?: SegmenterOptions): Promise; + close(): void; + /** + * Returns the delegate type used by this segmenter. + * + * @returns 'CPU' or 'GPU', or undefined if not supported. + */ + getDelegate?(): SegmenterDelegate; +} + +/** + * Factory interface for creating segmenter instances. + * + * Allows dependency injection of different segmenter implementations + * for testing or alternative ML backends. + */ +export interface SegmenterFactory { + create(init: SegmenterInit): SegmenterLike; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/compositeBlur.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/compositeBlur.frag new file mode 100644 index 00000000000..7f4e5b43041 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/compositeBlur.frag @@ -0,0 +1,55 @@ +#version 300 es + +/** + * Background blur compositing shader. + * + * Composites the original video with a blurred background using a segmentation mask. + * Applies soft edge blending for smooth transitions between foreground and background. + * + * Algorithm: + * 1. Extract mask value (0 = background, 1 = foreground) + * 2. Compute soft edge using smoothstep for smooth transitions + * 3. Blend video and blur based on blur strength + * 4. Final composite: use soft edge to blend between blurred and original video + * + * Uniforms: + * uVideo: Original video frame texture + * uBlur: Blurred background texture (downsampled and Gaussian blurred) + * uMask: Segmentation mask texture (grayscale, 0-1) + * uTexelSize: Texture texel size (1/width, 1/height) - unused but kept for consistency + * uSoftLow: Lower threshold for soft edge (0-1) + * uSoftHigh: Upper threshold for soft edge (0-1) + * uBlurStrength: Blur intensity (0-1, 0 = no blur, 1 = full blur) + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uVideo; +uniform sampler2D uBlur; +uniform sampler2D uMask; +uniform vec2 uTexelSize; +uniform float uSoftLow; +uniform float uSoftHigh; +uniform float uBlurStrength; + +void main() { + // Sample input textures + vec4 videoColor = texture(uVideo, vTexCoord); + vec4 blurColor = texture(uBlur, vTexCoord); + float mask = texture(uMask, vTexCoord).r; + + // Compute soft edge transition (smoothstep for smooth blending) + float edge = smoothstep(uSoftLow, uSoftHigh, mask); + + // Blend video and blur based on blur strength + vec4 blendedBlur = mix(videoColor, blurColor, uBlurStrength); + + // Final composite: use soft edge to blend between blurred and original + // Higher edge value = more original video (foreground), lower = more blur (background) + outColor = mix(blendedBlur, videoColor, edge); +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/compositeVirtual.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/compositeVirtual.frag new file mode 100644 index 00000000000..f18ddd2c75d --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/compositeVirtual.frag @@ -0,0 +1,58 @@ +#version 300 es + +/** + * Virtual background compositing shader. + * + * Composites the original video with a background image/video using a segmentation mask. + * Uses matte thresholds for hard cutoffs and smoothstep for edge blending. + * + * Algorithm: + * 1. Extract mask value (0 = background, 1 = foreground) + * 2. Compute matte using smoothstep for smooth edge transitions + * 3. Scale background UV coordinates for "cover" sizing (maintains aspect ratio) + * 4. Composite: mix background and video based on matte value + * + * Background scaling: + * uBgScale is computed to ensure background covers entire frame while maintaining + * aspect ratio. UV coordinates are centered and scaled for proper coverage. + * + * Uniforms: + * uVideo: Original video frame texture + * uMask: Segmentation mask texture (grayscale, 0-1) + * uBg: Background image/video texture + * uTexelSize: Texture texel size (1/width, 1/height) - unused but kept for consistency + * uMatteLow: Lower threshold for matte cutoff (0-1) + * uMatteHigh: Upper threshold for matte cutoff (0-1) + * uBgScale: Background UV scale factors [x, y] for cover sizing + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uVideo; +uniform sampler2D uMask; +uniform sampler2D uBg; +uniform vec2 uTexelSize; +uniform float uMatteLow; +uniform float uMatteHigh; +uniform vec2 uBgScale; + +void main() { + // Sample input textures + vec4 videoColor = texture(uVideo, vTexCoord); + float mask = texture(uMask, vTexCoord).r; + + // Compute matte using smoothstep for smooth edge transitions + float matte = smoothstep(uMatteLow, uMatteHigh, mask); + + // Scale background UV coordinates for "cover" sizing (centered, maintains aspect ratio) + vec2 bgUv = (vTexCoord - 0.5) * uBgScale + 0.5; + vec4 bgColor = texture(uBg, bgUv); + + // Composite: matte value determines blend (1 = foreground/video, 0 = background) + outColor = mix(bgColor, videoColor, matte); +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/debugOverlay.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/debugOverlay.frag new file mode 100644 index 00000000000..b2b04d961f5 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/debugOverlay.frag @@ -0,0 +1,83 @@ +#version 300 es + +/** + * Debug overlay visualization shader. + * + * Provides visualization modes for inspecting segmentation masks: + * - Mode 0: Normal rendering (passthrough) + * - Mode 1: Mask overlay (green tint on mask areas) + * - Mode 2: Mask only (grayscale mask visualization) + * - Mode 3: Edge only (highlights mask edges using smoothstep difference) + * - Mode 4: Class overlay (colorized multiclass segmentation on top of video) + * - Mode 5: Class only (colorized multiclass segmentation) + * + * Used for debugging segmentation quality, tuning matte thresholds, and + * verifying mask stability. + * + * Uniforms: + * uVideo: Original video frame texture + * uMask: Segmentation mask texture (grayscale, 0-1) + * uMode: Debug mode (0 = off, 1 = overlay, 2 = mask only, 3 = edge only, 4 = class overlay, 5 = class only) + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uVideo; +uniform sampler2D uMask; +uniform int uMode; + +vec3 hsv2rgb(vec3 c) { + vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0); + vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www); + return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y); +} + +void main() { + vec4 videoColor = texture(uVideo, vTexCoord); + float mask = texture(uMask, vTexCoord).r; + + // Mode 1: Mask overlay (green tint on mask areas) + if (uMode == 1) { + vec4 overlay = vec4(0.0, 1.0, 0.0, 0.5); // Semi-transparent green + outColor = mix(videoColor, overlay, mask); + return; + } + + // Mode 2: Mask only (grayscale mask visualization) + if (uMode == 2) { + outColor = vec4(vec3(mask), 1.0); + return; + } + + // Mode 3: Edge only (highlights edges using smoothstep difference) + if (uMode == 3) { + // Edge detection: difference of two smoothsteps creates a band highlighting edges + float edge = smoothstep(0.4, 0.6, mask) - smoothstep(0.6, 0.8, mask); + outColor = vec4(vec3(edge), 1.0); + return; + } + + // Mode 4: Class overlay (colorized multiclass segmentation on top of video) + if (uMode == 4) { + int classId = int(floor(mask * 255.0 + 0.5)); + vec3 classColor = classId == 0 ? vec3(0.0) : hsv2rgb(vec3(fract(float(classId) * 0.13), 0.75, 0.95)); + float alpha = classId == 0 ? 0.0 : 0.6; + outColor = mix(videoColor, vec4(classColor, 1.0), alpha); + return; + } + + // Mode 5: Class only (colorized multiclass segmentation) + if (uMode == 5) { + int classId = int(floor(mask * 255.0 + 0.5)); + vec3 classColor = classId == 0 ? vec3(0.0) : hsv2rgb(vec3(fract(float(classId) * 0.13), 0.75, 0.95)); + outColor = vec4(classColor, 1.0); + return; + } + + // Mode 0: Normal rendering (passthrough) + outColor = videoColor; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/downsample.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/downsample.frag new file mode 100644 index 00000000000..aa3b353c82b --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/downsample.frag @@ -0,0 +1,32 @@ +#version 300 es + +/** + * Downsample shader (passthrough). + * + * Simple passthrough shader used for downsampling textures. When rendering to + * a smaller framebuffer, the texture is automatically downsampled by the GPU's + * linear filtering. This shader just passes through the sampled color. + * + * Also used as the passthrough compositing shader when bypass mode is enabled + * or when no mask is available. + * + * Uniforms: + * uSrc: Source texture to downsample/passthrough + * uTexelSize: Texture texel size (1/width, 1/height) - unused but kept for consistency + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uSrc; +uniform vec2 uTexelSize; + +void main() { + // Simple passthrough - GPU linear filtering handles downsampling automatically + vec4 color = texture(uSrc, vTexCoord); + outColor = color; +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/fullscreen.vert b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/fullscreen.vert new file mode 100644 index 00000000000..10196e7b8d7 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/fullscreen.vert @@ -0,0 +1,35 @@ +#version 300 es + +/** + * Fullscreen quad vertex shader. + * + * Renders a fullscreen quad covering the entire viewport (-1 to 1 in NDC space). + * Used for all fullscreen rendering passes (blur, compositing, etc.). + * + * Handles Y-flip for coordinate system correction: + * - When rendering to canvas (fbo === null): flips Y to match WebGL coordinate system + * - When rendering to framebuffer: no flip needed + * + * Vertex attributes: + * aPosition: Vertex position in NDC space (location 0) + * aTexCoord: Texture coordinates 0-1 (location 1) + * + * Uniforms: + * uFlipY: If > 0.5, flips Y texture coordinate (1.0 - y) + */ + + +layout(location = 0) in vec2 aPosition; +layout(location = 1) in vec2 aTexCoord; + +uniform float uFlipY; + +out vec2 vTexCoord; + +void main() { + // Flip Y coordinate if needed (for canvas rendering) + vTexCoord = vec2(aTexCoord.x, uFlipY > 0.5 ? 1.0 - aTexCoord.y : aTexCoord.y); + // Pass position through (fullscreen quad in NDC space) + gl_Position = vec4(aPosition, 0.0, 1.0); +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/gaussianBlurH.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/gaussianBlurH.frag new file mode 100644 index 00000000000..a37bd5dbc62 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/gaussianBlurH.frag @@ -0,0 +1,67 @@ +#version 300 es + +/** + * Horizontal Gaussian blur shader. + * + * Implements separable Gaussian blur in the horizontal direction. Used as the + * first pass of a two-pass separable blur (horizontal + vertical) for efficiency. + * + * Algorithm: + * - Samples neighboring pixels in horizontal direction + * - Applies Gaussian weights based on distance from center + * - Normalizes by sum of weights + * + * Separable blur: O(n²) → O(2n) complexity by splitting into two 1D passes. + * This is much more efficient than a full 2D blur kernel. + * + * Uniforms: + * uSrc: Source texture to blur + * uTexelSize: Texture texel size (1/width, 1/height) for pixel offsets + * uRadius: Blur radius in pixels (clamped to MAX_RADIUS = 16) + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uSrc; +uniform vec2 uTexelSize; +uniform float uRadius; + +/** + * Gaussian function for computing blur weights. + * + * @param x - Distance from center pixel + * @param sigma - Standard deviation (controls blur amount) + * @returns Gaussian weight value + */ +float gaussian(float x, float sigma) { + return exp(-(x * x) / (2.0 * sigma * sigma)); +} + +void main() { + const int MAX_RADIUS = 16; // Maximum blur radius (performance limit) + float sigma = max(1.0, uRadius); // Sigma derived from radius + vec4 sum = vec4(0.0); + float weightSum = 0.0; + + // Sample neighboring pixels in horizontal direction + for (int i = -MAX_RADIUS; i <= MAX_RADIUS; i++) { + float fi = float(i); + // Skip samples outside blur radius + if (abs(fi) > uRadius) { + continue; + } + // Compute Gaussian weight for this sample + float weight = gaussian(fi, sigma); + // Sample texture and accumulate weighted color + sum += texture(uSrc, vTexCoord + vec2(fi, 0.0) * uTexelSize) * weight; + weightSum += weight; + } + + // Normalize by sum of weights + outColor = sum / max(0.0001, weightSum); +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/gaussianBlurV.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/gaussianBlurV.frag new file mode 100644 index 00000000000..8cc48c64d68 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/gaussianBlurV.frag @@ -0,0 +1,67 @@ +#version 300 es + +/** + * Vertical Gaussian blur shader. + * + * Implements separable Gaussian blur in the vertical direction. Used as the + * second pass of a two-pass separable blur (horizontal + vertical) for efficiency. + * + * Algorithm: + * - Samples neighboring pixels in vertical direction + * - Applies Gaussian weights based on distance from center + * - Normalizes by sum of weights + * + * Separable blur: O(n²) → O(2n) complexity by splitting into two 1D passes. + * This is much more efficient than a full 2D blur kernel. + * + * Uniforms: + * uSrc: Source texture to blur (typically output from horizontal blur pass) + * uTexelSize: Texture texel size (1/width, 1/height) for pixel offsets + * uRadius: Blur radius in pixels (clamped to MAX_RADIUS = 16) + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uSrc; +uniform vec2 uTexelSize; +uniform float uRadius; + +/** + * Gaussian function for computing blur weights. + * + * @param x - Distance from center pixel + * @param sigma - Standard deviation (controls blur amount) + * @returns Gaussian weight value + */ +float gaussian(float x, float sigma) { + return exp(-(x * x) / (2.0 * sigma * sigma)); +} + +void main() { + const int MAX_RADIUS = 16; // Maximum blur radius (performance limit) + float sigma = max(1.0, uRadius); // Sigma derived from radius + vec4 sum = vec4(0.0); + float weightSum = 0.0; + + // Sample neighboring pixels in vertical direction + for (int i = -MAX_RADIUS; i <= MAX_RADIUS; i++) { + float fi = float(i); + // Skip samples outside blur radius + if (abs(fi) > uRadius) { + continue; + } + // Compute Gaussian weight for this sample + float weight = gaussian(fi, sigma); + // Sample texture and accumulate weighted color + sum += texture(uSrc, vTexCoord + vec2(0.0, fi) * uTexelSize) * weight; + weightSum += weight; + } + + // Normalize by sum of weights + outColor = sum / max(0.0001, weightSum); +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/jointBilateralMask.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/jointBilateralMask.frag new file mode 100644 index 00000000000..d3393eb9cda --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/jointBilateralMask.frag @@ -0,0 +1,96 @@ +#version 300 es + +/** + * Joint bilateral filter for edge-preserving mask smoothing. + * + * Applies edge-preserving smoothing to the segmentation mask using the video + * frame as a guide image. This preserves sharp edges in the mask that correspond + * to edges in the video, while smoothing away noise and artifacts. + * + * Algorithm: + * - For each pixel, samples neighboring pixels within radius + * - Computes two weights: + * - Spatial weight: Gaussian based on distance (smooths spatially) + * - Range weight: Gaussian based on color difference from center (preserves edges) + * - Final weight = spatial × range (joint bilateral) + * - Normalizes weighted mask values + * + * The range weight uses video color differences, so mask edges are preserved + * where video has edges, and smoothed where video is uniform. + * + * Uniforms: + * uMask: Input mask texture to filter + * uVideo: Guide video texture (used for edge preservation) + * uTexelSize: Texture texel size (1/width, 1/height) for pixel offsets + * uSpatialSigma: Spatial smoothing parameter (higher = more smoothing) + * uRangeSigma: Range (color) smoothing parameter (higher = less edge preservation) + * uRadius: Filter radius in pixels (clamped to MAX_RADIUS = 8) + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uMask; +uniform sampler2D uVideo; +uniform vec2 uTexelSize; +uniform float uSpatialSigma; +uniform float uRangeSigma; +uniform float uRadius; + +/** + * Gaussian function for computing filter weights. + * + * @param x - Distance or difference value + * @param sigma - Standard deviation (controls smoothing amount) + * @returns Gaussian weight value + */ +float gaussian(float x, float sigma) { + return exp(-(x * x) / (2.0 * sigma * sigma)); +} + +void main() { + // Sample center pixel color from video (used as reference for range weight) + vec3 centerColor = texture(uVideo, vTexCoord).rgb; + float sum = 0.0; + float weightSum = 0.0; + + const int MAX_RADIUS = 8; // Maximum filter radius (performance limit) + // Sample neighboring pixels in a square kernel + for (int y = -MAX_RADIUS; y <= MAX_RADIUS; y++) { + for (int x = -MAX_RADIUS; x <= MAX_RADIUS; x++) { + float fx = float(x); + float fy = float(y); + // Skip samples outside filter radius + if (abs(fx) > uRadius || abs(fy) > uRadius) { + continue; + } + // Compute sample UV coordinate + vec2 offset = vec2(fx, fy) * uTexelSize; + vec2 sampleUv = vTexCoord + offset; + + // Sample mask and video at this location + float maskValue = texture(uMask, sampleUv).r; + vec3 sampleColor = texture(uVideo, sampleUv).rgb; + + // Compute spatial weight (based on distance from center) + float spatialWeight = gaussian(length(vec2(fx, fy)), max(0.001, uSpatialSigma)); + // Compute range weight (based on color difference from center) + float rangeWeight = gaussian(length(sampleColor - centerColor), max(0.001, uRangeSigma)); + + // Joint bilateral weight: spatial × range + float weight = spatialWeight * rangeWeight; + + // Accumulate weighted mask value + sum += maskValue * weight; + weightSum += weight; + } + } + + // Normalize by sum of weights (fallback to original if no samples) + float filtered = weightSum > 0.0 ? sum / weightSum : texture(uMask, vTexCoord).r; + outColor = vec4(filtered, filtered, filtered, 1.0); +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskDilate.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskDilate.frag new file mode 100644 index 00000000000..0bfaa52c162 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskDilate.frag @@ -0,0 +1,40 @@ +#version 300 es + +/** + * Morphological dilation pass for mask cleanup. + * + * Expands foreground regions by taking the max over a square window. + * + * Uniforms: + * uSrc: Source mask texture + * uTexelSize: Texture texel size (1/width, 1/height) + * uRadius: Radius in pixels (0-4) + */ + +precision highp float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uSrc; +uniform vec2 uTexelSize; +uniform float uRadius; + +void main() { + const int MAX_RADIUS = 4; + int radius = int(clamp(uRadius, 0.0, float(MAX_RADIUS))); + float maxVal = 0.0; + + for (int y = -MAX_RADIUS; y <= MAX_RADIUS; y += 1) { + for (int x = -MAX_RADIUS; x <= MAX_RADIUS; x += 1) { + if (abs(x) > radius || abs(y) > radius) { + continue; + } + vec2 offset = vec2(float(x), float(y)) * uTexelSize; + float value = texture(uSrc, vTexCoord + offset).r; + maxVal = max(maxVal, value); + } + } + + outColor = vec4(maxVal, maxVal, maxVal, 1.0); +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskErode.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskErode.frag new file mode 100644 index 00000000000..5803032c09f --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskErode.frag @@ -0,0 +1,40 @@ +#version 300 es + +/** + * Morphological erosion pass for mask cleanup. + * + * Shrinks foreground regions by taking the min over a square window. + * + * Uniforms: + * uSrc: Source mask texture + * uTexelSize: Texture texel size (1/width, 1/height) + * uRadius: Radius in pixels (0-4) + */ + +precision highp float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uSrc; +uniform vec2 uTexelSize; +uniform float uRadius; + +void main() { + const int MAX_RADIUS = 4; + int radius = int(clamp(uRadius, 0.0, float(MAX_RADIUS))); + float minVal = 1.0; + + for (int y = -MAX_RADIUS; y <= MAX_RADIUS; y += 1) { + for (int x = -MAX_RADIUS; x <= MAX_RADIUS; x += 1) { + if (abs(x) > radius || abs(y) > radius) { + continue; + } + vec2 offset = vec2(float(x), float(y)) * uTexelSize; + float value = texture(uSrc, vTexCoord + offset).r; + minVal = min(minVal, value); + } + } + + outColor = vec4(minVal, minVal, minVal, 1.0); +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskUpsample.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskUpsample.frag new file mode 100644 index 00000000000..5a7272735c2 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/maskUpsample.frag @@ -0,0 +1,35 @@ +#version 300 es + +/** + * Mask upsampling shader. + * + * Upsamples a low-resolution mask to a higher resolution. When rendering to + * a larger framebuffer, the GPU's linear filtering automatically interpolates + * between mask values, providing smooth upsampling. + * + * This shader is used to refine the segmentation mask from low-resolution + * (e.g., 256x144) to a higher resolution for better edge quality. + * + * The GPU's linear texture filtering handles the upsampling interpolation, + * so this shader simply passes through the sampled mask value. + * + * Uniforms: + * uSrc: Low-resolution mask texture to upsample + * uTexelSize: Texture texel size (1/width, 1/height) - unused but kept for consistency + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uSrc; +uniform vec2 uTexelSize; + +void main() { + // Sample mask value - GPU linear filtering handles upsampling interpolation + float maskValue = texture(uSrc, vTexCoord).r; + outColor = vec4(maskValue, maskValue, maskValue, 1.0); +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/temporalMask.frag b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/temporalMask.frag new file mode 100644 index 00000000000..42bda3382f5 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shaders/temporalMask.frag @@ -0,0 +1,48 @@ +#version 300 es + +/** + * Temporal mask stabilization shader. + * + * Applies temporal smoothing to the segmentation mask using exponential moving + * average (EMA) with the previous frame's mask. This reduces flickering and + * jitter by blending current and previous masks. + * + * Algorithm: + * - Samples current mask and previous frame's mask + * - Blends using alpha parameter: stabilized = mix(previous, current, alpha) + * - Higher alpha = more responsive (follows current mask more) + * - Lower alpha = more stable (follows previous mask more) + * + * This provides temporal consistency across frames, reducing artifacts from + * segmentation noise and improving visual quality. + * + * Uniforms: + * uMask: Current frame's mask texture + * uPrevMask: Previous frame's stabilized mask texture + * uTexelSize: Texture texel size (1/width, 1/height) - unused but kept for consistency + * uAlpha: Temporal smoothing alpha (0-1, higher = more responsive, lower = more stable) + */ + + +precision mediump float; + +in vec2 vTexCoord; +out vec4 outColor; + +uniform sampler2D uMask; +uniform sampler2D uPrevMask; +uniform vec2 uTexelSize; +uniform float uAlpha; + +void main() { + // Sample current and previous frame masks + float current = texture(uMask, vTexCoord).r; + float previous = texture(uPrevMask, vTexCoord).r; + + // Exponential moving average: blend previous and current + // uAlpha controls responsiveness (1.0 = fully current, 0.0 = fully previous) + float stabilized = mix(previous, current, uAlpha); + + outColor = vec4(stabilized, stabilized, stabilized, 1.0); +} + diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shared/mask.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/shared/mask.ts new file mode 100644 index 00000000000..8361d7690bf --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shared/mask.ts @@ -0,0 +1,117 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Source of mask data for rendering. + * + * Can provide mask as either ImageBitmap or WebGLTexture, along with + * dimensions and a release function for cleanup. + */ +export interface MaskSource { + mask: ImageBitmap | null; + maskTexture: WebGLTexture | null; + width: number; + height: number; + /** Function to release mask resources when no longer needed. */ + release: () => void; +} + +/** + * Mask input as ImageBitmap. + */ +export interface MaskInputBitmap { + type: 'bitmap'; + bitmap: ImageBitmap; + width: number; + /** Mask height in pixels. */ + height: number; +} + +/** + * Mask input as WebGLTexture. + */ +export interface MaskInputTexture { + type: 'texture'; + texture: WebGLTexture; + width: number; + /** Mask height in pixels. */ + height: number; +} + +/** + * Union type for mask input data. + * + * Masks can be provided as either ImageBitmap (for CPU pipelines) or + * WebGLTexture (for GPU pipelines) to avoid unnecessary conversions. + */ +export type MaskInput = MaskInputBitmap | MaskInputTexture; + +/** + * Result of building mask input from a mask source. + */ +export interface MaskBuildResult { + maskInput: MaskInput | null; + maskBitmap: ImageBitmap | null; + /** Function to release mask resources, or null if no mask. */ + release: (() => void) | null; +} + +/** + * Builds mask input from a mask source. + * + * Converts a MaskSource into the appropriate MaskInput format (bitmap or texture) + * based on what's available. Prefers texture if available (for GPU pipelines), + * otherwise uses bitmap. Returns null inputs if source is null. + * + * @param source - Mask source to convert, or null. + * @returns MaskBuildResult with mask input, bitmap (if applicable), and release function. + */ +export const buildMaskInput = (source: MaskSource | null): MaskBuildResult => { + if (!source) { + return {maskInput: null, maskBitmap: null, release: null}; + } + + if (source.maskTexture) { + return { + maskInput: { + type: 'texture', + texture: source.maskTexture, + width: source.width, + height: source.height, + }, + maskBitmap: null, + release: source.release, + }; + } + + if (source.mask) { + return { + maskInput: { + type: 'bitmap', + bitmap: source.mask, + width: source.width, + height: source.height, + }, + maskBitmap: source.mask, + release: source.release, + }; + } + + return {maskInput: null, maskBitmap: null, release: source.release}; +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/shared/timestamps.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/shared/timestamps.ts new file mode 100644 index 00000000000..449eb6b2a8f --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/shared/timestamps.ts @@ -0,0 +1,43 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Converts a timestamp to monotonic milliseconds, ensuring it never decreases. + * + * Ensures timestamps are strictly increasing by: + * 1. Converting seconds to milliseconds + * 2. Ensuring it's at least 1ms greater than the last timestamp + * 3. Ensuring it's not in the past (at least current time) + * + * This prevents issues with non-monotonic timestamps from video sources + * that can cause problems with temporal smoothing and frame ordering. + * + * @param sourceTimestampSeconds - Source timestamp in seconds. + * @param lastTimestampMs - Last processed timestamp in milliseconds. + * @param nowMs - Current time in milliseconds (defaults to performance.now()). + * @returns Monotonic timestamp in milliseconds. + */ +export const toMonotonicTimestampMs = ( + sourceTimestampSeconds: number, + lastTimestampMs: number, + nowMs: number = performance.now(), +): number => { + const candidate = Math.floor(sourceTimestampSeconds * 1000); + return Math.max(candidate, lastTimestampMs + 1, Math.floor(nowMs)); +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/testBasic.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/testBasic.ts new file mode 100644 index 00000000000..783edcc358c --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/testBasic.ts @@ -0,0 +1,560 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/* eslint-disable no-console */ +/** + * Basic test script for backgroundEffects module + * Run with: ts-node src/script/repositories/media/backgroundEffects/testBasic.ts + * + * Browser demo: + * Bundle this file as an ES module in the app build and open it in the browser. + * It will auto-run a live camera demo when window + mediaDevices are available. + */ + +import type {DebugMode, EffectMode, PipelineType, QualityMode, StartOptions} from './backgroundEffectsWorkerTypes'; +import {choosePipeline, detectCapabilities} from './effects/capability'; + +declare global { + interface Window { + __bgfxDemo?: { + status: 'starting' | 'running' | 'failed'; + error?: string; + options?: { + mode: EffectMode; + debugMode: DebugMode; + quality: QualityMode; + blurStrength: number; + targetFps: number; + backgroundKind: string; + pipeline: string; + }; + pipeline?: string; + requestedPipeline?: string; + stop?: () => void; + }; + } +} + +const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined'; +const isNode = typeof process !== 'undefined' && process.exit; + +type WebglDiagnostics = {webgl1: boolean; webgl2: boolean; renderer: string | null}; + +const getWebglDiagnostics = (): WebglDiagnostics => { + if (!isBrowser) { + return {webgl1: false, webgl2: false, renderer: null as string | null}; + } + const canvas1 = document.createElement('canvas'); + const canvas2 = document.createElement('canvas'); + const gl1 = canvas1.getContext('webgl') || canvas1.getContext('experimental-webgl'); + const gl2 = canvas2.getContext('webgl2'); + const gl = (gl2 || gl1) as WebGLRenderingContext | WebGL2RenderingContext | null; + let renderer: string | null = null; + if (gl) { + const debugExt = gl.getExtension('WEBGL_debug_renderer_info'); + if (debugExt) { + renderer = gl.getParameter(debugExt.UNMASKED_RENDERER_WEBGL) as string; + } + } + return {webgl1: !!gl1, webgl2: !!gl2, renderer}; +}; + +// Set up console output to display in the page (browser only) +if (isBrowser) { + // Use a small delay to ensure DOM is ready + const setupConsole = () => { + const outputDiv = document.getElementById('console-output'); + if (outputDiv) { + const originalLog = console.log; + const originalError = console.error; + const originalWarn = console.warn; + const originalInfo = console.info; + + const addOutput = (message: string, type: string = 'log') => { + if (!outputDiv) { + return; + } + const div = document.createElement('div'); + div.className = type; + div.textContent = message; + outputDiv.appendChild(div); + outputDiv.scrollTop = outputDiv.scrollHeight; + }; + + const formatConsoleArgs = (args: any[]): string => { + if (args.length === 0) { + return ''; + } + const [first, ...rest] = args; + if (typeof first === 'string' && first.includes('%c')) { + const cleaned = first.replace(/%c/g, '').trim(); + const nonStyleArgs = rest.filter(arg => typeof arg !== 'string' || !arg.includes(':')); + return [cleaned, ...nonStyleArgs.map(formatConsoleValue)].filter(Boolean).join(' '); + } + return args.map(formatConsoleValue).join(' '); + }; + + const formatConsoleValue = (value: any): string => { + if (value === null || value === undefined) { + return String(value); + } + if (typeof value === 'string') { + return value; + } + if (value instanceof Error) { + return value.stack ?? `${value.name}: ${value.message}`; + } + try { + return JSON.stringify(value); + } catch { + return String(value); + } + }; + + console.log = function (...args: any[]) { + originalLog.apply(console, args); + addOutput(formatConsoleArgs(args), 'log'); + }; + + console.error = function (...args: any[]) { + originalError.apply(console, args); + addOutput(formatConsoleArgs(args), 'error'); + }; + + console.warn = function (...args: any[]) { + originalWarn.apply(console, args); + addOutput(formatConsoleArgs(args), 'warning'); + }; + + console.info = function (...args: any[]) { + originalInfo.apply(console, args); + addOutput(formatConsoleArgs(args), 'info'); + }; + + (console as any).status = function (ok: boolean, ...args: any[]) { + originalLog.apply(console, args); + addOutput(formatConsoleArgs(args), ok ? 'success' : 'error'); + }; + } else { + const warnMsg = 'console-output div not found, console output will only appear in browser console'; + console.warn(warnMsg); + } + }; + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', setupConsole); + } else { + setupConsole(); + } +} + +// Only run Node.js tests in Node.js environment +if (!isBrowser && isNode) { + console.log('=== Background Effects V2 Basic Tests ===\n'); + + // Test 1: Module exports + console.log('Test 1: Module exports'); + try { + console.log(' ✓ detectCapabilities imported:', typeof detectCapabilities === 'function'); + console.log(' ✓ choosePipeline imported:', typeof choosePipeline === 'function'); + console.log(' Note: BackgroundEffectsController uses import.meta and requires ES modules'); + console.log(' It will be tested in browser-based tests or with proper ES module setup'); + } catch (error) { + console.error(' ✗ Module import failed:', error); + process.exit(1); + } + + // Test 2: Capability detection (requires browser environment) + console.log('\nTest 2: Capability detection'); + try { + // Mock browser globals for Node.js environment + (global as any).window = {}; + (global as any).document = { + createElement: (_tag?: string): {getContext: (contextId?: string) => WebGLRenderingContext | null} => ({ + getContext: (): WebGLRenderingContext | null => null, + }), + }; + + const caps = detectCapabilities(); + console.log(' Capabilities detected:'); + console.log(' - OffscreenCanvas:', caps.offscreenCanvas); + console.log(' - Worker:', caps.worker); + console.log(' - WebGL2:', caps.webgl2); + console.log(' - requestVideoFrameCallback:', caps.requestVideoFrameCallback); + + const pipeline = choosePipeline(caps, true); + console.log(' ✓ BackgroundEffectsRenderingPipeline chosen:', pipeline); + console.log(' Note: In browser environment, capabilities will be properly detected'); + } catch (error) { + console.log(' ⚠ Capability detection requires browser environment'); + console.log(' Error:', (error as Error).message); + console.log(' This is expected in Node.js - will work in browser'); + } + + // Test 3: Type definitions + console.log('\nTest 3: Type definitions validation'); + try { + // Test EffectMode type + const validModes: EffectMode[] = ['blur', 'virtual', 'passthrough']; + console.log(` ✓ EffectMode type valid (${validModes.length} modes)`); + + // Test DebugMode type + const validDebugModes: DebugMode[] = ['off', 'maskOverlay', 'maskOnly', 'edgeOnly']; + console.log(` ✓ DebugMode type valid (${validDebugModes.length} modes)`); + + // Test QualityMode type + const validQualityModes: QualityMode[] = ['auto', 'superhigh', 'high', 'medium', 'low', 'bypass']; + console.log(` ✓ QualityMode type valid (${validQualityModes.length} modes)`); + + console.log(' ✓ All type definitions are valid'); + } catch (error) { + console.error(' ✗ Type validation failed:', error); + process.exit(1); + } + + // Test 5: StartOptions interface + console.log('\nTest 5: StartOptions interface'); + try { + const validOptions: StartOptions = { + mode: 'blur', + blurStrength: 0.6, + quality: 'auto', + targetFps: 30, + debugMode: 'off', + }; + console.log(' ✓ StartOptions type is valid'); + console.log(' Sample options:', JSON.stringify(validOptions, null, 2)); + } catch (error) { + console.error(' ✗ StartOptions validation failed:', error); + process.exit(1); + } + + console.log('\n=== All basic tests passed! ==='); + console.log('\nNote: Full integration tests require browser environment with MediaStream API.'); + console.log('To test with actual video stream, use browser-based tests or manual testing.'); +} + +// Browser-specific code - run after DOM is ready +if (isBrowser) { + const runBrowserTests = () => { + // In browser, run capability detection and then demo + console.log('=== Background Effects V2 Browser Test ==='); + console.log('Script loaded successfully. Running tests...'); + + try { + const caps = detectCapabilities(); + const logExpectation = (label: string, value: boolean, expected = true) => { + const ok = value === expected; + const prefix = ok ? '✓' : '✗'; + const message = ` ${prefix} ${label}: ${value}`; + if ((console as any).status) { + (console as any).status(ok, message); + } else { + (ok ? console.log : console.error)(message); + } + }; + console.log('Capabilities detected:'); + logExpectation('OffscreenCanvas', caps.offscreenCanvas); + logExpectation('Worker', caps.worker); + logExpectation('WebGL2', caps.webgl2); + logExpectation('requestVideoFrameCallback', caps.requestVideoFrameCallback); + const webgl = getWebglDiagnostics(); + logExpectation('WebGL1', webgl.webgl1); + logExpectation('WebGL2 (context check)', webgl.webgl2); + if (webgl.renderer) { + console.log(' - WebGL renderer:', webgl.renderer); + } else { + console.log(' - WebGL renderer:', 'unavailable (context or extension missing)'); + } + + const pipeline = choosePipeline(caps, true); + console.log(' ✓ BackgroundEffectsRenderingPipeline chosen:', pipeline); + console.log('Starting browser demo...'); + } catch (error) { + console.error(' ✗ Capability detection failed:', error); + console.error('Error details:', error); + } + }; + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', runBrowserTests); + } else { + runBrowserTests(); + } +} + +async function runBrowserDemo() { + if (typeof window === 'undefined' || typeof document === 'undefined') { + return; + } + if (!navigator.mediaDevices?.getUserMedia) { + console.log('Browser demo skipped: MediaDevices API not available.'); + return; + } + + const params = new URLSearchParams(window.location.search); + const mode = (params.get('mode') as EffectMode) || 'virtual'; + const debugMode = (params.get('debug') as DebugMode) || 'off'; + const quality = (params.get('quality') as QualityMode) || 'auto'; + const blurStrength = Number(params.get('blur') ?? '0.7'); + const targetFps = Number(params.get('fps') ?? '30'); + const backgroundKind = params.get('bg') || (mode === 'virtual' ? 'gradient' : 'none'); + const pipelineParam = params.get('pipeline'); + const pipelineOverride = ( + pipelineParam && ['worker-webgl2', 'main-webgl2', 'canvas2d', 'passthrough'].includes(pipelineParam) + ? pipelineParam + : undefined + ) as PipelineType | undefined; + + const {BackgroundEffectsController} = await import('./effects/backgroundEffectsController'); + + const caps = detectCapabilities(); + const chosenPipeline = choosePipeline(caps, true); + const activePipeline = pipelineOverride ?? chosenPipeline; + + const root = document.getElementById('app-root') || document.createElement('div'); + if (!root.id) { + root.id = 'app-root'; + root.style.display = 'grid'; + root.style.gridTemplateColumns = '1fr 1fr'; + root.style.gap = '12px'; + root.style.padding = '12px'; + root.style.background = '#111'; + root.style.color = '#fff'; + root.style.font = '14px/1.4 sans-serif'; + document.body.appendChild(root); + } else { + // Use existing app-root, just set styles if needed + root.style.display = 'grid'; + root.style.gridTemplateColumns = '1fr 1fr'; + root.style.gap = '12px'; + root.style.padding = '12px'; + } + + const createPanel = (title: string) => { + const panel = document.createElement('div'); + const heading = document.createElement('div'); + heading.textContent = title; + heading.style.marginBottom = '6px'; + panel.appendChild(heading); + return panel; + }; + + const rawPanel = createPanel('Raw input'); + const processedPanel = createPanel('Processed output'); + const metricsPanel = createPanel('Performance'); + root.appendChild(rawPanel); + root.appendChild(processedPanel); + root.appendChild(metricsPanel); + + const rawVideo = document.createElement('video'); + rawVideo.id = 'bgfx-raw-video'; + rawVideo.autoplay = true; + rawVideo.muted = true; + rawVideo.playsInline = true; + rawVideo.style.width = '100%'; + rawVideo.style.background = '#000'; + rawPanel.appendChild(rawVideo); + + const processedVideo = document.createElement('video'); + processedVideo.id = 'bgfx-processed-video'; + processedVideo.autoplay = true; + processedVideo.muted = true; + processedVideo.playsInline = true; + processedVideo.style.width = '100%'; + processedVideo.style.background = '#000'; + processedPanel.appendChild(processedVideo); + + const status = document.createElement('div'); + status.id = 'bgfx-status'; + status.textContent = + `mode=${mode} debug=${debugMode} quality=${quality} blur=${blurStrength} ` + + `fps=${targetFps} bg=${backgroundKind} pipeline=${activePipeline}`; + status.style.gridColumn = '1 / -1'; + root.appendChild(status); + + const metricsLine = document.createElement('div'); + metricsLine.id = 'bgfx-metrics'; + metricsLine.textContent = 'metrics: waiting for frames...'; + metricsPanel.appendChild(metricsLine); + + const logGetUserMediaError = (error: unknown, label: string) => { + const err = error as DOMException & {constraint?: string}; + console.error(`${label} failed:`, err?.name ?? err, err?.message ?? ''); + if (err?.constraint) { + console.error(`${label} constraint:`, err.constraint); + } + }; + + window.__bgfxDemo = { + status: 'starting', + options: { + mode, + debugMode, + quality, + blurStrength, + targetFps, + backgroundKind, + pipeline: activePipeline, + }, + requestedPipeline: pipelineOverride ?? 'auto', + pipeline: activePipeline, + }; + + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + video: {width: {ideal: 1280}, height: {ideal: 720}}, + }); + } catch (error: unknown) { + logGetUserMediaError(error, 'getUserMedia (ideal 1280x720)'); + stream = await navigator.mediaDevices.getUserMedia({video: true}).catch((err: unknown) => { + logGetUserMediaError(err, 'getUserMedia (fallback)'); + throw err; + }); + } + const inputTrack = stream.getVideoTracks()[0]; + const settings = inputTrack.getSettings() as MediaTrackSettings & { + exposureMode?: string; + focusMode?: string; + resizeMode?: string; + whiteBalanceMode?: string; + }; + const fmt = (value: unknown) => (value === undefined ? 'n/a' : String(value)); + console.log( + 'Video track settings:', + `resolution=${fmt(settings.width)}x${fmt(settings.height)} ` + + `fps=${fmt(settings.frameRate)} ` + + `aspect=${fmt(settings.aspectRatio)} ` + + `exposure=${fmt(settings.exposureMode)} ` + + `whiteBalance=${fmt(settings.whiteBalanceMode)} ` + + `focus=${fmt(settings.focusMode)} ` + + `resizeMode=${fmt(settings.resizeMode)}`, + ); + console.log('Video track state:', inputTrack.readyState); + rawVideo.srcObject = new MediaStream([inputTrack]); + rawVideo.play().catch((error: unknown) => console.warn('Raw video play failed', error)); + + const backgroundImage = await createBackgroundImage(backgroundKind, settings); + const controller = new BackgroundEffectsController(); + const {outputTrack, stop} = await controller.start(inputTrack, { + mode, + debugMode, + quality, + blurStrength, + targetFps, + backgroundImage: backgroundImage ?? undefined, + pipelineOverride, + onMetrics: metrics => { + const budgetMs = 1000 / targetFps; + const totalMs = metrics.avgTotalMs || 0; + const utilization = budgetMs > 0 ? Math.min(999, (totalMs / budgetMs) * 100) : 0; + const mlShare = totalMs > 0 ? (metrics.avgSegmentationMs / totalMs) * 100 : 0; + const webglShare = totalMs > 0 ? (metrics.avgGpuMs / totalMs) * 100 : 0; + // Label ML phase based on actual delegate type + const mlLabel = metrics.segmentationDelegate ? `ML(${metrics.segmentationDelegate})` : 'ML'; + metricsLine.textContent = + `total=${totalMs.toFixed(1)}ms ` + + `seg=${metrics.avgSegmentationMs.toFixed(1)}ms ` + + `webgl=${metrics.avgGpuMs.toFixed(1)}ms ` + + `budget=${budgetMs.toFixed(1)}ms ` + + `util=${utilization.toFixed(0)}% ` + + `${mlLabel}=${mlShare.toFixed(0)}% ` + + `webgl=${webglShare.toFixed(0)}% ` + + `tier=${metrics.tier} dropped=${metrics.droppedFrames}`; + }, + }); + + processedVideo.srcObject = new MediaStream([outputTrack]); + window.__bgfxDemo = {...window.__bgfxDemo, status: 'running', stop}; + + console.log('Browser demo running.'); +} + +async function createBackgroundImage(kind: string, settings: MediaTrackSettings): Promise { + if (!kind || kind === 'none') { + return null; + } + const width = Math.max(1, Math.floor(settings.width ?? 1280)); + const height = Math.max(1, Math.floor(settings.height ?? 720)); + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (!ctx) { + return null; + } + + if (kind === 'gradient') { + const gradient = ctx.createLinearGradient(0, 0, width, height); + gradient.addColorStop(0, '#0f2027'); + gradient.addColorStop(0.5, '#203a43'); + gradient.addColorStop(1, '#2c5364'); + ctx.fillStyle = gradient; + ctx.fillRect(0, 0, width, height); + + ctx.fillStyle = 'rgba(255, 255, 255, 0.08)'; + for (let i = 0; i < 14; i += 1) { + const size = Math.floor(width * 0.08) + i * 6; + ctx.beginPath(); + ctx.arc(width * 0.2 + i * 30, height * 0.2 + i * 18, size, 0, Math.PI * 2); + ctx.fill(); + } + } else if (kind === 'grid') { + ctx.fillStyle = '#1b1b1b'; + ctx.fillRect(0, 0, width, height); + ctx.strokeStyle = 'rgba(255, 255, 255, 0.08)'; + ctx.lineWidth = 1; + const step = 40; + for (let x = 0; x <= width; x += step) { + ctx.beginPath(); + ctx.moveTo(x, 0); + ctx.lineTo(x, height); + ctx.stroke(); + } + for (let y = 0; y <= height; y += step) { + ctx.beginPath(); + ctx.moveTo(0, y); + ctx.lineTo(width, y); + ctx.stroke(); + } + ctx.fillStyle = 'rgba(255, 255, 255, 0.06)'; + ctx.fillRect(width * 0.55, height * 0.1, width * 0.35, height * 0.3); + } else { + return null; + } + + return createImageBitmap(canvas); +} + +// Run browser demo when in browser environment +if (isBrowser) { + const startDemo = () => { + runBrowserDemo().catch((error: unknown) => { + window.__bgfxDemo = {status: 'failed', error: (error as Error)?.message ?? String(error)}; + console.error('Browser demo failed:', error); + }); + }; + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', startDemo); + } else { + startDemo(); + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/worker/backgroundEffectsWorkerState.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/backgroundEffectsWorkerState.ts new file mode 100644 index 00000000000..b945bb05fe6 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/backgroundEffectsWorkerState.ts @@ -0,0 +1,103 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import type {DebugMode, EffectMode, Metrics, QualityMode, WorkerOptions} from '../backgroundEffectsWorkerTypes'; +import {createMetricsWindow, type MetricsWindow, QualityController} from '../quality'; +import {WebGlRenderer} from '../renderer/webGlRenderer'; +import {Segmenter} from '../segmentation/segmenter'; + +/** + * Global state for the background effects worker. + * + * Maintains all state needed for frame processing, including renderer, + * segmenter, quality controller, configuration, and metrics. + */ +export interface BackgroundEffectsWorkerState { + renderer: WebGlRenderer | null; + segmenter: Segmenter | null; + qualityController: QualityController | null; + options: WorkerOptions | null; + width: number; + height: number; + mode: EffectMode; + debugMode: DebugMode; + blurStrength: number; + quality: QualityMode; + currentModelPath: string | null; + pendingModelPath: string | null; + segmenterInitPromise: Promise | null; + metrics: Metrics; + frameCount: number; + background: ImageBitmap | null; + backgroundSize: {width: number; height: number} | null; + lastTimestampMs: number; + metricsWindow: MetricsWindow; + canvas: OffscreenCanvas | null; + /** Whether the WebGL context has been lost. */ + contextLost: boolean; + segmenterErrorCount: number; + fatalError: string | null; +} + +/** + * Maximum number of metrics samples to keep in the metrics window. + * + * Used for calculating rolling averages of performance metrics. + */ +export const METRICS_MAX_SAMPLES = 30; + +/** + * Global worker state instance. + * + * Shared state object used throughout the worker for frame processing, + * configuration, and resource management. + */ +export const state: BackgroundEffectsWorkerState = { + renderer: null, + segmenter: null, + qualityController: null, + options: null, + width: 0, + height: 0, + mode: 'blur', + debugMode: 'off', + blurStrength: 0.5, + quality: 'auto', + currentModelPath: null, + pendingModelPath: null, + segmenterInitPromise: null, + metrics: { + avgTotalMs: 0, + avgSegmentationMs: 0, + avgGpuMs: 0, + segmentationDelegate: null, + droppedFrames: 0, + tier: 'superhigh', + }, + metricsWindow: createMetricsWindow(METRICS_MAX_SAMPLES), + frameCount: 0, + background: null, + backgroundSize: null, + lastTimestampMs: 0, + canvas: null, + contextLost: false, + // Error handling + segmenterErrorCount: 0, + fatalError: null, +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/worker/bgfx.worker.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/bgfx.worker.ts new file mode 100644 index 00000000000..9b1b6192bb4 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/bgfx.worker.ts @@ -0,0 +1,133 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +/** + * Background Effects Web Worker + * + * This Web Worker processes video frames in a background thread to avoid blocking + * the main thread. It handles: + * - ML-based person segmentation (MediaPipe Selfie Segmentation) + * - GPU-accelerated rendering (WebGL2 via OffscreenCanvas) + * - Adaptive quality control + * - Background blur and virtual background effects + * + * Communication with the main thread is via postMessage/onmessage using + * structured cloneable types (ImageBitmap, OffscreenCanvas). + */ + +import {state} from './backgroundEffectsWorkerState'; +import {handleFrame} from './frameProcessor'; +import {cleanup, handleBackgroundImage, handleInit} from './handlers'; + +import type {WorkerMessage, WorkerResponse} from '../backgroundEffectsWorkerTypes'; + +self.addEventListener('error', event => { + try { + postMessage({ + type: 'workerError', + reason: 'error', + message: event.message, + filename: event.filename, + lineno: event.lineno, + colno: event.colno, + } as WorkerResponse); + } catch { + // ignore + } +}); + +self.addEventListener('unhandledrejection', event => { + try { + postMessage({ + type: 'workerError', + reason: 'unhandledrejection', + message: String(event.reason), + } as WorkerResponse); + } catch { + // ignore + } +}); + +/** + * Main message handler for worker communication. + * + * Handles all message types from the main thread: + * - 'init': Initialize renderer, segmenter, and quality controller + * - 'frame': Process a video frame (segmentation + rendering) + * - 'setMode': Update effect mode (blur/virtual/passthrough) + * - 'setBlurStrength': Update blur strength parameter + * - 'setDebugMode': Update debug visualization mode + * - 'setQuality': Update quality mode (auto or fixed tier) + * - 'setDroppedFrames': Update dropped frame counter + * - 'setBackgroundImage': Set background image for virtual background + * - 'setBackgroundVideo': Set background video for virtual background + * - 'stop': Clean up resources and terminate + */ +self.onmessage = async (event: MessageEvent) => { + const message = event.data; + switch (message.type) { + case 'init': + await handleInit(message.canvas, message.width, message.height, message.options); + postMessage({type: 'ready'} as WorkerResponse); + break; + case 'frame': + try { + await handleFrame(message.frame, message.timestamp, message.width, message.height); + } catch { + // Increment dropped frame counter on processing errors + state.metrics.droppedFrames += 1; + } finally { + // Always notify main thread that frame processing completed + postMessage({type: 'frameProcessed'} as WorkerResponse); + } + break; + case 'setMode': + state.mode = message.mode ?? state.mode; + break; + case 'setBlurStrength': + state.blurStrength = message.blurStrength ?? state.blurStrength; + break; + case 'setDebugMode': + state.debugMode = message.debugMode ?? state.debugMode; + break; + case 'setQuality': + state.quality = message.quality ?? state.quality; + // If quality is fixed (not auto), update quality controller tier + if (state.quality !== 'auto' && state.qualityController) { + state.qualityController.setTier(state.quality); + } + break; + case 'setDroppedFrames': + if (typeof message.droppedFrames === 'number') { + state.metrics.droppedFrames = message.droppedFrames; + } + break; + case 'setBackgroundImage': + await handleBackgroundImage(message.image ?? null, message.width ?? 0, message.height ?? 0); + break; + case 'setBackgroundVideo': + await handleBackgroundImage(message.video ?? null, message.width ?? 0, message.height ?? 0); + break; + case 'stop': + cleanup(); + break; + default: + break; + } +}; diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/worker/contextLoss.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/contextLoss.ts new file mode 100644 index 00000000000..b87e72b70eb --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/contextLoss.ts @@ -0,0 +1,129 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {state} from './backgroundEffectsWorkerState'; + +import type {WorkerResponse} from '../backgroundEffectsWorkerTypes'; +import {resolveSegmentationModelPath} from '../quality'; +import {WebGlRenderer} from '../renderer/webGlRenderer'; +import {Segmenter} from '../segmentation/segmenter'; + +let contextHandlersBound = false; + +/** + * Binds WebGL context loss and restoration event handlers to the canvas. + * + * Registers event listeners for 'webglcontextlost' and 'webglcontextrestored' + * events. Only binds handlers once per canvas to avoid duplicate listeners. + * Prevents default context loss behavior to allow graceful recovery. + * + * @param canvas - OffscreenCanvas to bind handlers to. + * @returns Nothing. + */ +export function bindContextLossHandlers(canvas: OffscreenCanvas): void { + if (contextHandlersBound || typeof canvas.addEventListener !== 'function') { + return; + } + canvas.addEventListener('webglcontextlost', handleContextLost as EventListener); + canvas.addEventListener('webglcontextrestored', handleContextRestored as EventListener); + contextHandlersBound = true; +} + +/** + * Resets the context loss handlers binding state. + * + * Allows handlers to be rebound after cleanup. Called when the worker + * is stopped to prepare for potential reinitialization. + * + * @returns Nothing. + */ +export function resetContextLossHandlers(): void { + contextHandlersBound = false; +} + +/** + * Handles WebGL context loss event. + * + * Prevents default context loss behavior, marks context as lost in state, + * notifies the main thread via postMessage, and cleans up renderer and + * segmenter resources. The main thread should handle fallback to another pipeline. + * + * @param event - WebGL context lost event. + * @returns Nothing. + */ +function handleContextLost(event: Event): void { + event.preventDefault(); + state.contextLost = true; + postMessage({type: 'contextLost'} as WorkerResponse); + try { + state.renderer?.destroy(); + } catch (error) { + console.warn('[bgfx.worker] Failed to destroy renderer after context loss', error); + } + state.renderer = null; + state.segmenter?.close(); + state.segmenter = null; +} + +/** + * Handles WebGL context restoration event. + * + * Recreates the WebGL renderer and reinitializes the segmenter if needed. + * Restores the previous quality tier and mode. If restoration fails, marks + * context as lost and notifies the main thread. + * + * @returns Promise that resolves when context restoration is complete. + */ +async function handleContextRestored(): Promise { + if (!state.canvas || !state.options) { + return; + } + try { + state.renderer = new WebGlRenderer(state.canvas, state.width, state.height); + } catch (error) { + console.warn('[bgfx.worker] Renderer restore failed', error); + state.renderer = null; + state.contextLost = true; + return; + } + + state.contextLost = false; + const tier = state.metrics?.tier ?? (state.quality === 'auto' ? state.options.initialTier : state.quality); + if (tier === 'bypass' || state.mode === 'passthrough') { + state.segmenter?.close(); + state.segmenter = null; + state.currentModelPath = null; + return; + } + const modelPath = resolveSegmentationModelPath( + tier, + state.options.segmentationModelByTier, + state.options.segmentationModelPath, + ); + state.segmenter?.close(); + state.segmenter = new Segmenter(modelPath, 'GPU', state.canvas ?? undefined); + try { + await state.segmenter.init(); + state.currentModelPath = modelPath; + } catch (error) { + console.warn('[bgfx.worker] Segmenter restore failed', error); + state.segmenter = null; + state.currentModelPath = null; + } +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/worker/frameProcessor.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/frameProcessor.ts new file mode 100644 index 00000000000..bb37274a4dc --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/frameProcessor.ts @@ -0,0 +1,313 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {state} from './backgroundEffectsWorkerState'; + +import type {QualityTier, QualityTierParams} from '../backgroundEffectsWorkerTypes'; +import { + buildMetrics, + isProcessingMode, + pushMetricsSample, + resolveQualityTierForEffectMode, + resolveSegmentationModelPath, +} from '../quality'; +import {Segmenter} from '../segmentation/segmenter'; +import {buildMaskInput, type MaskInput, type MaskSource} from '../shared/mask'; +import {toMonotonicTimestampMs} from '../shared/timestamps'; + +/** + * Processes a single video frame in the worker thread. + * + * Performs the complete frame processing pipeline: + * 1. Updates canvas dimensions if frame size changed + * 2. Resolves quality tier and ensures segmenter is configured + * 3. Performs segmentation (if cadence allows) + * 4. Configures renderer with current settings + * 5. Renders frame with effects applied + * 6. Updates performance metrics + * + * Handles context loss by checking state.contextLost and skipping processing. + * Always closes the input frame to prevent memory leaks. + * + * @param frame - Input video frame as ImageBitmap (will be closed). + * @param timestamp - Frame timestamp in seconds. + * @param width - Frame width in pixels. + * @param height - Frame height in pixels. + * @returns Promise that resolves when frame processing is complete. + */ +export async function handleFrame(frame: ImageBitmap, timestamp: number, width: number, height: number): Promise { + if (state.fatalError) { + frame.close(); + return; + } + + const renderer = state.renderer; + if (!renderer || state.contextLost) { + frame.close(); + return; + } + + let maskInput: MaskInput | null = null; + let maskBitmap: ImageBitmap | null = null; + let releaseMaskResources: (() => void) | null = null; + try { + if (width !== state.width || height !== state.height) { + state.width = width; + state.height = height; + if (state.canvas) { + state.canvas.width = width; + state.canvas.height = height; + } + } + + let qualityTier = resolveQualityTierParams(); + if (!qualityTier.bypass) { + ensureSegmenterForTier(qualityTier.tier); + if (!state.segmenter) { + qualityTier = {...qualityTier, bypass: true}; + } + } + let segmentationMs = 0; + + if (!qualityTier.bypass && state.segmenter && qualityTier.segmentationCadence > 0) { + if (state.frameCount % qualityTier.segmentationCadence === 0) { + state.segmenter.configure(qualityTier.segmentationWidth, qualityTier.segmentationHeight); + const timestampMs = nextTimestampMs(timestamp); + const includeClassMask = state.debugMode === 'classOverlay' || state.debugMode === 'classOnly'; + const result = await state.segmenter.segment(frame, timestampMs, {includeClassMask}); + const useClassMask = includeClassMask && result.classMask; + const maskSource: MaskSource = useClassMask + ? { + mask: result.classMask, + maskTexture: null, + width: result.width, + height: result.height, + release: result.release, + } + : result; + if (useClassMask) { + result.mask?.close(); + } else { + result.classMask?.close(); + } + const maskResult = buildMaskInput(maskSource); + releaseMaskResources = maskResult.release; + maskInput = maskResult.maskInput; + maskBitmap = maskResult.maskBitmap; + segmentationMs = result.durationMs; + } + } + + if (state.canvas && (state.canvas.width !== state.width || state.canvas.height !== state.height)) { + state.canvas.width = state.width; + state.canvas.height = state.height; + } + + renderer.configure(state.width, state.height, qualityTier, state.mode, state.debugMode, state.blurStrength); + + if (state.background && state.backgroundSize) { + renderer.setBackground(state.background, state.backgroundSize.width, state.backgroundSize.height); + } + + const gpuStart = performance.now(); + try { + renderer.render(frame, maskInput); + } finally { + maskBitmap?.close(); + releaseMaskResources?.(); + } + const gpuMs = performance.now() - gpuStart; + + state.frameCount += 1; + + const totalMs = segmentationMs + gpuMs; + updateMetrics(totalMs, segmentationMs, gpuMs, qualityTier.tier); + } finally { + frame.close(); + } +} + +/** + * Resolves quality tier parameters for the current configuration. + * + * Delegates to resolveQualityTierForEffectMode with the current quality + * controller, quality mode, and effect mode from worker state. + * + * @returns Quality tier parameters for current configuration. + */ +function resolveQualityTierParams(): QualityTierParams { + return resolveQualityTierForEffectMode(state.qualityController, state.quality, state.mode); +} + +/** + * Ensures the segmenter is initialized for the specified quality tier. + * + * If the tier requires a different model than currently loaded, initiates + * an asynchronous segmenter swap. Skips if tier is 'D' (bypass), if a swap + * is already in progress, or if the desired model is already loaded. + * + * Uses GPU delegate for worker pipeline. The swap happens asynchronously + * to avoid blocking frame processing. + * + * @param tier - Quality tier ('superhigh', 'high', 'medium', or 'low', or ''bypass). + * @returns Nothing. + */ +let currentInitId = 0; + +function ensureSegmenterForTier(tier: QualityTier): void { + if (!state.options || !state.canvas) { + return; + } + if (tier === 'bypass') { + return; + } + + const desiredPath = resolveSegmentationModelPath( + tier, + state.options.segmentationModelByTier, + state.options.segmentationModelPath, + ); + + if (state.currentModelPath === desiredPath && state.segmenter) { + return; + } + if (state.pendingModelPath === desiredPath) { + console.info('[bgfx.worker] Segmentation change for model swap, already in progress'); + return; + } + + const initId = ++currentInitId; + state.pendingModelPath = desiredPath; + + const nextSegmenter = new Segmenter(desiredPath, 'GPU', state.canvas as OffscreenCanvas); + + state.segmenterInitPromise = (async () => { + try { + console.info('[bgfx.worker] loading model', desiredPath); + + await nextSegmenter.init(); + + console.info('[bgfx.worker] model ready', desiredPath); + + state.segmenterErrorCount = 0; + state.fatalError = null; + } catch (error) { + console.warn('[bgfx.worker] Segmentation model swap failed, keeping previous model', error); + + state.segmenterErrorCount++; + self.postMessage({ + type: 'segmenterError', + model: desiredPath, + message: String(error), + }); + + if (state.segmenterErrorCount >= 3) { + state.fatalError = 'segmenter_failed_repeatedly'; + + self.postMessage({ + type: 'workerError', + reason: 'segmenter', + message: state.fatalError, + }); + } + nextSegmenter.close(); + if (initId === currentInitId) { + state.pendingModelPath = null; + + if (!state.segmenter) { + state.segmenter = null; + state.currentModelPath = null; + } + } + return; + } + + // In case meanwhile a new init process stated again we discard this segmenter + if (initId !== currentInitId) { + console.warn('[bgfx.worker] Segmentation model swap again, we use next segmenter and discard previous one'); + nextSegmenter.close(); + return; + } + + state.segmenter?.close(); + state.segmenter = nextSegmenter; + state.currentModelPath = desiredPath; + state.pendingModelPath = null; + })(); + + void state.segmenterInitPromise.finally(() => { + if (initId === currentInitId) { + state.segmenterInitPromise = null; + } + }); +} + +/** + * Updates performance metrics and sends them to the main thread. + * + * Updates the quality controller if in 'auto' mode, pushes metrics sample + * to the metrics window, builds aggregated metrics, and posts them to + * the main thread via postMessage. + * + * @param totalMs - Total frame processing time in milliseconds. + * @param segmentationMs - Segmentation processing time in milliseconds. + * @param gpuMs - GPU rendering time in milliseconds. + * @param tier - Current quality tier. + * @returns Nothing. + */ +function updateMetrics(totalMs: number, segmentationMs: number, gpuMs: number, tier: QualityTier): void { + if (!state.qualityController) { + return; + } + + const processingMode = isProcessingMode(state.mode) ? state.mode : null; + const params = processingMode + ? state.quality === 'auto' + ? state.qualityController.update({totalMs, segmentationMs, gpuMs}, processingMode) + : state.qualityController.getTier(processingMode) + : null; + + pushMetricsSample(state.metricsWindow, {totalMs, segmentationMs, gpuMs}); + // Get segmentation delegate type (null if no segmenter) + const segmentationDelegate = state.segmenter?.getDelegate() ?? null; + state.metrics = buildMetrics( + state.metricsWindow, + state.metrics.droppedFrames, + state.quality === 'auto' && params ? params.tier : tier, + segmentationDelegate, + ); + + postMessage({type: 'metrics', metrics: state.metrics}); +} + +/** + * Converts frame timestamp to monotonic milliseconds. + * + * Ensures timestamps are strictly increasing and never in the past by + * using toMonotonicTimestampMs. Updates state.lastTimestampMs with the + * result for the next frame. + * + * @param sourceTimestampSeconds - Frame timestamp in seconds. + * @returns Monotonic timestamp in milliseconds. + */ +function nextTimestampMs(sourceTimestampSeconds: number): number { + const monotonic = toMonotonicTimestampMs(sourceTimestampSeconds, state.lastTimestampMs); + state.lastTimestampMs = monotonic; + return monotonic; +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffects/worker/handlers.ts b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/handlers.ts new file mode 100644 index 00000000000..e3d61ab8f8f --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffects/worker/handlers.ts @@ -0,0 +1,146 @@ +/* + * Wire + * Copyright (C) 2025 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {state} from './backgroundEffectsWorkerState'; +import {bindContextLossHandlers, resetContextLossHandlers} from './contextLoss'; + +import type {WorkerOptions, WorkerResponse} from '../backgroundEffectsWorkerTypes'; +import {QualityController, resolveSegmentationModelPath} from '../quality'; +import {WebGlRenderer} from '../renderer/webGlRenderer'; +import {Segmenter} from '../segmentation/segmenter'; + +/** + * Handles worker initialization message. + * + * Sets up the WebGL renderer, quality controller, and segmenter based on + * the provided options. Binds context loss handlers to the canvas. + * Initializes segmenter asynchronously if needed (non-bypass tier). + * + * If segmenter initialization fails, sends 'segmenterError' message to + * main thread and continues in bypass mode. + * + * @param canvas - OffscreenCanvas for WebGL rendering. + * @param width - Initial canvas width in pixels. + * @param height - Initial canvas height in pixels. + * @param options - Worker initialization options. + * @returns Promise that resolves when initialization is complete. + */ +export async function handleInit( + canvas: OffscreenCanvas, + width: number, + height: number, + options: WorkerOptions, +): Promise { + state.options = options; + state.width = width; + state.height = height; + state.mode = options.mode; + state.debugMode = options.debugMode; + state.blurStrength = options.blurStrength; + state.quality = options.quality; + state.canvas = canvas; + + const renderer = new WebGlRenderer(canvas, width, height); + state.renderer = renderer; + + state.qualityController = new QualityController(options.targetFps ?? 15); + if (state.quality === 'auto') { + state.qualityController.setTier(options.initialTier); + } else { + state.qualityController.setTier(state.quality); + } + + const initialTier = options.quality === 'auto' ? options.initialTier : options.quality; + const shouldInitSegmenter = initialTier !== 'bypass' && options.mode !== 'passthrough'; + if (shouldInitSegmenter) { + const modelPath = resolveSegmentationModelPath( + initialTier, + options.segmentationModelByTier, + options.segmentationModelPath, + ); + state.segmenterInitPromise = (async () => { + const segmenter = new Segmenter(modelPath, 'GPU', canvas); + try { + await segmenter.init(); + state.segmenter?.close(); + state.segmenter = segmenter; + state.currentModelPath = modelPath; + } catch (error) { + console.warn('[bgfx.worker] Segmenter init failed, running in bypass mode.', error); + postMessage({type: 'segmenterError', error: String(error)} as WorkerResponse); + segmenter.close(); + state.segmenter = null; + state.currentModelPath = null; + } finally { + state.segmenterInitPromise = null; + } + })(); + } else { + state.segmenter = null; + state.currentModelPath = null; + state.segmenterInitPromise = null; + } + + bindContextLossHandlers(canvas); +} + +/** + * Handles background image update message. + * + * Stores the background image bitmap and dimensions for virtual background + * mode. Closes any existing background bitmap before storing the new one. + * If null is provided, clears the background. + * + * @param bitmap - Background image as ImageBitmap, or null to clear. + * @param width - Image width in pixels. + * @param height - Image height in pixels. + * @returns Promise that resolves immediately. + */ +export async function handleBackgroundImage(bitmap: ImageBitmap | null, width: number, height: number): Promise { + if (!bitmap) { + state.background?.close(); + state.background = null; + state.backgroundSize = null; + return; + } + state.background?.close(); + state.background = bitmap; + state.backgroundSize = {width, height}; +} + +/** + * Cleans up all worker resources. + * + * Closes segmenter, destroys renderer, releases background bitmap, clears + * canvas reference, resets context loss state, and unbinds context loss handlers. + * Should be called when the worker is being stopped. + * + * @returns Nothing. + */ +export function cleanup(): void { + state.segmenter?.close(); + state.segmenter = null; + state.renderer?.destroy(); + state.renderer = null; + state.background?.close(); + state.background = null; + state.canvas = null; + state.contextLost = false; + resetContextLossHandlers(); +} diff --git a/apps/webapp/src/script/repositories/media/backgroundEffectsHandler.test.ts b/apps/webapp/src/script/repositories/media/backgroundEffectsHandler.test.ts new file mode 100644 index 00000000000..8cc6011bcb7 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffectsHandler.test.ts @@ -0,0 +1,206 @@ +/* + * Wire + * Copyright (C) 2026 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {BackgroundEffectsHandler, ReleasableMediaStream} from './backgroundEffectsHandler'; +import {backgroundEffectsStore} from './useBackgroundEffectsStore'; + +// Mocks +jest.mock('Util/localStorage', () => ({ + getStorage: jest.fn(), +})); + +jest.mock('Repositories/media/VideoBackgroundEffects', () => ({ + BLUR_STRENGTHS: {high: 10}, + DEFAULT_BACKGROUND_EFFECT: {type: 'none'}, + DEFAULT_BUILTIN_BACKGROUND_ID: 'default-id', + loadBackgroundSource: jest.fn(), +})); + +jest.mock('Util/logger', () => ({ + getLogger: () => ({ + warn: jest.fn(), + error: jest.fn(), + }), +})); + +describe('BackgroundEffectsHandler', () => { + let mockController: any; + let mockStorage: any; + + afterEach(() => { + backgroundEffectsStore.getState().setIsFeatureEnabled(false); + backgroundEffectsStore.getState().setPreferredEffect({type: 'none'}); + backgroundEffectsStore.getState().setMetrics(undefined); + }); + + beforeEach(() => { + mockController = { + isProcessing: jest.fn().mockReturnValue(false), + start: jest.fn(), + stop: jest.fn(), + setMode: jest.fn(), + setBlurStrength: jest.fn(), + setBackgroundSource: jest.fn(), + }; + + mockStorage = { + getItem: jest.fn(), + setItem: jest.fn(), + }; + + const {getStorage} = require('Util/localStorage'); + getStorage.mockReturnValue(mockStorage); + }); + + function createMockStream(withTrack = true): MediaStream { + return { + getVideoTracks: () => + withTrack + ? [ + { + stop: jest.fn(), + }, + ] + : [], + } as unknown as MediaStream; + } + + it('returns original stream if effect is none', async () => { + const handler = new BackgroundEffectsHandler(mockController); + + const stream = createMockStream(); + + const result = await handler.applyBackgroundEffect(stream); + + expect(result.applied).toBe(false); + expect(result.media).toBeInstanceOf(ReleasableMediaStream); + }); + + it('returns original stream if no video track exists', async () => { + const handler = new BackgroundEffectsHandler(mockController); + + const stream = createMockStream(false); + + const result = await handler.applyBackgroundEffect(stream); + + expect(result.applied).toBe(false); + }); + + it('applies blur effect successfully', async () => { + const handler = new BackgroundEffectsHandler(mockController); + + handler.setPreferredBackgroundEffect({type: 'blur', level: 'high'} as any); + + const outputTrack = {stop: jest.fn()}; + + mockController.start.mockResolvedValue({ + outputTrack, + stop: jest.fn(), + }); + + const stream = createMockStream(); + + const result = await handler.applyBackgroundEffect(stream); + + expect(result.applied).toBe(true); + expect(mockController.start).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + mode: 'blur', + }), + ); + expect(mockController.setMode).not.toHaveBeenCalled(); + expect(mockController.setBlurStrength).not.toHaveBeenCalled(); + }); + + it('applies virtual background successfully', async () => { + const {loadBackgroundSource} = require('Repositories/media/VideoBackgroundEffects'); + + loadBackgroundSource.mockResolvedValue('mock-bg'); + + const handler = new BackgroundEffectsHandler(mockController); + + handler.setPreferredBackgroundEffect({ + type: 'virtual', + backgroundId: 'bg1', + }); + + const outputTrack = {stop: jest.fn()}; + + mockController.start.mockResolvedValue({ + outputTrack, + stop: jest.fn(), + }); + + const stream = createMockStream(); + + const result = await handler.applyBackgroundEffect(stream); + + expect(result.applied).toBe(true); + expect(loadBackgroundSource).toHaveBeenCalledWith('bg1'); + expect(mockController.start).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + mode: 'virtual', + }), + ); + expect(mockController.setMode).not.toHaveBeenCalled(); + expect(mockController.setBackgroundSource).not.toHaveBeenCalled(); + }); + + it('handles controller error gracefully', async () => { + const handler = new BackgroundEffectsHandler(mockController); + + handler.setPreferredBackgroundEffect({type: 'blur', level: 'high'} as any); + + mockController.start.mockRejectedValue(new Error('fail')); + + const stream = createMockStream(); + + const result = await handler.applyBackgroundEffect(stream); + + expect(result.applied).toBe(false); + expect(mockController.stop).toHaveBeenCalled(); + }); + + it('falls back to default virtual when custom has no source', () => { + const handler = new BackgroundEffectsHandler(mockController); + + handler.setPreferredBackgroundEffect({type: 'custom'} as any); + + expect(backgroundEffectsStore.getState().preferredEffect.type).toBe('virtual'); + }); + + it('saves feature flag to storage', () => { + const handler = new BackgroundEffectsHandler(mockController); + + handler.saveFeatureEnabledStateInStore(true); + + expect(mockStorage.setItem).toHaveBeenCalledWith('video-background-effects-feature-enabled', 'true'); + expect(backgroundEffectsStore.getState().isFeatureEnabled).toBe(true); + }); + + it('reads feature flag from storage', () => { + mockStorage.getItem.mockReturnValue('true'); + + new BackgroundEffectsHandler(mockController); + + expect(backgroundEffectsStore.getState().isFeatureEnabled).toBe(true); + }); +}); diff --git a/apps/webapp/src/script/repositories/media/backgroundEffectsHandler.ts b/apps/webapp/src/script/repositories/media/backgroundEffectsHandler.ts new file mode 100644 index 00000000000..0eea8cdfa73 --- /dev/null +++ b/apps/webapp/src/script/repositories/media/backgroundEffectsHandler.ts @@ -0,0 +1,284 @@ +/* + * Wire + * Copyright (C) 2026 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {Metrics, QualityMode} from 'Repositories/media/backgroundEffects'; +import {CapabilityInfo} from 'Repositories/media/backgroundEffects/backgroundEffectsWorkerTypes'; +import {BackgroundEffectsController} from 'Repositories/media/backgroundEffects/effects/backgroundEffectsController'; +import { + BackgroundEffectSelection, + BackgroundSource, + BLUR_STRENGTHS, + DEFAULT_BACKGROUND_EFFECT, + DEFAULT_BUILTIN_BACKGROUND_ID, + loadBackgroundSource, +} from 'Repositories/media/VideoBackgroundEffects'; +import {getStorage} from 'Util/localStorage'; +import {getLogger, Logger} from 'Util/logger'; + +import {backgroundEffectsStore, RenderMetrics} from './useBackgroundEffectsStore'; + +export const TARGET_FPS = 15; +export const DEBOUNCE_TIMER = 500; + +const VIDEO_BACKGROUND_EFFECT_STORAGE_KEY = 'video-background-effects'; +const VIDEO_BACKGROUND_EFFECTS_FEATURE_STORAGE_KEY = 'video-background-effects-feature-enabled'; + +export class BackgroundEffectsHandler { + private readonly logger: Logger = getLogger('BackgroundEffectsHandler'); + private readonly storage: Storage | undefined; + private customBackground: BackgroundSource | undefined = undefined; + private saveDebounceTimer: ReturnType | null = null; + private currentReleasableStream: ReleasableMediaStream | undefined = undefined; + + constructor(private readonly controller: BackgroundEffectsController) { + this.storage = getStorage(); + backgroundEffectsStore.getState().setIsFeatureEnabled(this.readFeatureEnabledStateFromStore()); + backgroundEffectsStore.getState().setPreferredEffect(this.readPreferredBackgroundEffectFromStore()); + + backgroundEffectsStore.subscribe((state, prevState) => { + if (state.preferredEffect !== prevState.preferredEffect) { + if (this.saveDebounceTimer) { + clearTimeout(this.saveDebounceTimer); + } + this.saveDebounceTimer = setTimeout( + () => this.savePreferredBackgroundEffectInStore(state.preferredEffect), + DEBOUNCE_TIMER, + ); + } + }); + } + + public async applyBackgroundEffect( + originalVideoStream: MediaStream, + ): Promise<{applied: boolean; media: ReleasableMediaStream}> { + const {preferredEffect} = backgroundEffectsStore.getState(); + + if (preferredEffect.type === 'none') { + return {applied: false, media: new ReleasableMediaStream(originalVideoStream)}; + } + + const videoTrack = originalVideoStream.getVideoTracks()[0]; + + if (!videoTrack) { + return {applied: false, media: new ReleasableMediaStream(originalVideoStream)}; + } + + const isVirtual = preferredEffect.type === 'virtual' || preferredEffect.type === 'custom'; + const blurStrength = preferredEffect.type === 'blur' ? BLUR_STRENGTHS[preferredEffect.level] : BLUR_STRENGTHS.high; + + const backgroundSource: BackgroundSource | undefined = isVirtual + ? await this.loadBackgroundSource(preferredEffect) + : undefined; + + // If the pipeline is already running, update its parameters in-place and return the same stream object. + if (this.controller.isProcessing() && this.currentReleasableStream) { + if (isVirtual) { + this.controller.setMode('virtual'); + if (backgroundSource) { + this.controller.setBackgroundSource(backgroundSource); + } + } else { + this.controller.setMode('blur'); + this.controller.setBlurStrength(blurStrength); + } + return {applied: true, media: this.currentReleasableStream}; + } + + try { + const {outputTrack, stop} = await this.controller.start(videoTrack, { + mode: isVirtual ? 'virtual' : 'blur', + blurStrength, + quality: 'auto', + targetFps: TARGET_FPS, + debugMode: 'off', + ...(isVirtual && backgroundSource ? {backgroundImage: backgroundSource} : {}), + onMetrics: (metrics: Metrics) => this.onMetrics(metrics), + onModelChange: (model: string) => this.onModelChange(model), + }); + const processedStream = new MediaStream([outputTrack]); + this.currentReleasableStream = new ReleasableMediaStream(processedStream, () => { + this.currentReleasableStream = undefined; + stop(); + outputTrack.stop(); + }); + } catch (error) { + await this.controller.stop(); + this.logger.warn('BackgroundEffectsController failed with error:', error); + return {applied: false, media: new ReleasableMediaStream(originalVideoStream)}; + } + + return {applied: true, media: this.currentReleasableStream!}; + } + + public setPreferredBackgroundEffect(effect: BackgroundEffectSelection, customBackground?: BackgroundSource) { + backgroundEffectsStore.getState().setPreferredEffect(effect); + if (effect.type === 'custom') { + if (!customBackground) { + backgroundEffectsStore + .getState() + .setPreferredEffect({type: 'virtual', backgroundId: DEFAULT_BUILTIN_BACKGROUND_ID}); + this.logger.warn('No custom background image was set, switch to default virtual background'); + } + this.customBackground = customBackground; + } + } + + /** + * Load virtual or custom background + * + * @param effect BackgroundEffectSelection + * @private + */ + private async loadBackgroundSource(effect: BackgroundEffectSelection): Promise { + let backgroundSource: BackgroundSource | undefined = undefined; + try { + if (effect.type === 'virtual') { + backgroundSource = await loadBackgroundSource(effect.backgroundId); + } else if (effect.type === 'custom') { + if (!this.customBackground) { + this.logger.warn('Failed to load custom background source'); + } + backgroundSource = this.customBackground; + } + } catch (error) { + this.logger.warn('Failed to load background source', error); + } + + return backgroundSource; + } + + public isBackgroundEffectEnabled(): boolean { + const {isFeatureEnabled, preferredEffect} = backgroundEffectsStore.getState(); + return isFeatureEnabled && preferredEffect.type !== 'none'; + } + + public readFeatureEnabledStateFromStore(): boolean { + if (this.storage === undefined) { + return false; + } + + try { + const isEnabled = this.storage.getItem(VIDEO_BACKGROUND_EFFECTS_FEATURE_STORAGE_KEY); + return isEnabled === 'true'; + } catch (error) { + console.error('Failed to read video background effect feature state', error); + return false; + } + } + + public saveFeatureEnabledStateInStore(flag: boolean): boolean { + if (this.storage === undefined) { + backgroundEffectsStore.getState().setIsFeatureEnabled(flag); + return false; + } + + try { + this.storage.setItem(VIDEO_BACKGROUND_EFFECTS_FEATURE_STORAGE_KEY, `${flag}`); + } catch (error) { + console.error('Failed to persisted video background effect feature state', error); + backgroundEffectsStore.getState().setIsFeatureEnabled(flag); + return false; + } + backgroundEffectsStore.getState().setIsFeatureEnabled(flag); + return flag; + } + + private readPreferredBackgroundEffectFromStore(): BackgroundEffectSelection { + if (this.storage === undefined) { + return DEFAULT_BACKGROUND_EFFECT; + } + + try { + const stored = this.storage.getItem(VIDEO_BACKGROUND_EFFECT_STORAGE_KEY); + if (stored === null) { + return DEFAULT_BACKGROUND_EFFECT; + } + + const parsed = JSON.parse(stored); + + if (!parsed?.type) { + return DEFAULT_BACKGROUND_EFFECT; + } + + return parsed as BackgroundEffectSelection; + } catch (error) { + console.error('Failed to read persisted preferred video background effect', error); + return DEFAULT_BACKGROUND_EFFECT; + } + } + + private savePreferredBackgroundEffectInStore(effect: BackgroundEffectSelection): void { + if (this.storage === undefined) { + return; + } + + try { + const serialized = JSON.stringify(effect); + this.storage.setItem(VIDEO_BACKGROUND_EFFECT_STORAGE_KEY, serialized); + } catch (error) { + this.logger.error('Failed to persist preferred video background effect', error); + } + } + + public applyQuality(quality: QualityMode) { + this.controller.setQuality(quality); + } + + public getQuality(): QualityMode { + return this.controller.getQuality(); + } + + public enableSuperhighQualityTier(enable: boolean) { + if (enable) { + this.controller.setMaxQualityTier('superhigh'); + } else { + this.controller.setMaxQualityTier('high'); + } + } + + public isSuperhighQualityTierAllowed(): boolean { + return this.controller.getMaxQualityTier() === 'superhigh'; + } + + getCapabilityInfo(): CapabilityInfo { + return this.controller.getCapabilityInfo(); + } + + private onMetrics(metrics: Metrics): void { + const budget = 1000 / TARGET_FPS; + const total = metrics.avgTotalMs || 0; + const utilShare = budget > 0 ? Math.min(999, (total / budget) * 100) : 0; + const mlShare = total > 0 ? (metrics.avgSegmentationMs / total) * 100 : 0; + const webglShare = total > 0 ? (metrics.avgGpuMs / total) * 100 : 0; + const ml = metrics.segmentationDelegate ? `ML(${metrics.segmentationDelegate})` : 'ML'; + const renderMetrics = {...metrics, webglShare, utilShare, mlShare, budget, ml} as RenderMetrics; + backgroundEffectsStore.getState().setMetrics(renderMetrics); + } + private onModelChange(modelPath: string): void { + const model = modelPath.split('/').pop(); + backgroundEffectsStore.getState().setModel(model); + } +} + +export class ReleasableMediaStream { + constructor( + public stream: MediaStream, + public release: () => void = () => null, + ) {} +} diff --git a/apps/webapp/src/script/repositories/media/useBackgroundEffectsStore.ts b/apps/webapp/src/script/repositories/media/useBackgroundEffectsStore.ts new file mode 100644 index 00000000000..98101da6b9e --- /dev/null +++ b/apps/webapp/src/script/repositories/media/useBackgroundEffectsStore.ts @@ -0,0 +1,78 @@ +/* + * Wire + * Copyright (C) 2026 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + */ + +import {useStore} from 'zustand'; +import {immer} from 'zustand/middleware/immer'; +import {createStore} from 'zustand/vanilla'; + +import {Metrics, QualityTier} from 'Repositories/media/backgroundEffects'; +import {BackgroundEffectSelection, DEFAULT_BACKGROUND_EFFECT} from 'Repositories/media/VideoBackgroundEffects'; + +export interface RenderMetrics extends Metrics { + budget: number; + utilShare: number; + mlShare: number; + webglShare: number; + ml: 'ML(CPU)' | 'ML(GPU)' | 'ML'; + tier: QualityTier; +} + +export type BackgroundEffectsState = { + isFeatureEnabled: boolean; + preferredEffect: BackgroundEffectSelection; + metrics: RenderMetrics | undefined; + model: string; + + setIsFeatureEnabled(value: boolean): void; + setPreferredEffect(effect: BackgroundEffectSelection): void; + setMetrics(metrics: RenderMetrics | undefined): void; + setModel(model: string | undefined): void; +}; + +export const backgroundEffectsStore = createStore()( + immer(set => ({ + isFeatureEnabled: false, + preferredEffect: DEFAULT_BACKGROUND_EFFECT, + metrics: undefined, + model: 'unknown', + + setIsFeatureEnabled: value => + set(state => { + state.isFeatureEnabled = value; + }), + + setPreferredEffect: effect => + set(state => { + state.preferredEffect = effect; + }), + + setMetrics: metrics => + set(state => { + state.metrics = metrics; + }), + + setModel: model => + set(state => { + state.model = model; + }), + })), +); + +export const useBackgroundEffectsStore = (selector: (state: BackgroundEffectsState) => T): T => + useStore(backgroundEffectsStore, selector); diff --git a/apps/webapp/src/script/util/debugUtil.ts b/apps/webapp/src/script/util/debugUtil.ts index 20b462e344e..fb786765f5e 100644 --- a/apps/webapp/src/script/util/debugUtil.ts +++ b/apps/webapp/src/script/util/debugUtil.ts @@ -248,10 +248,6 @@ export class DebugUtil { this.apiClient.transport.http.toggleGzip(enabled); } - enableCameraBlur(flag: boolean) { - return this.callingRepository.switchVideoBackgroundBlur(flag); - } - reconnectWebSocket({dryRun} = {dryRun: false}) { const teamFeatures = this.teamState.teamFeatures(); const useAsyncNotificationStream = @@ -289,6 +285,14 @@ export class DebugUtil { ); } + isVideoBackgroundEffectsFeatureEnabled(): boolean { + return this.callingRepository.getBackgroundEffectsHandler().readFeatureEnabledStateFromStore(); + } + + enableVideoBackgroundEffectsFeature(flag: boolean) { + return this.callingRepository.getBackgroundEffectsHandler().saveFeatureEnabledStateInStore(flag); + } + setupAvsDebugger() { if (this.isEnabledAvsDebugger()) { this.enableAvsDebugger(true); diff --git a/apps/webapp/src/script/util/emojiUtil.ts b/apps/webapp/src/script/util/emojiUtil.ts index ce5d93839ad..50558880cf0 100644 --- a/apps/webapp/src/script/util/emojiUtil.ts +++ b/apps/webapp/src/script/util/emojiUtil.ts @@ -58,14 +58,16 @@ Object.keys(emojiesList).forEach(key => { return; } - const emojiObject = emojiValue[0]; - const emojiNames = emojiObject.n; + if (emojiValue[0] !== undefined) { + const emojiObject = emojiValue[0]; + const emojiNames = emojiObject.n; - // Replace hyphens with spaces, but only if not followed by a number - // example- thumbs down emoji name is -1 - const formattedEmojiName = emojiNames[0].replace(/-(?![0-9])/g, ' '); + // Replace hyphens with spaces, but only if not followed by a number + // example- thumbs down emoji name is -1 + const formattedEmojiName = emojiNames[0].replace(/-(?![0-9])/g, ' '); - emojiDictionary.set(key, formattedEmojiName); + emojiDictionary.set(key, formattedEmojiName); + } }); // Function to get the emoji without skintone modifiers diff --git a/apps/webapp/src/types/i18n.d.ts b/apps/webapp/src/types/i18n.d.ts index a482e03e15c..c4a949b16bc 100644 --- a/apps/webapp/src/types/i18n.d.ts +++ b/apps/webapp/src/types/i18n.d.ts @@ -2036,8 +2036,28 @@ declare module 'I18n/en-US.json' { 'verify.headline': `You’ve got mail`; 'verify.resendCode': `Resend code`; 'verify.subhead': `Enter the six-digit verification code we sent to{newline}{email}`; + 'videoCallBackgroundAdd': `Add background...`; + 'videoCallBackgroundBlurHigh': `High`; + 'videoCallBackgroundBlurLow': `Low`; + 'videoCallBackgroundBlurSectionLabel': `Blur`; + 'videoCallBackgroundEffectsLabel': `Background Settings`; + 'videoCallBackgroundEnableHighQualityBlur': `Enable high quality blur`; + 'videoCallBackgroundNoEffect': `No background effect`; + 'videoCallBackgroundNone': `None`; + 'videoCallBackgroundOffice1': `Office 1`; + 'videoCallBackgroundOffice2': `Office 2`; + 'videoCallBackgroundOffice3': `Office 3`; + 'videoCallBackgroundOffice4': `Office 4`; + 'videoCallBackgroundOffice5': `Office 5`; + 'videoCallBackgroundSettings': `Background Settings`; + 'videoCallBackgroundUpload': `Upload background`; + 'videoCallBackgroundVirtual': `Virtual Background`; + 'videoCallBackgroundVirtualSectionLabel': `Virtual Backgrounds`; + 'videoCallBackgroundWire1': `Wire 1`; + 'videoCallBackgroundsLabel': `Backgrounds`; 'videoCallMenuMoreAddReaction': `Add reaction`; 'videoCallMenuMoreAudioSettings': `Audio Settings`; + 'videoCallMenuMoreCameraSettings': `Camera Settings`; 'videoCallMenuMoreChangeView': `Change view`; 'videoCallMenuMoreCloseReactions': `Close reactions`; 'videoCallMenuMoreHideParticipants': `Hide participants`; diff --git a/apps/webapp/webpack.config.common.js b/apps/webapp/webpack.config.common.js index 24079dc294c..a909b14eedd 100644 --- a/apps/webapp/webpack.config.common.js +++ b/apps/webapp/webpack.config.common.js @@ -149,7 +149,7 @@ module.exports = { }, { loader: 'raw-loader', - test: /\.glsl$/, + test: /\.(glsl|vert|frag)$/, }, { test: /\.less$/i, diff --git a/libraries/config/src/client.config.ts b/libraries/config/src/client.config.ts index 99633001d1a..f7193d904fc 100644 --- a/libraries/config/src/client.config.ts +++ b/libraries/config/src/client.config.ts @@ -81,7 +81,7 @@ export function generateConfig(params: ConfigGeneratorParams, env: Env) { ENABLE_SSO: env.FEATURE_ENABLE_SSO == 'true', ENFORCE_CONSTANT_BITRATE: env.FEATURE_ENFORCE_CONSTANT_BITRATE == 'true', ENABLE_ENCRYPTION_AT_REST: env.FEATURE_ENABLE_ENCRYPTION_AT_REST == 'true', - ENABLE_BLUR_BACKGROUND: env.FEATURE_ENABLE_BLUR_BACKGROUND == 'true', + MULTICLASS_MODEL_PATH: env.FEATURE_MULTICLASS_MODEL_PATH, ENABLE_MESSAGE_FORMAT_BUTTONS: env.FEATURE_ENABLE_MESSAGE_FORMAT_BUTTONS == 'true', ENABLE_VIRTUALIZED_MESSAGES_LIST: env.FEATURE_ENABLE_VIRTUALIZED_MESSAGES_LIST == 'true', ENABLE_PRESS_SPACE_TO_UNMUTE: env.FEATURE_ENABLE_PRESS_SPACE_TO_UNMUTE == 'true', diff --git a/libraries/config/src/env.ts b/libraries/config/src/env.ts index 2c8c4a19bbe..270afa73d9b 100644 --- a/libraries/config/src/env.ts +++ b/libraries/config/src/env.ts @@ -113,8 +113,8 @@ export type Env = { /** Feature toggle for advanced filters */ FEATURE_ENABLE_ADVANCED_FILTERS: string; - /** Feature toggle to blur the background during video call */ - FEATURE_ENABLE_BLUR_BACKGROUND: string; + /** Optional override path for the multiclass segmentation model */ + FEATURE_MULTICLASS_MODEL_PATH: string; /** Feature toggle for debug utils. Can be set to true or false */ FEATURE_ENABLE_DEBUG: string; diff --git a/nx.json b/nx.json index 7ddf3f93908..bea2e81a2de 100644 --- a/nx.json +++ b/nx.json @@ -3,24 +3,24 @@ "defaultBase": "dev", "targetDefaults": { "build": { - "dependsOn": ["^build"], // build all upstream dependencies first + "dependsOn": ["^build"], "cache": true, - "inputs": ["production"], // skip test-only files when hashing builds + "inputs": ["production"], "outputs": ["{projectRoot}/dist"] }, "test": { - "dependsOn": ["^build"], // build all upstream dependencies first + "dependsOn": ["^build"], "cache": true, - "inputs": ["default", "^production", "{workspaceRoot}/jest.preset.js"] // include shared preset in cache key + "inputs": ["default", "^production", "{workspaceRoot}/jest.preset.js"] }, "lint": { - "dependsOn": ["^build"], // build all upstream dependencies first + "dependsOn": ["^build"], "cache": true, - "inputs": ["default", "{workspaceRoot}/eslint.config.ts"] // workspace lint config in hash + "inputs": ["default", "{workspaceRoot}/eslint.config.ts"] } }, "namedInputs": { - "default": ["{projectRoot}/**/*", "sharedGlobals"], // everything under the project by default + "default": ["{projectRoot}/**/*", "sharedGlobals"], "production": [ "default", "!{projectRoot}/**/?(*.)+(spec|test).[jt]s?(x)?(.snap)", @@ -30,7 +30,7 @@ "!{projectRoot}/src/test-setup.[jt]s", "!{projectRoot}/test-setup.[jt]s" ], - "sharedGlobals": [] // hook for root-level shared files if needed + "sharedGlobals": [] }, "defaultProject": "webapp", "neverConnectToCloud": true, @@ -52,5 +52,6 @@ "createRelease": "github" } } - } + }, + "analytics": false }