feat: background effects [wpb-23253]

.gitattributes

@@ -13,3 +13,8 @@
 *.ttf binary
 *.woff binary
 *.woff2 binary
+# Machine Learning Models
+*.tflite binary
+*.onnx binary
+*.pb binary
+*.pt binary

.ls-lint.yml

@@ -71,6 +71,12 @@ ls:
     .ts: camelCase
     .tsx: camelCase
     .test.tsx: camelCase
+  apps/webapp/src/script/repositories/media/backgroundEffects:
+    .dir: camelCase
+    .ts: camelCase
+    .tsx: camelCase
+    .test.tsx: camelCase
+    .types.ts: camelCase
   apps/webapp/src/script/repositories/assets:
     .dir: camelCase
     .ts: camelCase

apps/webapp/src/i18n/en-US.json

@@ -2032,8 +2032,28 @@
   "verify.headline": "You’ve got mail",
   "verify.resendCode": "Resend code",
   "verify.subhead": "Enter the six-digit verification code we sent to{newline}{email}",
+  "videoCallBackgroundAdd": "Add background...",
+  "videoCallBackgroundBlurHigh": "High",
+  "videoCallBackgroundBlurLow": "Low",
+  "videoCallBackgroundBlurSectionLabel": "Blur",
+  "videoCallBackgroundEffectsLabel": "Background Settings",
+  "videoCallBackgroundEnableHighQualityBlur": "Enable high quality blur",
+  "videoCallBackgroundNoEffect": "No background effect",
+  "videoCallBackgroundNone": "None",
+  "videoCallBackgroundOffice1": "Office 1",
+  "videoCallBackgroundOffice2": "Office 2",
+  "videoCallBackgroundOffice3": "Office 3",
+  "videoCallBackgroundOffice4": "Office 4",
+  "videoCallBackgroundOffice5": "Office 5",
+  "videoCallBackgroundSettings": "Background Settings",
+  "videoCallBackgroundUpload": "Upload background",
+  "videoCallBackgroundVirtual": "Virtual Background",
+  "videoCallBackgroundVirtualSectionLabel": "Virtual Backgrounds",
+  "videoCallBackgroundWire1": "Wire 1",
+  "videoCallBackgroundsLabel": "Backgrounds",
   "videoCallMenuMoreAddReaction": "Add reaction",
   "videoCallMenuMoreAudioSettings": "Audio Settings",
+  "videoCallMenuMoreCameraSettings": "Camera Settings",
   "videoCallMenuMoreChangeView": "Change view",
   "videoCallMenuMoreCloseReactions": "Close reactions",
   "videoCallMenuMoreHideParticipants": "Hide participants",

apps/webapp/src/script/auth/util/test/TestUtil.tsx

@@ -57,9 +57,12 @@ import sk from 'I18n/sk-SK.json';
 import sl from 'I18n/sl-SI.json';
 import tr from 'I18n/tr-TR.json';
 import uk from 'I18n/uk-UA.json';
+import {CallingRepository} from 'Repositories/calling/CallingRepository';
 import {Participant} from 'Repositories/calling/Participant';
 import {Conversation} from 'Repositories/entity/Conversation';
 import {User} from 'Repositories/entity/User';
+import {BackgroundEffectsController} from 'Repositories/media/backgroundEffects/effects/backgroundEffectsController';
+import {BackgroundEffectsHandler} from 'Repositories/media/backgroundEffectsHandler';
 import {MediaDevicesHandler} from 'Repositories/media/MediaDevicesHandler';
 import {setStrings} from 'Util/localizerUtil';
 import {createUuid} from 'Util/uuid';
@@ -190,3 +193,16 @@ export const buildMediaDevicesHandler = () => {
     setOnMediaDevicesRefreshHandler: jest.fn(),
   } as unknown as MediaDevicesHandler;
 };
+
+export const buildCallingRepository = () => {
+  const controller: BackgroundEffectsController = {
+    getQuality: jest.fn(),
+    getCapabilityInfo: jest.fn(),
+  } as unknown as BackgroundEffectsController;
+  const backgroundEffectsHandler = new BackgroundEffectsHandler(controller);
+
+  return {
+    getBackgroundEffectsHandler: () => backgroundEffectsHandler,
+    isSuperhighQualityTierAllowed: jest.fn(),
+  } as unknown as CallingRepository;
+};

apps/webapp/src/script/components/ConfigToolbar/ConfigToolbar.tsx

@@ -43,6 +43,9 @@
   const wrapperRef = useRef(null);
   const [avsDebuggerEnabled, setAvsDebuggerEnabled] = useState(!!window.wire?.app?.debug?.isEnabledAvsDebugger());
   const [avsRustSftEnabled, setAvsRustSftEnabled] = useState(!!window.wire?.app?.debug?.isEnabledAvsRustSFT());
+  const [videoBackgroundEffectsFeatureEnabled, setVideoBackgroundEffectsFeatureEnabled] = useState(
+    !!window.wire?.app?.debug?.isVideoBackgroundEffectsFeatureEnabled(),
+  );
   const [coreCryptoLevel, setCoreCryptoLevel] = useState<CoreCryptoLogLevel>(CoreCryptoLogLevel.Info);
 
   // Toggle config tool on 'cmd/ctrl + shift + 2'
@@ -219,6 +222,24 @@
     );
   };
 
+  const handleBackgroundEffectsFeature = (isChecked: boolean) => {
+    setVideoBackgroundEffectsFeatureEnabled(!!window.wire?.app?.debug?.enableVideoBackgroundEffectsFeature(isChecked));
+  };
+  const renderBackgroundEffectsFeatureSelect = () => {
+    return (
+      <div style={{marginBottom: '10px'}}>
+        <label htmlFor="video-background-effects-feature-checkbox" style={{display: 'block', fontWeight: 'bold'}}>
+          ENABLE VIDEO BACKGROUND EFFECTS
+        </label>
+        <Switch
+          id="video-background-effects-feature-checkbox"
+          checked={videoBackgroundEffectsFeatureEnabled}
+          onToggle={isChecked => handleBackgroundEffectsFeature(isChecked)}
+        />
+      </div>
+    );
+  };
+
   const renderGzipSwitch = () => {
     return (
       <div style={{marginBottom: '10px'}}>
@@ -318,6 +339,10 @@
 
       <hr />
 
+      <div>{renderBackgroundEffectsFeatureSelect()}</div>
+
+      <hr />
+
       <div>{renderGzipSwitch()}</div>
 
       <hr />

apps/webapp/src/script/components/calling/CallingOverlayContainer.tsx

@@ -170,7 +170,7 @@ const CallingContainer = ({
           switchCameraInput={switchCameraInput}
           switchMicrophoneInput={switchMicrophoneInput}
           switchSpeakerOutput={switchSpeakerOutput}
-          switchBlurredBackground={status => callingRepository.switchVideoBackgroundBlur(status)}
+          switchVideoBackgroundEffect={effect => callingRepository.switchVideoBackgroundEffect(effect)}
           setMaximizedParticipant={setMaximizedParticipant}
           setActiveCallViewTab={setActiveCallViewTab}
           toggleMute={toggleMute}

apps/webapp/src/script/components/calling/FullscreenVideoCall.tsx

@@ -17,26 +17,27 @@
  *
  */
 
-import React, {useEffect, useState} from 'react';
+import {ChangeEvent, useEffect, useState} from 'react';
 
 import {DefaultConversationRoleName} from '@wireapp/api-client/lib/conversation/';
 import cx from 'classnames';
 import {container} from 'tsyringe';
 
 import {
-  TabIndex,
   Checkbox,
   CheckboxLabel,
   CloseDetachedWindowIcon,
   IconButton,
   IconButtonVariant,
   OpenDetachedWindowIcon,
   QUERY,
+  TabIndex,
 } from '@wireapp/react-ui-kit';
 import {WebAppEvents} from '@wireapp/webapp-events';
 
 import {useAppNotification} from 'Components/AppNotification/AppNotification';
 import {useCallAlertState} from 'Components/calling/useCallAlertState';
+import {VideoBackgroundPerformancePanel} from 'Components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel';
 import {ConversationClassifiedBar} from 'Components/ClassifiedBar/ClassifiedBar';
 import * as Icon from 'Components/Icon';
 import {ModalComponent} from 'Components/Modals/ModalComponent';
@@ -47,6 +48,9 @@ import {Participant} from 'Repositories/calling/Participant';
 import type {Grid} from 'Repositories/calling/videoGridHandler';
 import type {Conversation} from 'Repositories/entity/Conversation';
 import {MediaDevicesHandler} from 'Repositories/media/MediaDevicesHandler';
+import {useBackgroundEffectsStore} from 'Repositories/media/useBackgroundEffectsStore';
+import type {BackgroundEffectSelection} from 'Repositories/media/VideoBackgroundEffects';
+import {BUILTIN_BACKGROUNDS} from 'Repositories/media/VideoBackgroundEffects';
 import {PropertiesRepository} from 'Repositories/properties/PropertiesRepository';
 import {TeamState} from 'Repositories/team/TeamState';
 import {useActiveWindowMatchMedia} from 'src/script/hooks/useActiveWindowMatchMedia';
@@ -63,14 +67,15 @@ import {Duration} from './Duration';
 import {
   classifiedBarStyles,
   headerActionsWrapperStyles,
-  paginationWrapperStyles,
-  videoTopBarStyles,
   minimizeButtonStyles,
   openDetachedWindowButtonStyles,
   paginationStyles,
+  paginationWrapperStyles,
+  videoTopBarStyles,
 } from './FullscreenVideoCall.styles';
 import {GroupVideoGrid} from './GroupVideoGrid';
 import {Pagination} from './Pagination/Pagination';
+import {VideoBackgroundSettings} from './VideoControls/VideoBackgroundSettings/VideoBackgroundSettings';
 import {VideoControls} from './VideoControls/VideoControls';
 
 import {useWarningsState} from '../../view_model/WarningsContainer/WarningsState';
@@ -95,7 +100,7 @@ export interface FullscreenVideoCallProps {
   switchCameraInput: (deviceId: string) => void;
   switchMicrophoneInput: (deviceId: string) => void;
   switchSpeakerOutput: (deviceId: string) => void;
-  switchBlurredBackground: (status: boolean) => void;
+  switchVideoBackgroundEffect: (effect: BackgroundEffectSelection) => void;
   teamState?: TeamState;
   callState?: CallState;
   toggleCamera: (call: Call) => void;
@@ -125,7 +130,7 @@ const FullscreenVideoCall = ({
   switchCameraInput,
   switchMicrophoneInput,
   switchSpeakerOutput,
-  switchBlurredBackground,
+  switchVideoBackgroundEffect,
   setMaximizedParticipant,
   setActiveCallViewTab,
   toggleMute,
@@ -189,6 +194,7 @@ const FullscreenVideoCall = ({
   const openPopup = () => callingRepository.setViewModeDetached();
 
   const [isParticipantsListOpen, toggleParticipantsList] = useToggleState(false);
+  const [isBackgroundSidebarOpen, setIsBackgroundSidebarOpen] = useState(false);
 
   const callNotification = useAppNotification({
     activeWindow: viewMode === CallingViewMode.DETACHED_WINDOW ? detachedWindow! : window,
@@ -276,6 +282,17 @@ const FullscreenVideoCall = ({
   const isPaginationVisible = !maximizedParticipant && activeCallViewTab === CallViewTab.ALL && totalPages > 1;
 
   const isModerator = selfUser && roles[selfUser.id] === DefaultConversationRoleName.WIRE_ADMIN;
+  const backgroundEffectsHandler = callingRepository.getBackgroundEffectsHandler();
+
+  const selectedBackgroundEffect = useBackgroundEffectsStore(state => state.preferredEffect);
+
+  const handleBackgroundSidebarSelect = (effect: BackgroundEffectSelection) => {
+    void switchVideoBackgroundEffect(effect);
+  };
+
+  const handleEnableHighQualityBlur = (event: ChangeEvent<HTMLInputElement>) => {
+    callingRepository.allowSuperhighQualityTier(event.target.checked);
+  };
 
   return (
     <div
@@ -395,6 +412,16 @@ const FullscreenVideoCall = ({
               onClose={toggleParticipantsList}
             />
           )}
+          {isMobile && isBackgroundSidebarOpen && (
+            <VideoBackgroundSettings
+              selectedEffect={selectedBackgroundEffect}
+              backgrounds={BUILTIN_BACKGROUNDS}
+              onSelectEffect={handleBackgroundSidebarSelect}
+              onEnableHighQualityBlur={handleEnableHighQualityBlur}
+              onClose={() => setIsBackgroundSidebarOpen(false)}
+              highQualityBlurAllowed={callingRepository.isSuperhighQualityTierAllowed()}
+            />
+          )}
         </div>
 
         {isMobile && isPaginationVisible && (
@@ -445,11 +472,12 @@ const FullscreenVideoCall = ({
               toggleIsHandRaised={toggleIsHandRaised}
               switchMicrophoneInput={switchMicrophoneInput}
               switchSpeakerOutput={switchSpeakerOutput}
-              switchBlurredBackground={switchBlurredBackground}
+              switchVideoBackgroundEffect={switchVideoBackgroundEffect}
               switchCameraInput={switchCameraInput}
               setActiveCallViewTab={setActiveCallViewTab}
               setMaximizedParticipant={setMaximizedParticipant}
               sendEmoji={sendEmoji}
+              onOpenBackgroundSettings={() => setIsBackgroundSidebarOpen(true)}
             />
           </>
         )}
@@ -466,6 +494,16 @@ const FullscreenVideoCall = ({
           onClose={toggleParticipantsList}
         />
       )}
+      {!isMobile && isBackgroundSidebarOpen && (
+        <VideoBackgroundSettings
+          selectedEffect={selectedBackgroundEffect}
+          backgrounds={BUILTIN_BACKGROUNDS}
+          onSelectEffect={handleBackgroundSidebarSelect}
+          onEnableHighQualityBlur={handleEnableHighQualityBlur}
+          onClose={() => setIsBackgroundSidebarOpen(false)}
+          highQualityBlurAllowed={callingRepository.isSuperhighQualityTierAllowed()}
+        />
+      )}
       <ModalComponent
         isShown={isConfirmCloseModalOpen}
         onClosed={() => setIsConfirmCloseModalOpen(false)}
@@ -490,7 +528,7 @@ const FullscreenVideoCall = ({
                 wrapperCSS={{marginTop: 16}}
                 data-uie-name="do-not-ask-again-checkbox"
                 id="do-not-ask-again-checkbox"
-                onChange={(event: React.ChangeEvent<HTMLInputElement>) =>
+                onChange={(event: ChangeEvent<HTMLInputElement>) =>
                   localStorage.setItem(
                     LOCAL_STORAGE_KEY_FOR_SCREEN_SHARING_CONFIRM_MODAL,
                     event.target.checked.toString(),
@@ -535,6 +573,7 @@ const FullscreenVideoCall = ({
           </>
         )}
       </ModalComponent>
+      <VideoBackgroundPerformancePanel backgroundEffectsHandler={backgroundEffectsHandler} />
     </div>
   );
 };

apps/webapp/src/script/components/calling/GroupVideoGrid.tsx

@@ -144,7 +144,7 @@ const GroupVideoGrid = ({
     'hasActiveVideo',
     'sharesScreen',
     'videoStream',
-    'blurredVideoStream',
+    'processedVideoStream',
   ]);
 
   const [rowsAndColumns, setRowsAndColumns] = useState<RowsAndColumns>(
@@ -287,7 +287,7 @@ const GroupVideoGrid = ({
             css={{
               transform: thumbnail.hasActiveVideo && !thumbnail.sharesScreen ? 'rotateY(180deg)' : 'initial',
             }}
-            srcObject={thumbnail.blurredVideoStream?.stream ?? thumbnail.videoStream}
+            srcObject={thumbnail.processedVideoStream?.stream ?? thumbnail.videoStream}
           />
           {selfIsMuted && !minimized && (
             <span className="group-video-grid__element__label__icon" data-uie-name="status-call-audio-muted">

apps/webapp/src/script/components/calling/GroupVideoGridTile.tsx

@@ -65,15 +65,15 @@ const GroupVideoGridTile = ({
     videoState,
     handRaisedAt,
     videoStream,
-    blurredVideoStream,
+    processedVideoStream,
     isActivelySpeaking,
     isAudioEstablished,
     isSwitchingVideoResolution,
   } = useKoSubscribableChildren(participant, [
     'isMuted',
     'handRaisedAt',
     'videoStream',
-    'blurredVideoStream',
+    'processedVideoStream',
     'isActivelySpeaking',
     'videoState',
     'isAudioEstablished',
@@ -142,7 +142,7 @@ const GroupVideoGridTile = ({
                see https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide.
             */
             muted
-            srcObject={blurredVideoStream?.stream ?? videoStream}
+            srcObject={processedVideoStream?.stream ?? videoStream}
             className="group-video-grid__element-video"
             css={groupVideoElementVideo(isMaximized || sharesScreen, participant === selfParticipant && sharesCamera)}
           />