VIDEO-11239 Add Krisp Noise Cancellation (#750)

* add krisp
* krisp implementation
* ui
* update UI
* add marginRight to MuiSwitch for theme
* move krisp state to useappstate
* remove css from settings button on desktop
* remove some padding from intro container
* refactor device selection screen for noise cancellation
* add tool tip
* krisp logo and disabling track ui
* refactor device selection screen
* add krisp to device selection dialog
* use krisp toggle useeffect
* remove browser supporession
* move toggle back to original spot
* polish css
* remove krisp sdk
* move isKrispInstalled to useLocalTracks hook
* remove unnecessary useEffect in useKrispToggle
* fix switch button logic
* fix flicker in UI for unsupported browsers
* VIDEO-11239 Changes to make noise cancellation work.
* VIDEO-11239 Updating twilio-video and CHANGELOG.md.
* VIDEO-11239 UI changes.
* VIDEO-11239 Prep for 0.8.0.
* VIDEO-11239 Get current unit tests to pass.
* VIDEO-11239 Updating twilio-video to 2.24.2.
* VIDEO-11239 Make Krisp dependency optional.
* VIDEO-11239 Fixing unit tests.
* VIDEO-11239 s/suppression/cancellation/g in CHANGELOG.md.
* VIDEO-11239 Update README section for noise cancellation.

Co-authored-by: mmalavalli <mmalavalli@twilio.com>

.circleci/config.yml

@@ -14,7 +14,11 @@ jobs:
             - v1-deps-{{ .Branch }}
             - v1-deps
 
-      - run: npm ci
+      - run:
+          name: 'Install Dependencies'
+          command: |
+            npm ci
+            npm run noisecancellation:krisp
 
       - save_cache:
           key: v1-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}

.gitignore

@@ -26,10 +26,12 @@ yarn-debug.log*
 yarn-error.log*
 
 .env
+.env.*
 .vscode
 
 test-reports
 junit.xml
 serviceAccountKey.json
 
+public/noisecancellation/
 public/virtualbackground/

CHANGELOG.md

@@ -1,9 +1,19 @@
+## 0.9.0 (November 22, 2022)
+
+### New Features
+
+- Krisp audio noise cancellation has been added. [#750](https://github.com/twilio/twilio-video-app-react/pull/750)
+
 ## 0.8.0 (November 14, 2022)
 
 ### New Feature
 
 - This release adds the ability to maintain audio continuity when the default audio input device changes. If the user chooses a specific audio device from the audio settings, then this feature does not apply.
 
+### Dependency Changes
+
+- `twilio-video` has been upgraded from 2.23.0 to 2.25.0.
+
 ## 0.7.1 (August 5, 2022)
 
 ### Dependency Upgrades

README.md

@@ -39,6 +39,10 @@ Run `npm install` inside the main project folder to install all dependencies fro
 
 If you want to use `yarn` to install dependencies, first run the [yarn import](https://classic.yarnpkg.com/en/docs/cli/import/) command. This will ensure that yarn installs the package versions that are specified in `package-lock.json`.
 
+### Add Noise Cancellation
+
+Twilio Video has partnered with [Krisp Technologies Inc.](https://krisp.ai/) to add [noise cancellation](https://www.twilio.com/docs/video/noise-cancellation) to the local audio track. This feature is licensed under the [Krisp Plugin for Twilio](https://twilio.github.io/krisp-audio-plugin/LICENSE.html). In order to add this feature to your application, please run `npm install noisecancellation:krisp` immediately after the [previous step](#install-dependencies).
+
 ## Install Twilio CLI and RTC Plugin
 
 ### Install the Twilio CLI

package.json

@@ -86,6 +86,7 @@
   },
   "scripts": {
     "postinstall": "rimraf public/virtualbackground && copyfiles -f node_modules/@twilio/video-processors/dist/build/* public/virtualbackground",
+    "noisecancellation:krisp": "npm install @twilio/krisp-audio-plugin && rimraf public/noisecancellation && copyfiles -f \"node_modules/@twilio/krisp-audio-plugin/dist/*\" public/noisecancellation && copyfiles -f \"node_modules/@twilio/krisp-audio-plugin/dist/weights/*\" public/noisecancellation/weights",
     "start": "concurrently npm:server npm:dev",
     "dev": "react-scripts start",
     "build": "node ./scripts/build.js",

server/__tests__/createExpressHandler.test.ts

@@ -1,4 +1,5 @@
 /* eslint-disable import/first */
+process.env.REACT_APP_TWILIO_ENVIRONMENT = 'prod';
 process.env.TWILIO_ACCOUNT_SID = 'mockAccountSid';
 process.env.TWILIO_API_KEY_SID = 'mockApiKeySid';
 process.env.TWILIO_API_KEY_SECRET = 'mockApiKeySecret';

src/components/DeviceSelectionDialog/AudioInputList/AudioInputList.tsx

@@ -12,9 +12,10 @@ export default function AudioInputList() {
   const { localTracks } = useVideoContext();
 
   const localAudioTrack = localTracks.find(track => track.kind === 'audio') as LocalAudioTrack;
+  const srcMediaStreamTrack = localAudioTrack?.noiseCancellation?.sourceTrack;
   const mediaStreamTrack = useMediaStreamTrack(localAudioTrack);
-  const localAudioInputDeviceId = mediaStreamTrack?.getSettings().deviceId;
-
+  const localAudioInputDeviceId =
+    srcMediaStreamTrack?.getSettings().deviceId || mediaStreamTrack?.getSettings().deviceId;
   function replaceTrack(newDeviceId: string) {
     window.localStorage.setItem(SELECTED_AUDIO_INPUT_KEY, newDeviceId);
     localAudioTrack?.restart({ deviceId: { exact: newDeviceId } });

src/components/DeviceSelectionDialog/DeviceSelectionDialog.tsx

@@ -12,10 +12,19 @@ import {
   Theme,
   DialogTitle,
   Hidden,
+  FormControlLabel,
+  Switch,
+  Tooltip,
 } from '@material-ui/core';
 import { makeStyles } from '@material-ui/core/styles';
 import VideoInputList from './VideoInputList/VideoInputList';
 import MaxGalleryViewParticipants from './MaxGalleryViewParticipants/MaxGalleryViewParticipants';
+import { useKrispToggle } from '../../hooks/useKrispToggle/useKrispToggle';
+import SmallCheckIcon from '../../icons/SmallCheckIcon';
+import InfoIconOutlined from '../../icons/InfoIconOutlined';
+import KrispLogo from '../../icons/KrispLogo';
+import { useAppState } from '../../state';
+import useVideoContext from '../../hooks/useVideoContext/useVideoContext';
 
 const useStyles = makeStyles((theme: Theme) => ({
   container: {
@@ -46,9 +55,28 @@ const useStyles = makeStyles((theme: Theme) => ({
       margin: '1em 0 2em 0',
     },
   },
+  noiseCancellationContainer: {
+    display: 'flex',
+    justifyContent: 'space-between',
+  },
+  krispContainer: {
+    display: 'flex',
+    alignItems: 'center',
+    '& svg': {
+      '&:not(:last-child)': {
+        margin: '0 0.3em',
+      },
+    },
+  },
+  krispInfoText: {
+    margin: '0 0 1.5em 0.5em',
+  },
 }));
 
 export default function DeviceSelectionDialog({ open, onClose }: { open: boolean; onClose: () => void }) {
+  const { isAcquiringLocalTracks } = useVideoContext();
+  const { isKrispEnabled, isKrispInstalled } = useAppState();
+  const { toggleKrisp } = useKrispToggle();
   const classes = useStyles();
 
   return (
@@ -67,6 +95,45 @@ export default function DeviceSelectionDialog({ open, onClose }: { open: boolean
           <Typography variant="h6" className={classes.headline}>
             Audio
           </Typography>
+
+          {isKrispInstalled && (
+            <div className={classes.noiseCancellationContainer}>
+              <div className={classes.krispContainer}>
+                <Typography variant="subtitle2">Noise Cancellation powered by </Typography>
+                <KrispLogo />
+                <Tooltip
+                  title="Suppress background noise from your microphone"
+                  interactive
+                  leaveDelay={250}
+                  leaveTouchDelay={15000}
+                  enterTouchDelay={0}
+                >
+                  <div>
+                    <InfoIconOutlined />
+                  </div>
+                </Tooltip>
+              </div>
+              <FormControlLabel
+                control={
+                  <Switch
+                    checked={!!isKrispEnabled}
+                    checkedIcon={<SmallCheckIcon />}
+                    disableRipple={true}
+                    onClick={toggleKrisp}
+                  />
+                }
+                label={isKrispEnabled ? 'Enabled' : 'Disabled'}
+                style={{ marginRight: 0 }}
+                disabled={isAcquiringLocalTracks}
+              />
+            </div>
+          )}
+          {isKrispInstalled && (
+            <Typography variant="body1" color="textSecondary" className={classes.krispInfoText}>
+              Suppress background noise from your microphone.
+            </Typography>
+          )}
+
           <AudioInputList />
         </div>
         <div className={classes.listSection}>

src/components/IntroContainer/IntroContainer.tsx

@@ -73,7 +73,7 @@ const useStyles = makeStyles((theme: Theme) => ({
   content: {
     background: 'white',
     width: '100%',
-    padding: '4em',
+    padding: '3em 4em',
     flex: 1,
     [theme.breakpoints.down('sm')]: {
       padding: '2em',

src/components/MenuBar/Menu/Menu.test.tsx

@@ -20,7 +20,10 @@ import useLocalVideoToggle from '../../../hooks/useLocalVideoToggle/useLocalVide
 jest.mock('../../../hooks/useFlipCameraToggle/useFlipCameraToggle');
 jest.mock('@material-ui/core/useMediaQuery');
 jest.mock('../../../state');
-jest.mock('../../../hooks/useVideoContext/useVideoContext', () => () => ({ room: { sid: 'mockRoomSid' } }));
+jest.mock('../../../hooks/useVideoContext/useVideoContext', () => () => ({
+  localTracks: [],
+  room: { sid: 'mockRoomSid' },
+}));
 jest.mock('../../../hooks/useIsRecording/useIsRecording');
 jest.mock('../../../hooks/useChatContext/useChatContext');
 jest.mock('../../../hooks/useLocalVideoToggle/useLocalVideoToggle');

src/components/PreJoinScreens/DeviceSelectionScreen/DeviceSelectionScreen.test.tsx

@@ -25,6 +25,7 @@ mockUseVideoContext.mockImplementation(() => ({
   connect: mockConnect,
   isAcquiringLocalTracks: false,
   isConnecting: false,
+  localTracks: [],
 }));
 
 describe('the DeviceSelectionScreen component', () => {
@@ -38,6 +39,7 @@ describe('the DeviceSelectionScreen component', () => {
       connect: mockConnect,
       isAcquiringLocalTracks: false,
       isConnecting: true,
+      localTracks: [],
     }));
 
     const wrapper = shallow(<DeviceSelectionScreen name="test name" roomName="test room name" setStep={() => {}} />);
@@ -60,6 +62,7 @@ describe('the DeviceSelectionScreen component', () => {
       connect: mockConnect,
       isAcquiringLocalTracks: true,
       isConnecting: false,
+      localTracks: [],
     }));
 
     const wrapper = shallow(<DeviceSelectionScreen name="test name" roomName="test room name" setStep={() => {}} />);
@@ -82,6 +85,7 @@ describe('the DeviceSelectionScreen component', () => {
       connect: mockConnect,
       isAcquiringLocalTracks: false,
       isConnecting: false,
+      localTracks: [],
     }));
     mockUseAppState.mockImplementationOnce(() => ({ getToken: mockGetToken, isFetching: true }));
     const wrapper = shallow(<DeviceSelectionScreen name="test name" roomName="test room name" setStep={() => {}} />);

src/components/PreJoinScreens/DeviceSelectionScreen/DeviceSelectionScreen.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
-import { makeStyles, Typography, Grid, Button, Theme, Hidden } from '@material-ui/core';
+import { makeStyles, Typography, Grid, Button, Theme, Hidden, Switch, Tooltip } from '@material-ui/core';
 import CircularProgress from '@material-ui/core/CircularProgress';
+import Divider from '@material-ui/core/Divider';
 import LocalVideoPreview from './LocalVideoPreview/LocalVideoPreview';
 import SettingsMenu from './SettingsMenu/SettingsMenu';
 import { Steps } from '../PreJoinScreens';
@@ -9,6 +10,10 @@ import ToggleVideoButton from '../../Buttons/ToggleVideoButton/ToggleVideoButton
 import { useAppState } from '../../../state';
 import useChatContext from '../../../hooks/useChatContext/useChatContext';
 import useVideoContext from '../../../hooks/useVideoContext/useVideoContext';
+import FormControlLabel from '@material-ui/core/FormControlLabel';
+import { useKrispToggle } from '../../../hooks/useKrispToggle/useKrispToggle';
+import SmallCheckIcon from '../../../icons/SmallCheckIcon';
+import InfoIconOutlined from '../../../icons/InfoIconOutlined';
 
 const useStyles = makeStyles((theme: Theme) => ({
   gutterBottom: {
@@ -24,6 +29,7 @@ const useStyles = makeStyles((theme: Theme) => ({
   },
   localPreviewContainer: {
     paddingRight: '2em',
+    marginBottom: '2em',
     [theme.breakpoints.down('sm')]: {
       padding: '0 2.5em',
     },
@@ -50,6 +56,17 @@ const useStyles = makeStyles((theme: Theme) => ({
     padding: '0.8em 0',
     margin: 0,
   },
+  toolTipContainer: {
+    display: 'flex',
+    alignItems: 'center',
+    '& div': {
+      display: 'flex',
+      alignItems: 'center',
+    },
+    '& svg': {
+      marginLeft: '0.3em',
+    },
+  },
 }));
 
 interface DeviceSelectionScreenProps {
@@ -60,9 +77,10 @@ interface DeviceSelectionScreenProps {
 
 export default function DeviceSelectionScreen({ name, roomName, setStep }: DeviceSelectionScreenProps) {
   const classes = useStyles();
-  const { getToken, isFetching } = useAppState();
+  const { getToken, isFetching, isKrispEnabled, isKrispInstalled } = useAppState();
   const { connect: chatConnect } = useChatContext();
   const { connect: videoConnect, isAcquiringLocalTracks, isConnecting } = useVideoContext();
+  const { toggleKrisp } = useKrispToggle();
   const disableButtons = isFetching || isAcquiringLocalTracks || isConnecting;
 
   const handleJoin = () => {
@@ -102,32 +120,89 @@ export default function DeviceSelectionScreen({ name, roomName, setStep }: Devic
             <Hidden mdUp>
               <ToggleAudioButton className={classes.mobileButton} disabled={disableButtons} />
               <ToggleVideoButton className={classes.mobileButton} disabled={disableButtons} />
+              <SettingsMenu mobileButtonClass={classes.mobileButton} />
             </Hidden>
-            <SettingsMenu mobileButtonClass={classes.mobileButton} />
           </div>
         </Grid>
         <Grid item md={5} sm={12} xs={12}>
-          <Grid container direction="column" justifyContent="space-between" style={{ height: '100%' }}>
+          <Grid container direction="column" justifyContent="space-between" style={{ alignItems: 'normal' }}>
             <div>
               <Hidden smDown>
                 <ToggleAudioButton className={classes.deviceButton} disabled={disableButtons} />
                 <ToggleVideoButton className={classes.deviceButton} disabled={disableButtons} />
               </Hidden>
             </div>
-            <div className={classes.joinButtons}>
-              <Button variant="outlined" color="primary" onClick={() => setStep(Steps.roomNameStep)}>
-                Cancel
-              </Button>
-              <Button
-                variant="contained"
-                color="primary"
-                data-cy-join-now
-                onClick={handleJoin}
-                disabled={disableButtons}
-              >
-                Join Now
-              </Button>
-            </div>
+          </Grid>
+        </Grid>
+
+        <Grid item md={12} sm={12} xs={12}>
+          {isKrispInstalled && (
+            <Grid
+              container
+              direction="row"
+              justifyContent="space-between"
+              alignItems="center"
+              style={{ marginBottom: '1em' }}
+            >
+              <div className={classes.toolTipContainer}>
+                <Typography variant="subtitle2">Noise Cancellation</Typography>
+                <Tooltip
+                  title="Suppress background noise from your microphone"
+                  interactive
+                  leaveDelay={250}
+                  leaveTouchDelay={15000}
+                  enterTouchDelay={0}
+                >
+                  <div>
+                    <InfoIconOutlined />
+                  </div>
+                </Tooltip>
+              </div>
+
+              <FormControlLabel
+                control={
+                  <Switch
+                    checked={!!isKrispEnabled}
+                    checkedIcon={<SmallCheckIcon />}
+                    disableRipple={true}
+                    onClick={toggleKrisp}
+                  />
+                }
+                label={isKrispEnabled ? 'Enabled' : 'Disabled'}
+                style={{ marginRight: 0 }}
+                // Prevents <Switch /> from being temporarily enabled (and then quickly disabled) in unsupported browsers after
+                // isAcquiringLocalTracks becomes false:
+                disabled={isKrispEnabled && isAcquiringLocalTracks}
+              />
+            </Grid>
+          )}
+          <Divider />
+        </Grid>
+
+        <Grid item md={12} sm={12} xs={12}>
+          <Grid container direction="row" alignItems="center" style={{ marginTop: '1em' }}>
+            <Hidden smDown>
+              <Grid item md={7} sm={12} xs={12}>
+                <SettingsMenu mobileButtonClass={classes.mobileButton} />
+              </Grid>
+            </Hidden>
+
+            <Grid item md={5} sm={12} xs={12}>
+              <div className={classes.joinButtons}>
+                <Button variant="outlined" color="primary" onClick={() => setStep(Steps.roomNameStep)}>
+                  Cancel
+                </Button>
+                <Button
+                  variant="contained"
+                  color="primary"
+                  data-cy-join-now
+                  onClick={handleJoin}
+                  disabled={disableButtons}
+                >
+                  Join Now
+                </Button>
+              </div>
+            </Grid>
           </Grid>
         </Grid>
       </Grid>