From da482d5404292eea7aeabdc32c3d0ea70191118f Mon Sep 17 00:00:00 2001 From: Manjesh Malavalli Date: Fri, 30 Jun 2023 13:57:10 -0700 Subject: [PATCH] VIDEO-12945 Upgrade to twilio-video@2.27.0 and @twilio/video-processors@2.0.0 (#808) * VIDEO-12945 Upgrade to twilio-video@2.27.0 and @twilio/video-processors@2.0.0. * VIDEO-12945 Disable debounce for Chrome, and switch to 480p when background substitution is turned on only for desktop Chrome and browsers not supporting WebAssembly SIMD. --- CHANGELOG.md | 9 +- package-lock.json | 333 ++++++------------ package.json | 4 +- .../useBackgroundSettings.test.tsx | 60 +++- .../useBackgroundSettings.ts | 42 ++- src/constants.ts | 6 + 6 files changed, 205 insertions(+), 249 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbacf5c26..57201299d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ +## 0.10.0 + +### Changes + +- `twilio-video` has been upgraded to 2.27.0. +- `@twilio/video-processors` has been upgraded to 2.0.0, which uses GPU where possible to improve the user experience of virtual backgrounds. + ## 0.9.1 (January 10, 2023) -## Changes +### Changes - Modified CircleCI deploy script to store gcloud service account credentials in proper JSON format. diff --git a/package-lock.json b/package-lock.json index b1b91ac03..4d4d3dd6d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "@material-ui/lab": "^4.0.0-alpha.60", "@twilio-labs/plugin-rtc": "^0.8.4", "@twilio/conversations": "^2.1.0", - "@twilio/video-processors": "^1.0.1", + "@twilio/video-processors": "^2.0.0", "@twilio/video-room-monitor": "^1.0.1", "@types/d3-timer": "^1.0.9", "@types/dotenv": "^8.2.0", @@ -51,7 +51,7 @@ "swiper": "^8.1.5", "ts-node": "^9.1.1", "twilio": "^3.63.1", - "twilio-video": "^2.25.0", + "twilio-video": "^2.27.0", "typescript": "^4.6.2" }, "devDependencies": { @@ -6991,8 +6991,15 @@ } }, "node_modules/@storybook/addon-controls/node_modules/watchpack/chokidar2": { + "version": "2.0.0", "dev": true, - "optional": true + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } }, "node_modules/@storybook/addon-controls/node_modules/webpack": { "version": "4.43.0", @@ -8021,8 +8028,15 @@ } }, "node_modules/@storybook/addon-docs/node_modules/watchpack/chokidar2": { + "version": "2.0.0", "dev": true, - "optional": true + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } }, "node_modules/@storybook/addon-docs/node_modules/webpack": { "version": "4.43.0", @@ -8972,8 +8986,15 @@ } }, "node_modules/@storybook/addon-essentials/node_modules/watchpack/chokidar2": { + "version": "2.0.0", "dev": true, - "optional": true + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } }, "node_modules/@storybook/addon-essentials/node_modules/webpack": { "version": "4.43.0", @@ -11065,8 +11086,15 @@ } }, "node_modules/@storybook/builder-webpack4/node_modules/watchpack/chokidar2": { + "version": "2.0.0", "dev": true, - "optional": true + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } }, "node_modules/@storybook/builder-webpack4/node_modules/webpack": { "version": "4.43.0", @@ -13808,8 +13836,15 @@ } }, "node_modules/@storybook/core-server/node_modules/webpack/node_modules/watchpack/chokidar2": { + "version": "2.0.0", "dev": true, - "optional": true + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } }, "node_modules/@storybook/core-server/node_modules/y18n": { "version": "4.0.3", @@ -15511,8 +15546,15 @@ } }, "node_modules/@storybook/manager-webpack4/node_modules/watchpack/chokidar2": { + "version": "2.0.0", "dev": true, - "optional": true + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } }, "node_modules/@storybook/manager-webpack4/node_modules/webpack": { "version": "4.43.0", @@ -17343,8 +17385,15 @@ } }, "node_modules/@storybook/react/node_modules/watchpack/chokidar2": { + "version": "2.0.0", "dev": true, - "optional": true + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } }, "node_modules/@storybook/react/node_modules/webpack": { "version": "4.43.0", @@ -18404,8 +18453,15 @@ } }, "node_modules/@storybook/telemetry/node_modules/watchpack/chokidar2": { + "version": "2.0.0", "dev": true, - "optional": true + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } }, "node_modules/@storybook/telemetry/node_modules/webpack": { "version": "4.43.0", @@ -18760,76 +18816,6 @@ "node": ">=10" } }, - "node_modules/@tensorflow-models/body-pix": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-2.2.0.tgz", - "integrity": "sha512-e8PEf8xWlWhXJtGvlTQpdRyUuDXKAFJT/f8SWiBfvim5BAUyFp2dTeVV3gjDDp8MUEo7T3lofTZUexy/aG4YEA==", - "peerDependencies": { - "@tensorflow/tfjs-backend-webgl": "^3.6.0", - "@tensorflow/tfjs-converter": "^3.6.0", - "@tensorflow/tfjs-core": "^3.6.0" - } - }, - "node_modules/@tensorflow/tfjs-backend-cpu": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.18.0.tgz", - "integrity": "sha512-LcSqlylzGtpgngcMFIL3q9Q3eVaPRJ7ITZt7ivhzkCj4R5ZsnPa9qM3DCVihkQ77heAwSw4hPTo2jp5C4mJ4Cg==", - "dependencies": { - "@types/seedrandom": "2.4.27", - "seedrandom": "2.4.3" - }, - "engines": { - "yarn": ">= 1.3.2" - }, - "peerDependencies": { - "@tensorflow/tfjs-core": "3.18.0" - } - }, - "node_modules/@tensorflow/tfjs-backend-webgl": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.18.0.tgz", - "integrity": "sha512-3NknSzS1oX2BEBOrpjPMZl823S12RgshQthmIbG6QADHb4bCJA8aM4UjWpw+3bNQnRKbRDQdFbuvj10Un79s2A==", - "dependencies": { - "@tensorflow/tfjs-backend-cpu": "3.18.0", - "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "2.4.27", - "@types/webgl-ext": "0.0.30", - "@types/webgl2": "0.0.6", - "seedrandom": "2.4.3" - }, - "engines": { - "yarn": ">= 1.3.2" - }, - "peerDependencies": { - "@tensorflow/tfjs-core": "3.18.0" - } - }, - "node_modules/@tensorflow/tfjs-converter": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.18.0.tgz", - "integrity": "sha512-hpChA+zVNQOVwRnCfqDb1WI9jbEAKA6DuEm4m75Zb3dIlE6VVooDmAaHBhlc++z2q2G1sBzF9A4Bv48SUpN6vA==", - "peerDependencies": { - "@tensorflow/tfjs-core": "3.18.0" - } - }, - "node_modules/@tensorflow/tfjs-core": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.18.0.tgz", - "integrity": "sha512-gMxisZozqsr5sCKlphF/eVBLg91MjlBiN60tjX8hJAu0WlSn6Gi5k65GNIL+Pq6hrxpvImcfdCmTH/2XJVZ0Mg==", - "dependencies": { - "@types/long": "^4.0.1", - "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "2.4.27", - "@types/webgl-ext": "0.0.30", - "@webgpu/types": "^0.1.16", - "long": "4.0.0", - "node-fetch": "~2.6.1", - "seedrandom": "2.4.3" - }, - "engines": { - "yarn": ">= 1.3.2" - } - }, "node_modules/@testing-library/dom": { "version": "6.16.0", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-6.16.0.tgz", @@ -19250,16 +19236,9 @@ } }, "node_modules/@twilio/video-processors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@twilio/video-processors/-/video-processors-1.0.2.tgz", - "integrity": "sha512-61pLa5sC8ndetzlyJCLl77kV9UtMcnkmAN52V+hSvVtrSmNj1Z9t22NthymmViYQcebyUAtw3+IiGHzSHBFYNw==", - "dependencies": { - "@tensorflow-models/body-pix": "^2.1.0", - "@tensorflow/tfjs-backend-cpu": "^3.3.0", - "@tensorflow/tfjs-backend-webgl": "^3.2.0", - "@tensorflow/tfjs-converter": "^3.2.0", - "@tensorflow/tfjs-core": "^3.2.0" - }, + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@twilio/video-processors/-/video-processors-2.0.0.tgz", + "integrity": "sha512-JbcSe032zppy11LXKV3WXHqSmntTvu5oEspZwyiWcjR5DbjJ4+SnQc6zK3iUaoMOER2r6k+295cwCiLftAlA5Q==", "engines": { "node": ">=14" } @@ -19814,11 +19793,6 @@ "integrity": "sha512-WKG4gTr8przEZBiJ5r3s8ZIAoMXNbOgQ+j/d5O4X3x6kZJRLNvyUJuUK/KoG3+8BaOHPhp2m7WC6JKKeovDSzQ==", "dev": true }, - "node_modules/@types/offscreencanvas": { - "version": "2019.3.0", - "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", - "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==" - }, "node_modules/@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -19954,11 +19928,6 @@ "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" }, - "node_modules/@types/seedrandom": { - "version": "2.4.27", - "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz", - "integrity": "sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==" - }, "node_modules/@types/serve-index": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.1.tgz", @@ -20142,16 +20111,6 @@ "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==", "dev": true }, - "node_modules/@types/webgl-ext": { - "version": "0.0.30", - "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", - "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==" - }, - "node_modules/@types/webgl2": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.6.tgz", - "integrity": "sha512-50GQhDVTq/herLMiqSQkdtRu+d5q/cWHn4VvKJtrj4DJAjo1MNkWYa2MA41BaBO1q1HgsUjuQvEOk0QHvlnAaQ==" - }, "node_modules/@types/webpack": { "version": "4.41.32", "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.41.32.tgz", @@ -21023,11 +20982,6 @@ "@xtuc/long": "4.2.2" } }, - "node_modules/@webgpu/types": { - "version": "0.1.21", - "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.21.tgz", - "integrity": "sha512-pUrWq3V5PiSGFLeLxoGqReTZmiiXwY3jRkIG5sLLKjyqNxrwm/04b4nw7LSmGWJcKk59XOM/YRTUwOzo4MMlow==" - }, "node_modules/@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", @@ -43795,11 +43749,6 @@ "resolved": "https://registry.npmjs.org/scmp/-/scmp-2.1.0.tgz", "integrity": "sha512-o/mRQGk9Rcer/jEEw/yw4mwo3EU/NvYvp577/Btqrym9Qy5/MdWGBqipbALgd2lrdWTJ5/gqDusxfnQBxOxT2Q==" }, - "node_modules/seedrandom": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz", - "integrity": "sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==" - }, "node_modules/select-hose": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", @@ -46651,9 +46600,9 @@ } }, "node_modules/twilio-video": { - "version": "2.25.0", - "resolved": "https://registry.npmjs.org/twilio-video/-/twilio-video-2.25.0.tgz", - "integrity": "sha512-6bfGBmBRB7qXDQHEjKutFc2eRv7PhXZ3znf84pE0nCNQuMOGWT95EAaZvfobQpMR8ZqnZsgO/1KuBYcwW0NrdA==", + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/twilio-video/-/twilio-video-2.27.0.tgz", + "integrity": "sha512-hdiYOKuujtLdG7A8Ae/8qeJWfL/zZXVTrh79fWq7d/QAgZ966R7QJdWkTAxdairhlhGLCu4cN4jOBxQ1Bqe9MQ==", "dependencies": { "events": "^3.3.0", "util": "^0.12.4", @@ -54736,7 +54685,10 @@ } }, "chokidar2": { - "version": "file:node_modules/@storybook/addon-docs/node_modules/watchpack/chokidar2" + "version": "file:node_modules/@storybook/addon-docs/node_modules/watchpack/chokidar2", + "requires": { + "chokidar": "^2.1.8" + } }, "enhanced-resolve": { "version": "4.5.0", @@ -55471,7 +55423,10 @@ } }, "chokidar2": { - "version": "file:node_modules/@storybook/addon-essentials/node_modules/watchpack/chokidar2" + "version": "file:node_modules/@storybook/addon-essentials/node_modules/watchpack/chokidar2", + "requires": { + "chokidar": "^2.1.8" + } }, "enhanced-resolve": { "version": "4.5.0", @@ -57021,7 +56976,10 @@ } }, "chokidar2": { - "version": "file:node_modules/@storybook/builder-webpack4/node_modules/watchpack/chokidar2" + "version": "file:node_modules/@storybook/builder-webpack4/node_modules/watchpack/chokidar2", + "requires": { + "chokidar": "^2.1.8" + } }, "cosmiconfig": { "version": "6.0.0", @@ -59385,7 +59343,10 @@ } }, "chokidar2": { - "version": "file:node_modules/@storybook/core-server/node_modules/webpack/node_modules/watchpack/chokidar2" + "version": "file:node_modules/@storybook/core-server/node_modules/webpack/node_modules/watchpack/chokidar2", + "requires": { + "chokidar": "^2.1.8" + } }, "enhanced-resolve": { "version": "4.5.0", @@ -60481,7 +60442,10 @@ } }, "chokidar2": { - "version": "file:node_modules/@storybook/manager-webpack4/node_modules/watchpack/chokidar2" + "version": "file:node_modules/@storybook/manager-webpack4/node_modules/watchpack/chokidar2", + "requires": { + "chokidar": "^2.1.8" + } }, "enhanced-resolve": { "version": "4.5.0", @@ -61779,7 +61743,10 @@ } }, "chokidar2": { - "version": "file:node_modules/@storybook/react/node_modules/watchpack/chokidar2" + "version": "file:node_modules/@storybook/react/node_modules/watchpack/chokidar2", + "requires": { + "chokidar": "^2.1.8" + } }, "enhanced-resolve": { "version": "4.5.0", @@ -62705,7 +62672,10 @@ } }, "chokidar2": { - "version": "file:node_modules/@storybook/telemetry/node_modules/watchpack/chokidar2" + "version": "file:node_modules/@storybook/telemetry/node_modules/watchpack/chokidar2", + "requires": { + "chokidar": "^2.1.8" + } }, "enhanced-resolve": { "version": "4.5.0", @@ -63184,55 +63154,6 @@ "defer-to-connect": "^2.0.0" } }, - "@tensorflow-models/body-pix": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-2.2.0.tgz", - "integrity": "sha512-e8PEf8xWlWhXJtGvlTQpdRyUuDXKAFJT/f8SWiBfvim5BAUyFp2dTeVV3gjDDp8MUEo7T3lofTZUexy/aG4YEA==", - "requires": {} - }, - "@tensorflow/tfjs-backend-cpu": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.18.0.tgz", - "integrity": "sha512-LcSqlylzGtpgngcMFIL3q9Q3eVaPRJ7ITZt7ivhzkCj4R5ZsnPa9qM3DCVihkQ77heAwSw4hPTo2jp5C4mJ4Cg==", - "requires": { - "@types/seedrandom": "2.4.27", - "seedrandom": "2.4.3" - } - }, - "@tensorflow/tfjs-backend-webgl": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.18.0.tgz", - "integrity": "sha512-3NknSzS1oX2BEBOrpjPMZl823S12RgshQthmIbG6QADHb4bCJA8aM4UjWpw+3bNQnRKbRDQdFbuvj10Un79s2A==", - "requires": { - "@tensorflow/tfjs-backend-cpu": "3.18.0", - "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "2.4.27", - "@types/webgl-ext": "0.0.30", - "@types/webgl2": "0.0.6", - "seedrandom": "2.4.3" - } - }, - "@tensorflow/tfjs-converter": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.18.0.tgz", - "integrity": "sha512-hpChA+zVNQOVwRnCfqDb1WI9jbEAKA6DuEm4m75Zb3dIlE6VVooDmAaHBhlc++z2q2G1sBzF9A4Bv48SUpN6vA==", - "requires": {} - }, - "@tensorflow/tfjs-core": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.18.0.tgz", - "integrity": "sha512-gMxisZozqsr5sCKlphF/eVBLg91MjlBiN60tjX8hJAu0WlSn6Gi5k65GNIL+Pq6hrxpvImcfdCmTH/2XJVZ0Mg==", - "requires": { - "@types/long": "^4.0.1", - "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "2.4.27", - "@types/webgl-ext": "0.0.30", - "@webgpu/types": "^0.1.16", - "long": "4.0.0", - "node-fetch": "~2.6.1", - "seedrandom": "2.4.3" - } - }, "@testing-library/dom": { "version": "6.16.0", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-6.16.0.tgz", @@ -63572,16 +63493,9 @@ } }, "@twilio/video-processors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@twilio/video-processors/-/video-processors-1.0.2.tgz", - "integrity": "sha512-61pLa5sC8ndetzlyJCLl77kV9UtMcnkmAN52V+hSvVtrSmNj1Z9t22NthymmViYQcebyUAtw3+IiGHzSHBFYNw==", - "requires": { - "@tensorflow-models/body-pix": "^2.1.0", - "@tensorflow/tfjs-backend-cpu": "^3.3.0", - "@tensorflow/tfjs-backend-webgl": "^3.2.0", - "@tensorflow/tfjs-converter": "^3.2.0", - "@tensorflow/tfjs-core": "^3.2.0" - } + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@twilio/video-processors/-/video-processors-2.0.0.tgz", + "integrity": "sha512-JbcSe032zppy11LXKV3WXHqSmntTvu5oEspZwyiWcjR5DbjJ4+SnQc6zK3iUaoMOER2r6k+295cwCiLftAlA5Q==" }, "@twilio/video-room-monitor": { "version": "1.0.1", @@ -64100,11 +64014,6 @@ "integrity": "sha512-WKG4gTr8przEZBiJ5r3s8ZIAoMXNbOgQ+j/d5O4X3x6kZJRLNvyUJuUK/KoG3+8BaOHPhp2m7WC6JKKeovDSzQ==", "dev": true }, - "@types/offscreencanvas": { - "version": "2019.3.0", - "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", - "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==" - }, "@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -64242,11 +64151,6 @@ "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" }, - "@types/seedrandom": { - "version": "2.4.27", - "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz", - "integrity": "sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==" - }, "@types/serve-index": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.1.tgz", @@ -64417,16 +64321,6 @@ "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==", "dev": true }, - "@types/webgl-ext": { - "version": "0.0.30", - "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", - "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==" - }, - "@types/webgl2": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.6.tgz", - "integrity": "sha512-50GQhDVTq/herLMiqSQkdtRu+d5q/cWHn4VvKJtrj4DJAjo1MNkWYa2MA41BaBO1q1HgsUjuQvEOk0QHvlnAaQ==" - }, "@types/webpack": { "version": "4.41.32", "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.41.32.tgz", @@ -65144,11 +65038,6 @@ "@xtuc/long": "4.2.2" } }, - "@webgpu/types": { - "version": "0.1.21", - "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.21.tgz", - "integrity": "sha512-pUrWq3V5PiSGFLeLxoGqReTZmiiXwY3jRkIG5sLLKjyqNxrwm/04b4nw7LSmGWJcKk59XOM/YRTUwOzo4MMlow==" - }, "@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", @@ -67145,7 +67034,10 @@ } }, "chokidar2": { - "version": "file:node_modules/@storybook/addon-controls/node_modules/watchpack/chokidar2" + "version": "file:node_modules/@storybook/addon-controls/node_modules/watchpack/chokidar2", + "requires": { + "chokidar": "^2.1.8" + } }, "chownr": { "version": "1.1.4", @@ -82336,11 +82228,6 @@ "resolved": "https://registry.npmjs.org/scmp/-/scmp-2.1.0.tgz", "integrity": "sha512-o/mRQGk9Rcer/jEEw/yw4mwo3EU/NvYvp577/Btqrym9Qy5/MdWGBqipbALgd2lrdWTJ5/gqDusxfnQBxOxT2Q==" }, - "seedrandom": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz", - "integrity": "sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==" - }, "select-hose": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", @@ -84549,9 +84436,9 @@ } }, "twilio-video": { - "version": "2.25.0", - "resolved": "https://registry.npmjs.org/twilio-video/-/twilio-video-2.25.0.tgz", - "integrity": "sha512-6bfGBmBRB7qXDQHEjKutFc2eRv7PhXZ3znf84pE0nCNQuMOGWT95EAaZvfobQpMR8ZqnZsgO/1KuBYcwW0NrdA==", + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/twilio-video/-/twilio-video-2.27.0.tgz", + "integrity": "sha512-hdiYOKuujtLdG7A8Ae/8qeJWfL/zZXVTrh79fWq7d/QAgZ966R7QJdWkTAxdairhlhGLCu4cN4jOBxQ1Bqe9MQ==", "requires": { "events": "^3.3.0", "util": "^0.12.4", diff --git a/package.json b/package.json index 9dd1fe958..c96d35137 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "@material-ui/lab": "^4.0.0-alpha.60", "@twilio-labs/plugin-rtc": "^0.8.4", "@twilio/conversations": "^2.1.0", - "@twilio/video-processors": "^1.0.1", + "@twilio/video-processors": "^2.0.0", "@twilio/video-room-monitor": "^1.0.1", "@types/d3-timer": "^1.0.9", "@types/dotenv": "^8.2.0", @@ -45,7 +45,7 @@ "swiper": "^8.1.5", "ts-node": "^9.1.1", "twilio": "^3.63.1", - "twilio-video": "^2.25.0", + "twilio-video": "^2.27.0", "typescript": "^4.6.2" }, "devDependencies": { diff --git a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx index 166572e5e..57508bbbb 100644 --- a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx +++ b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx @@ -102,10 +102,16 @@ describe('The useBackgroundSettings hook ', () => { }); backgroundSettings = renderResult.current[0]; expect(backgroundSettings.type).toEqual('blur'); - expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith({ - loadModel: mockLoadModel, - name: 'GaussianBlurBackgroundProcessor', - }); + expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith( + { + loadModel: mockLoadModel, + name: 'GaussianBlurBackgroundProcessor', + }, + { + inputFrameBufferType: 'video', + outputFrameBufferContextType: 'webgl2', + } + ); }); it('should set the background settings correctly and remove the video processor when "none" is selected', async () => { @@ -127,11 +133,17 @@ describe('The useBackgroundSettings hook ', () => { backgroundSettings = renderResult.current[0]; expect(backgroundSettings.type).toEqual('image'); expect(backgroundSettings.index).toEqual(2); - expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith({ - backgroundImage: expect.any(Object), - loadModel: mockLoadModel, - name: 'VirtualBackgroundProcessor', - }); + expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith( + { + backgroundImage: expect.any(Object), + loadModel: mockLoadModel, + name: 'VirtualBackgroundProcessor', + }, + { + inputFrameBufferType: 'video', + outputFrameBufferContextType: 'webgl2', + } + ); }); describe('The setBackgroundSettings function ', () => { @@ -165,10 +177,16 @@ describe('The useBackgroundSettings hook ', () => { await act(async () => { setBackgroundSettings(imgSettings); }); - expect(mockVideoTrack.addProcessor).not.toHaveBeenCalledWith({ - loadModel: mockLoadModel, - name: 'GaussianBlurBackgroundProcessor', - }); + expect(mockVideoTrack.addProcessor).not.toHaveBeenCalledWith( + { + loadModel: mockLoadModel, + name: 'GaussianBlurBackgroundProcessor', + }, + { + inputFrameBufferType: 'video', + outputFrameBufferContextType: 'webgl2', + } + ); expect(window.localStorage.getItem(SELECTED_BACKGROUND_SETTINGS_KEY)).toEqual(JSON.stringify(imgSettings)); }); @@ -177,11 +195,17 @@ describe('The useBackgroundSettings hook ', () => { await act(async () => { setBackgroundSettings(blurSettings); }); - expect(mockVideoTrack.addProcessor).not.toHaveBeenCalledWith({ - loadModel: mockLoadModel, - backgroundImage: expect.any(Object), - name: 'VirtualBackgroundProcessor', - }); + expect(mockVideoTrack.addProcessor).not.toHaveBeenCalledWith( + { + loadModel: mockLoadModel, + backgroundImage: expect.any(Object), + name: 'VirtualBackgroundProcessor', + }, + { + inputFrameBufferType: 'video', + outputFrameBufferContextType: 'webgl2', + } + ); expect(window.localStorage.getItem(SELECTED_BACKGROUND_SETTINGS_KEY)).toEqual(JSON.stringify(blurSettings)); }); diff --git a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts index e08270494..14660d226 100644 --- a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts +++ b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts @@ -1,11 +1,15 @@ import { LocalVideoTrack, Room } from 'twilio-video'; -import { useEffect, useCallback } from 'react'; -import { SELECTED_BACKGROUND_SETTINGS_KEY } from '../../../constants'; +import { useCallback, useEffect } from 'react'; +import { + BACKGROUND_FILTER_VIDEO_CONSTRAINTS, + DEFAULT_VIDEO_CONSTRAINTS, + SELECTED_BACKGROUND_SETTINGS_KEY, +} from '../../../constants'; import { GaussianBlurBackgroundProcessor, - VirtualBackgroundProcessor, ImageFit, isSupported, + VirtualBackgroundProcessor, } from '@twilio/video-processors'; import Abstract from '../../../images/Abstract.jpg'; import AbstractThumb from '../../../images/thumb/Abstract.jpg'; @@ -104,6 +108,7 @@ const rawImagePaths = [ SanFrancisco, ]; +const isDesktopChrome = /Chrome/.test(navigator.userAgent); let imageElements = new Map(); const getImage = (index: number): Promise => { @@ -136,6 +141,16 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde { type: 'none', index: 0 } ); + const setCaptureConstraints = useCallback(async () => { + const { mediaStreamTrack, processor } = videoTrack ?? {}; + const { type } = backgroundSettings; + if (type === 'none' && processor) { + return mediaStreamTrack?.applyConstraints(DEFAULT_VIDEO_CONSTRAINTS as MediaTrackConstraints); + } else if (type !== 'none' && !processor) { + return mediaStreamTrack?.applyConstraints(BACKGROUND_FILTER_VIDEO_CONSTRAINTS as MediaTrackConstraints); + } + }, [backgroundSettings, videoTrack]); + const removeProcessor = useCallback(() => { if (videoTrack && videoTrack.processor) { videoTrack.removeProcessor(videoTrack.processor); @@ -148,7 +163,10 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde return; } removeProcessor(); - videoTrack.addProcessor(processor); + videoTrack.addProcessor(processor, { + inputFrameBufferType: 'video', + outputFrameBufferContextType: 'webgl2', + }); }, [videoTrack, removeProcessor] ); @@ -163,6 +181,9 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde if (!blurProcessor) { blurProcessor = new GaussianBlurBackgroundProcessor({ assetsPath: virtualBackgroundAssets, + // Disable debounce only on desktop Chrome as other browsers either + // do not support WebAssembly SIMD or they degrade performance. + debounce: !isDesktopChrome, }); await blurProcessor.loadModel(); } @@ -170,6 +191,9 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde virtualBackgroundProcessor = new VirtualBackgroundProcessor({ assetsPath: virtualBackgroundAssets, backgroundImage: await getImage(0), + // Disable debounce only on desktop Chrome as other browsers either + // do not support WebAssembly SIMD or they degrade performance. + debounce: !isDesktopChrome, fitType: ImageFit.Cover, }); await virtualBackgroundProcessor.loadModel(); @@ -178,6 +202,14 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde return; } + // Switch to 640x480 dimensions on desktop Chrome or browsers that + // do not support WebAssembly SIMD to achieve optimum performance. + const processor = blurProcessor || virtualBackgroundProcessor; + // @ts-ignore + if (!processor._isSimdEnabled || isDesktopChrome) { + await setCaptureConstraints(); + } + if (backgroundSettings.type === 'blur') { addProcessor(blurProcessor); } else if (backgroundSettings.type === 'image' && typeof backgroundSettings.index === 'number') { @@ -188,7 +220,7 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde } }; handleProcessorChange(); - }, [backgroundSettings, videoTrack, room, addProcessor, removeProcessor]); + }, [backgroundSettings, videoTrack, room, addProcessor, removeProcessor, setCaptureConstraints]); return [backgroundSettings, setBackgroundSettings] as const; } diff --git a/src/constants.ts b/src/constants.ts index bfc833cfe..036e8c89e 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,3 +1,9 @@ +export const BACKGROUND_FILTER_VIDEO_CONSTRAINTS: MediaStreamConstraints['video'] = { + width: 640, + height: 480, + frameRate: 24, +}; + export const DEFAULT_VIDEO_CONSTRAINTS: MediaStreamConstraints['video'] = { width: 1280, height: 720,