Skip to content
This repository has been archived by the owner on Sep 11, 2024. It is now read-only.

Commit

Permalink
Prevent Element appearing in system media controls (#10995)
Browse files Browse the repository at this point in the history
* Use WebAudio API to play notification sound

So that it won't appear in system media control.

* Run prettier

* Chosse from mp3 and ogg

* Run prettier

* Use WebAudioAPI everywhere

There's still one remoteAudio. I'm not sure what it does. It seems it's
only used in tests...

* Run prettier

* Eliminate a stupid error

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update setupManualMocks.ts

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* delint

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* mocks

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* mocks

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Simplify

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* covg

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>
  • Loading branch information
3 people authored Jul 4, 2024
1 parent c61eca8 commit e288f61
Show file tree
Hide file tree
Showing 12 changed files with 255 additions and 164 deletions.
2 changes: 1 addition & 1 deletion jest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ const config: Config = {
testEnvironment: "jsdom",
testMatch: ["<rootDir>/test/**/*-test.[jt]s?(x)"],
globalSetup: "<rootDir>/test/globalSetup.ts",
setupFiles: ["jest-canvas-mock"],
setupFiles: ["jest-canvas-mock", "web-streams-polyfill/polyfill"],
setupFilesAfterEnv: ["<rootDir>/test/setupTests.ts"],
moduleNameMapper: {
"\\.(gif|png|ttf|woff2)$": "<rootDir>/__mocks__/imageMock.js",
Expand Down
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,8 @@
"stylelint-config-standard": "^36.0.0",
"stylelint-scss": "^6.0.0",
"ts-node": "^10.9.1",
"typescript": "5.5.2"
"typescript": "5.5.2",
"web-streams-polyfill": "^4.0.0"
},
"peerDependencies": {
"postcss": "^8.4.19",
Expand Down
130 changes: 26 additions & 104 deletions src/LegacyCallHandler.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ import { localNotificationsAreSilenced } from "./utils/notifications";
import { SdkContextClass } from "./contexts/SDKContext";
import { showCantStartACallDialog } from "./voice-broadcast/utils/showCantStartACallDialog";
import { isNotNull } from "./Typeguards";
import { BackgroundAudio } from "./audio/BackgroundAudio";

export const PROTOCOL_PSTN = "m.protocol.pstn";
export const PROTOCOL_PSTN_PREFIXED = "im.vector.protocol.pstn";
Expand Down Expand Up @@ -157,8 +158,6 @@ export default class LegacyCallHandler extends EventEmitter {
// Calls started as an attended transfer, ie. with the intention of transferring another
// call with a different party to this one.
private transferees = new Map<string, MatrixCall>(); // callId (target) -> call (transferee)
private audioPromises = new Map<AudioID, Promise<void>>();
private audioElementsWithListeners = new Map<HTMLMediaElement, boolean>();
private supportsPstnProtocol: boolean | null = null;
private pstnSupportPrefixed: boolean | null = null; // True if the server only support the prefixed pstn protocol
private supportsSipNativeVirtual: boolean | null = null; // im.vector.protocol.sip_virtual and im.vector.protocol.sip_native
Expand All @@ -170,6 +169,9 @@ export default class LegacyCallHandler extends EventEmitter {

private silencedCalls = new Set<string>(); // callIds

private backgroundAudio = new BackgroundAudio();
private playingSources: Record<string, AudioBufferSourceNode> = {}; // Record them for stopping

public static get instance(): LegacyCallHandler {
if (!window.mxLegacyCallHandler) {
window.mxLegacyCallHandler = new LegacyCallHandler();
Expand Down Expand Up @@ -199,61 +201,18 @@ export default class LegacyCallHandler extends EventEmitter {
}

public start(): void {
// add empty handlers for media actions, otherwise the media keys
// end up causing the audio elements with our ring/ringback etc
// audio clips in to play.
if (navigator.mediaSession) {
navigator.mediaSession.setActionHandler("play", function () {});
navigator.mediaSession.setActionHandler("pause", function () {});
navigator.mediaSession.setActionHandler("seekbackward", function () {});
navigator.mediaSession.setActionHandler("seekforward", function () {});
navigator.mediaSession.setActionHandler("previoustrack", function () {});
navigator.mediaSession.setActionHandler("nexttrack", function () {});
}

if (SettingsStore.getValue(UIFeature.Voip)) {
MatrixClientPeg.safeGet().on(CallEventHandlerEvent.Incoming, this.onCallIncoming);
}

this.checkProtocols(CHECK_PROTOCOLS_ATTEMPTS);

// Add event listeners for the <audio> elements
Object.values(AudioID).forEach((audioId) => {
const audioElement = document.getElementById(audioId) as HTMLMediaElement;
if (audioElement) {
this.addEventListenersForAudioElement(audioElement);
} else {
logger.warn(`LegacyCallHandler: missing <audio id="${audioId}"> from page`);
}
});
}

public stop(): void {
const cli = MatrixClientPeg.get();
if (cli) {
cli.removeListener(CallEventHandlerEvent.Incoming, this.onCallIncoming);
}

// Remove event listeners for the <audio> elements
Array.from(this.audioElementsWithListeners.keys()).forEach((audioElement) => {
this.removeEventListenersForAudioElement(audioElement);
});
}

private addEventListenersForAudioElement(audioElement: HTMLMediaElement): void {
// Only need to setup the listeners once
if (!this.audioElementsWithListeners.get(audioElement)) {
MEDIA_EVENT_TYPES.forEach((errorEventType) => {
audioElement.addEventListener(errorEventType, this);
this.audioElementsWithListeners.set(audioElement, true);
});
}
}

private removeEventListenersForAudioElement(audioElement: HTMLMediaElement): void {
MEDIA_EVENT_TYPES.forEach((errorEventType) => {
audioElement.removeEventListener(errorEventType, this);
});
}

/* istanbul ignore next (remove if we start using this function for things other than debug logging) */
Expand Down Expand Up @@ -465,74 +424,37 @@ export default class LegacyCallHandler extends EventEmitter {
return this.transferees.get(callId);
}

public play(audioId: AudioID): void {
public async play(audioId: AudioID): Promise<void> {
const logPrefix = `LegacyCallHandler.play(${audioId}):`;
logger.debug(`${logPrefix} beginning of function`);
// TODO: Attach an invisible element for this instead
// which listens?
const audio = document.getElementById(audioId) as HTMLMediaElement;
if (audio) {
this.addEventListenersForAudioElement(audio);
const playAudio = async (): Promise<void> => {
try {
if (audio.muted) {
logger.error(
`${logPrefix} <audio> element was unexpectedly muted but we recovered ` +
`gracefully by unmuting it`,
);
// Recover gracefully
audio.muted = false;
}

// This still causes the chrome debugger to break on promise rejection if
// the promise is rejected, even though we're catching the exception.
logger.debug(`${logPrefix} attempting to play audio at volume=${audio.volume}`);
await audio.play();
logger.debug(`${logPrefix} playing audio successfully`);
} catch (e) {
// This is usually because the user hasn't interacted with the document,
// or chrome doesn't think so and is denying the request. Not sure what
// we can really do here...
// https://github.com/vector-im/element-web/issues/7657
logger.warn(`${logPrefix} unable to play audio clip`, e);
}
};
if (this.audioPromises.has(audioId)) {
this.audioPromises.set(
audioId,
this.audioPromises.get(audioId)!.then(() => {
audio.load();
return playAudio();
}),
);
} else {
this.audioPromises.set(audioId, playAudio());
}
} else {
logger.warn(`${logPrefix} unable to find <audio> element for ${audioId}`);
}
const audioInfo: Record<AudioID, [prefix: string, loop: boolean]> = {
[AudioID.Ring]: [`./media/ring`, true],
[AudioID.Ringback]: [`./media/ringback`, true],
[AudioID.CallEnd]: [`./media/callend`, false],
[AudioID.Busy]: [`./media/busy`, false],
};

const [urlPrefix, loop] = audioInfo[audioId];
const source = await this.backgroundAudio.pickFormatAndPlay(urlPrefix, ["mp3", "ogg"], loop);
this.playingSources[audioId] = source;
logger.debug(`${logPrefix} playing audio successfully`);
}

public pause(audioId: AudioID): void {
const logPrefix = `LegacyCallHandler.pause(${audioId}):`;
logger.debug(`${logPrefix} beginning of function`);
// TODO: Attach an invisible element for this instead
// which listens?
const audio = document.getElementById(audioId) as HTMLMediaElement;
const pauseAudio = (): void => {
logger.debug(`${logPrefix} pausing audio`);
// pause doesn't return a promise, so just do it
audio.pause();
};
if (audio) {
if (this.audioPromises.has(audioId)) {
this.audioPromises.set(audioId, this.audioPromises.get(audioId)!.then(pauseAudio));
} else {
pauseAudio();
}
} else {
logger.warn(`${logPrefix} unable to find <audio> element for ${audioId}`);

const source = this.playingSources[audioId];
if (!source) {
logger.debug(`${logPrefix} audio not playing`);
return;
}

source.stop();
delete this.playingSources[audioId];

logger.debug(`${logPrefix} paused audio`);
}

private matchesCallForThisRoom(call: MatrixCall): boolean {
Expand Down
27 changes: 8 additions & 19 deletions src/Notifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ import ToastStore from "./stores/ToastStore";
import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoEventType } from "./voice-broadcast";
import { getSenderName } from "./utils/event/getSenderName";
import { stripPlainReply } from "./utils/Reply";
import { BackgroundAudio } from "./audio/BackgroundAudio";

/*
* Dispatches:
Expand Down Expand Up @@ -112,6 +113,8 @@ class NotifierClass {
private toolbarHidden?: boolean;
private isSyncing?: boolean;

private backgroundAudio = new BackgroundAudio();

public notificationMessageForEvent(ev: MatrixEvent): string | null {
const msgType = ev.getContent().msgtype;
if (msgType && msgTypeHandlers.hasOwnProperty(msgType)) {
Expand Down Expand Up @@ -226,28 +229,14 @@ class NotifierClass {
return;
}

// Play notification sound here
const sound = this.getSoundForRoom(room.roomId);
logger.log(`Got sound ${(sound && sound.name) || "default"} for ${room.roomId}`);

try {
const selector = document.querySelector<HTMLAudioElement>(
sound ? `audio[src='${sound.url}']` : "#messageAudio",
);
let audioElement = selector;
if (!audioElement) {
if (!sound) {
logger.error("No audio element or sound to play for notification");
return;
}
audioElement = new Audio(sound.url);
if (sound.type) {
audioElement.type = sound.type;
}
document.body.appendChild(audioElement);
}
await audioElement.play();
} catch (ex) {
logger.warn("Caught error when trying to fetch room notification sound:", ex);
if (sound) {
await this.backgroundAudio.play(sound.url);
} else {
await this.backgroundAudio.pickFormatAndPlay("media/message", ["mp3", "ogg"]);
}
}

Expand Down
74 changes: 74 additions & 0 deletions src/audio/BackgroundAudio.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
Copyright 2024 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import { logger } from "matrix-js-sdk/src/logger";

import { createAudioContext } from "./compat";

const formatMap = {
mp3: "audio/mpeg",
ogg: "audio/ogg",
};

export class BackgroundAudio {
private audioContext = createAudioContext();
private sounds: Record<string, AudioBuffer> = {};

public async pickFormatAndPlay<F extends Array<keyof typeof formatMap>>(
urlPrefix: string,
formats: F,
loop = false,
): Promise<AudioBufferSourceNode> {
const format = this.pickFormat(...formats);
if (!format) {
console.log("Browser doesn't support any of the formats", formats);
// Will probably never happen. If happened, format="" and will fail to load audio. Who cares...
}

return this.play(`${urlPrefix}.${format}`, loop);
}

public async play(url: string, loop = false): Promise<AudioBufferSourceNode> {
if (!this.sounds.hasOwnProperty(url)) {
// No cache, fetch it
const response = await fetch(url);
if (response.status != 200) {
logger.warn("Failed to fetch error audio");
}
const buffer = await response.arrayBuffer();
const sound = await this.audioContext.decodeAudioData(buffer);
this.sounds[url] = sound;
}
const source = this.audioContext.createBufferSource();
source.buffer = this.sounds[url];
source.loop = loop;
source.connect(this.audioContext.destination);
source.start();
return source;
}

private pickFormat<F extends Array<keyof typeof formatMap>>(...formats: F): F[number] | null {
// Detect supported formats
const audioElement = document.createElement("audio");

for (const format of formats) {
if (audioElement.canPlayType(formatMap[format])) {
return format;
}
}
return null;
}
}
12 changes: 3 additions & 9 deletions src/voice-broadcast/models/VoiceBroadcastRecording.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import { VoiceBroadcastChunkEvents } from "../utils/VoiceBroadcastChunkEvents";
import { RelationsHelper, RelationsHelperEvent } from "../../events/RelationsHelper";
import { createReconnectedListener } from "../../utils/connection";
import { localNotificationsAreSilenced } from "../../utils/notifications";
import { BackgroundAudio } from "../../audio/BackgroundAudio";

export enum VoiceBroadcastRecordingEvent {
StateChanged = "liveness_changed",
Expand Down Expand Up @@ -75,6 +76,7 @@ export class VoiceBroadcastRecording
private reconnectedListener: ClientEventHandlerMap[ClientEvent.Sync];
private roomId: string;
private infoEventId: string;
private backgroundAudio = new BackgroundAudio();

/**
* Broadcast chunks have a sequence number to bring them in the correct order and to know if a message is missing.
Expand Down Expand Up @@ -346,15 +348,7 @@ export class VoiceBroadcastRecording
return;
}

// Audio files are added to the document in Element Web.
// See <audio> elements in https://github.com/vector-im/element-web/blob/develop/src/vector/index.html
const audioElement = document.querySelector<HTMLAudioElement>("audio#errorAudio");

try {
await audioElement?.play();
} catch (e) {
logger.warn("error playing 'errorAudio'", e);
}
await this.backgroundAudio.pickFormatAndPlay("./media/error", ["mp3", "ogg"]);
}

private async uploadFile(chunk: ChunkRecordedPayload): ReturnType<typeof uploadFile> {
Expand Down
Loading

0 comments on commit e288f61

Please sign in to comment.