Generalise VoiceRecording (#9304)

Author: weeman1337

src/audio/VoiceMessageRecording.ts

@@ -0,0 +1,166 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { IEncryptedFile, MatrixClient } from "matrix-js-sdk/src/matrix";
+import { SimpleObservable } from "matrix-widget-api";
+
+import { uploadFile } from "../ContentMessages";
+import { IDestroyable } from "../utils/IDestroyable";
+import { Singleflight } from "../utils/Singleflight";
+import { Playback } from "./Playback";
+import { IRecordingUpdate, RecordingState, VoiceRecording } from "./VoiceRecording";
+
+export interface IUpload {
+    mxc?: string; // for unencrypted uploads
+    encrypted?: IEncryptedFile;
+}
+
+/**
+ * This class can be used to record a single voice message.
+ */
+export class VoiceMessageRecording implements IDestroyable {
+    private lastUpload: IUpload;
+    private buffer = new Uint8Array(0); // use this.audioBuffer to access
+    private playback: Playback;
+
+    public constructor(
+        private matrixClient: MatrixClient,
+        private voiceRecording: VoiceRecording,
+    ) {
+        this.voiceRecording.onDataAvailable = this.onDataAvailable;
+    }
+
+    public async start(): Promise<void> {
+        if (this.lastUpload || this.hasRecording) {
+            throw new Error("Recording already prepared");
+        }
+
+        return this.voiceRecording.start();
+    }
+
+    public async stop(): Promise<Uint8Array> {
+        await this.voiceRecording.stop();
+        return this.audioBuffer;
+    }
+
+    public on(event: string | symbol, listener: (...args: any[]) => void): this {
+        this.voiceRecording.on(event, listener);
+        return this;
+    }
+
+    public off(event: string | symbol, listener: (...args: any[]) => void): this {
+        this.voiceRecording.off(event, listener);
+        return this;
+    }
+
+    public emit(event: string, ...args: any[]): boolean {
+        return this.voiceRecording.emit(event, ...args);
+    }
+
+    public get hasRecording(): boolean {
+        return this.buffer.length > 0;
+    }
+
+    public get isRecording(): boolean {
+        return this.voiceRecording.isRecording;
+    }
+
+    /**
+     * Gets a playback instance for this voice recording. Note that the playback will not
+     * have been prepared fully, meaning the `prepare()` function needs to be called on it.
+     *
+     * The same playback instance is returned each time.
+     *
+     * @returns {Playback} The playback instance.
+     */
+    public getPlayback(): Playback {
+        this.playback = Singleflight.for(this, "playback").do(() => {
+            return new Playback(this.audioBuffer.buffer, this.voiceRecording.amplitudes); // cast to ArrayBuffer proper;
+        });
+        return this.playback;
+    }
+
+    public async upload(inRoomId: string): Promise<IUpload> {
+        if (!this.hasRecording) {
+            throw new Error("No recording available to upload");
+        }
+
+        if (this.lastUpload) return this.lastUpload;
+
+        try {
+            this.emit(RecordingState.Uploading);
+            const { url: mxc, file: encrypted } = await uploadFile(
+                this.matrixClient,
+                inRoomId,
+                new Blob(
+                    [this.audioBuffer],
+                    {
+                        type: this.contentType,
+                    },
+                ),
+            );
+            this.lastUpload = { mxc, encrypted };
+            this.emit(RecordingState.Uploaded);
+        } catch (e) {
+            this.emit(RecordingState.Ended);
+            throw e;
+        }
+        return this.lastUpload;
+    }
+
+    public get durationSeconds(): number {
+        return this.voiceRecording.durationSeconds;
+    }
+
+    public get contentType(): string {
+        return this.voiceRecording.contentType;
+    }
+
+    public get contentLength(): number {
+        return this.buffer.length;
+    }
+
+    public get liveData(): SimpleObservable<IRecordingUpdate> {
+        return this.voiceRecording.liveData;
+    }
+
+    public get isSupported(): boolean {
+        return this.voiceRecording.isSupported;
+    }
+
+    destroy(): void {
+        this.playback?.destroy();
+        this.voiceRecording.destroy();
+    }
+
+    private onDataAvailable = (data: ArrayBuffer) => {
+        const buf = new Uint8Array(data);
+        const newBuf = new Uint8Array(this.buffer.length + buf.length);
+        newBuf.set(this.buffer, 0);
+        newBuf.set(buf, this.buffer.length);
+        this.buffer = newBuf;
+    };
+
+    private get audioBuffer(): Uint8Array {
+        // We need a clone of the buffer to avoid accidentally changing the position
+        // on the real thing.
+        return this.buffer.slice(0);
+    }
+}
+
+export const createVoiceMessageRecording = (matrixClient: MatrixClient) => {
+    return new VoiceMessageRecording(matrixClient, new VoiceRecording());
+};

src/audio/VoiceRecording.ts

@@ -16,20 +16,16 @@ limitations under the License.
 
 import * as Recorder from 'opus-recorder';
 import encoderPath from 'opus-recorder/dist/encoderWorker.min.js';
-import { MatrixClient } from "matrix-js-sdk/src/client";
 import { SimpleObservable } from "matrix-widget-api";
 import EventEmitter from "events";
-import { IEncryptedFile } from "matrix-js-sdk/src/@types/event";
 import { logger } from "matrix-js-sdk/src/logger";
 
 import MediaDeviceHandler from "../MediaDeviceHandler";
 import { IDestroyable } from "../utils/IDestroyable";
 import { Singleflight } from "../utils/Singleflight";
 import { PayloadEvent, WORKLET_NAME } from "./consts";
 import { UPDATE_EVENT } from "../stores/AsyncStore";
-import { Playback } from "./Playback";
 import { createAudioContext } from "./compat";
-import { uploadFile } from "../ContentMessages";
 import { FixedRollingArray } from "../utils/FixedRollingArray";
 import { clamp } from "../utils/numbers";
 import mxRecorderWorkletPath from "./RecorderWorklet";
@@ -55,38 +51,23 @@ export enum RecordingState {
     Uploaded = "uploaded",
 }
 
-export interface IUpload {
-    mxc?: string; // for unencrypted uploads
-    encrypted?: IEncryptedFile;
-}
-
 export class VoiceRecording extends EventEmitter implements IDestroyable {
     private recorder: Recorder;
     private recorderContext: AudioContext;
     private recorderSource: MediaStreamAudioSourceNode;
     private recorderStream: MediaStream;
     private recorderWorklet: AudioWorkletNode;
     private recorderProcessor: ScriptProcessorNode;
-    private buffer = new Uint8Array(0); // use this.audioBuffer to access
-    private lastUpload: IUpload;
     private recording = false;
     private observable: SimpleObservable<IRecordingUpdate>;
-    private amplitudes: number[] = []; // at each second mark, generated
-    private playback: Playback;
+    public amplitudes: number[] = []; // at each second mark, generated
     private liveWaveform = new FixedRollingArray(RECORDING_PLAYBACK_SAMPLES, 0);
-
-    public constructor(private client: MatrixClient) {
-        super();
-    }
+    public onDataAvailable: (data: ArrayBuffer) => void;
 
     public get contentType(): string {
         return "audio/ogg";
     }
 
-    public get contentLength(): number {
-        return this.buffer.length;
-    }
-
     public get durationSeconds(): number {
         if (!this.recorder) throw new Error("Duration not available without a recording");
         return this.recorderContext.currentTime;
@@ -165,13 +146,9 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
                 encoderComplexity: 3, // 0-10, 10 is slow and high quality.
                 resampleQuality: 3, // 0-10, 10 is slow and high quality
             });
-            this.recorder.ondataavailable = (a: ArrayBuffer) => {
-                const buf = new Uint8Array(a);
-                const newBuf = new Uint8Array(this.buffer.length + buf.length);
-                newBuf.set(this.buffer, 0);
-                newBuf.set(buf, this.buffer.length);
-                this.buffer = newBuf;
-            };
+
+            // not using EventEmitter here because it leads to detached bufferes
+            this.recorder.ondataavailable = (data: ArrayBuffer) => this?.onDataAvailable(data);
         } catch (e) {
             logger.error("Error starting recording: ", e);
             if (e instanceof DOMException) { // Unhelpful DOMExceptions are common - parse them sanely
@@ -191,12 +168,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         }
     }
 
-    private get audioBuffer(): Uint8Array {
-        // We need a clone of the buffer to avoid accidentally changing the position
-        // on the real thing.
-        return this.buffer.slice(0);
-    }
-
     public get liveData(): SimpleObservable<IRecordingUpdate> {
         if (!this.recording) throw new Error("No observable when not recording");
         return this.observable;
@@ -206,10 +177,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         return !!Recorder.isRecordingSupported();
     }
 
-    public get hasRecording(): boolean {
-        return this.buffer.length > 0;
-    }
-
     private onAudioProcess = (ev: AudioProcessingEvent) => {
         this.processAudioUpdate(ev.playbackTime);
 
@@ -251,9 +218,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
     };
 
     public async start(): Promise<void> {
-        if (this.lastUpload || this.hasRecording) {
-            throw new Error("Recording already prepared");
-        }
         if (this.recording) {
             throw new Error("Recording already in progress");
         }
@@ -267,7 +231,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         this.emit(RecordingState.Started);
     }
 
-    public async stop(): Promise<Uint8Array> {
+    public async stop(): Promise<void> {
         return Singleflight.for(this, "stop").do(async () => {
             if (!this.recording) {
                 throw new Error("No recording to stop");
@@ -293,54 +257,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
             this.recording = false;
             await this.recorder.close();
             this.emit(RecordingState.Ended);
-
-            return this.audioBuffer;
         });
     }
 
-    /**
-     * Gets a playback instance for this voice recording. Note that the playback will not
-     * have been prepared fully, meaning the `prepare()` function needs to be called on it.
-     *
-     * The same playback instance is returned each time.
-     *
-     * @returns {Playback} The playback instance.
-     */
-    public getPlayback(): Playback {
-        this.playback = Singleflight.for(this, "playback").do(() => {
-            return new Playback(this.audioBuffer.buffer, this.amplitudes); // cast to ArrayBuffer proper;
-        });
-        return this.playback;
-    }
-
     public destroy() {
         // noinspection JSIgnoredPromiseFromCall - not concerned about stop() being called async here
         this.stop();
         this.removeAllListeners();
+        this.onDataAvailable = undefined;
         Singleflight.forgetAllFor(this);
         // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
-        this.playback?.destroy();
         this.observable.close();
     }
-
-    public async upload(inRoomId: string): Promise<IUpload> {
-        if (!this.hasRecording) {
-            throw new Error("No recording available to upload");
-        }
-
-        if (this.lastUpload) return this.lastUpload;
-
-        try {
-            this.emit(RecordingState.Uploading);
-            const { url: mxc, file: encrypted } = await uploadFile(this.client, inRoomId, new Blob([this.audioBuffer], {
-                type: this.contentType,
-            }));
-            this.lastUpload = { mxc, encrypted };
-            this.emit(RecordingState.Uploaded);
-        } catch (e) {
-            this.emit(RecordingState.Ended);
-            throw e;
-        }
-        return this.lastUpload;
-    }
 }

src/components/views/audio_messages/LiveRecordingClock.tsx

@@ -16,12 +16,13 @@ limitations under the License.
 
 import React from "react";
 
-import { IRecordingUpdate, VoiceRecording } from "../../../audio/VoiceRecording";
+import { IRecordingUpdate } from "../../../audio/VoiceRecording";
 import Clock from "./Clock";
 import { MarkedExecution } from "../../../utils/MarkedExecution";
+import { VoiceMessageRecording } from "../../../audio/VoiceMessageRecording";
 
 interface IProps {
-    recorder: VoiceRecording;
+    recorder: VoiceMessageRecording;
 }
 
 interface IState {

src/components/views/audio_messages/LiveRecordingWaveform.tsx

@@ -16,13 +16,14 @@ limitations under the License.
 
 import React from "react";
 
-import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording } from "../../../audio/VoiceRecording";
+import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES } from "../../../audio/VoiceRecording";
 import { arrayFastResample, arraySeed } from "../../../utils/arrays";
 import Waveform from "./Waveform";
 import { MarkedExecution } from "../../../utils/MarkedExecution";
+import { VoiceMessageRecording } from "../../../audio/VoiceMessageRecording";
 
 interface IProps {
-    recorder: VoiceRecording;
+    recorder: VoiceMessageRecording;
 }
 
 interface IState {