@remotion/webcodecs: createVideoDecoder() and createAudioDecoder() APIs

packages/docs/docs/media-parser/samples.mdx

@@ -1,4 +1,5 @@
 ---
+image: /generated/articles-docs-media-parser-samples.png
 id: samples
 title: Extract samples
 slug: /media-parser/samples

packages/docs/docs/webcodecs/TableOfContents.tsx

@@ -106,10 +106,22 @@ export const TableOfContents: React.FC = () => {
 				<TOCItem link="/docs/webcodecs/convert-audiodata">
 					<strong>{'convertAudioData()'}</strong>
 					<div>
-						Change the format or sample rate of an <code>AudioData</code>
+						Change the format or sample rate of an <code>AudioData</code>{' '}
 						object.
 					</div>
 				</TOCItem>
+				<TOCItem link="/docs/webcodecs/create-audio-decoder">
+					<strong>{'createAudioDecoder()'}</strong>
+					<div>
+						Create an <code>AudioDecoder</code> object.
+					</div>
+				</TOCItem>
+				<TOCItem link="/docs/webcodecs/create-video-decoder">
+					<strong>{'createVideoDecoder()'}</strong>
+					<div>
+						Create a <code>VideoDecoder</code> object.
+					</div>
+				</TOCItem>
 			</Grid>
 		</div>
 	);

packages/docs/docs/webcodecs/create-audio-decoder.mdx

@@ -0,0 +1,151 @@
+---
+image: /generated/articles-docs-webcodecs-create-audio-decoder.png
+sidebar_label: createAudioDecoder()
+title: createAudioDecoder()
+slug: /webcodecs/create-audio-decoder
+crumb: '@remotion/webcodecs'
+---
+
+import {UnstableDisclaimer} from './UnstableDisclaimer';
+
+:::warning
+
+<UnstableDisclaimer />
+:::
+
+# createAudioDecoder()<AvailableFrom v="4.0.307" />
+
+This function is a wrapper around the [`AudioDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder) Web API.
+
+```tsx twoslash title="Create an audio decoder"
+import type {MediaParserAudioTrack} from '@remotion/media-parser';
+
+const track = {} as unknown as MediaParserAudioTrack;
+
+// ---cut---
+
+import {createAudioDecoder} from '@remotion/webcodecs';
+
+const decoder = createAudioDecoder({
+  track,
+  onFrame: console.log,
+  onError: console.error,
+});
+```
+
+## Differences to `AudioDecoder`
+
+- Samples with a `codec` of `pcm-s16` are accepted and passed through, even if the `AudioDecoder` object does not exist or support it.
+- Two new methods are added: [`.waitForQueueToBeLessThan()`](#waitforqueuetobelessthanqueue-size-number) and [`.waitForFinish()`](#waitforfinish).
+- The [`dequeue`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder/dequeue_event) event is not supported as it is not reliable across browsers.
+- In addition to [`EncodedAudioChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedAudioChunk), [`EncodedAudioChunkInit`](https://www.w3.org/TR/webcodecs/#dictdef-encodedaudiochunkinit) objects are also accepted for [`.decode()`](#decode).
+- A [`webcodecsController()`](/docs/webcodecs/webcodecs-controller) instance can be passed in to the function, allowing for decoding to be paused, resumed and aborted.
+- [`.decode()`](#decode) is async, and returns a promise, allowing for a halt if the decoder is paused.
+- Only samples with a size of 16 bytes or more are actually being input, [to avoid a bug in Chrome](https://github.com/remotion-dev/remotion/blob/c7f18a85bcb1bc58d3811efcd9e68c96ff38ccae/packages/webcodecs/src/create-audio-decoder.ts#L121-L129).
+- A [`logLevel`](#loglevel) can be passed in to the function, allowing the queue to be debugged.
+- The [`onFrame`](#onframe) callback is being awaited. When rejected, the error lands in the [`onError`](#onerror) callback. When resolved, only then the queue size counter will be decreased.
+
+## API
+
+Takes an object with the following properties:
+
+### `track`
+
+An [`AudioDecoderConfig`](https://www.w3.org/TR/webcodecs/#dictdef-audiodecoderconfig) object.  
+You may pass a [`MediaParserAudioTrack`](/docs/media-parser/types#mediaparseraudiotrack) object from [`parseMedia()`](/docs/media-parser/parse-media), which also is an `AudioDecoderConfig` object.
+
+### `onFrame`
+
+A callback that is called when a frame is decoded.
+
+Takes a single argument, which is an [`AudioData`](https://developer.mozilla.org/en-US/docs/Web/API/AudioData) object.
+
+Only once the callback resolves, the queue size counter will be decreased.
+
+### `onError`
+
+A callback that is called when an error occurs or the decode is aborted through the controller.
+
+Takes a single argument, which is an [`Error`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error) object.
+
+### `controller?`
+
+A [`webcodecsController()`](/docs/webcodecs/webcodecs-controller) instance.
+
+If provided, you can call [`.pause()`](/docs/webcodecs/webcodecs-controller#pause), [`.resume()`](/docs/webcodecs/webcodecs-controller#resume) and [`.abort()`](/docs/webcodecs/webcodecs-controller#abort) on the controller to pause, resume and abort the decoding.
+
+
+### `logLevel?`
+
+_string_ <TsType type="LogLevel" source="@remotion/media-parser"/>
+
+One of `"error"`, `"warn"`, `"info"`, `"debug"`, `"trace"`.  
+Default value: `"info"`, which logs only important information.
+
+## Return type
+
+Returns an object with the following properties:
+
+
+### `decode(sample: EncodedAudioChunkInit | EncodedAudioChunk)`
+
+Decodes a sample. Same as [`AudioDecoder.decode()`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder/decode).  
+You can pass in a [`MediaParserAudioSample`](/docs/media-parser/types#mediaparseraudiosample) object from [`parseMedia()`](/docs/media-parser/parse-media), which also satisfies the [`EncodedAudioChunkInit`](https://www.w3.org/TR/webcodecs/#dictdef-encodedaudiochunkinit) interface.
+
+
+### `waitForQueueToBeLessThan(queueSize: number)`
+
+A promise that resolves when the queue size is less than the given number.  
+The queue is only decremented when the[ `onFrame`](#onframe) callback resolves.
+
+### `waitForFinish()`
+
+A promise that resolves when the decoder is finished.
+
+### `flush()`
+
+Flushes the decoder. Same as [`AudioDecoder.flush()`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder/flush).
+
+### `close()`
+
+Closes the decoder. Same as [`AudioDecoder.close()`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder/close).
+
+## Example usage with `@remotion/media-parser`
+
+In this example, the whole audio track is decoded and the decoder is closed when the track is done.  
+By using `async` / `await`, the [`parseMedia()`](/docs/media-parser/parse-media) call is artificially slowed down to not flood the `AudioDecoder` and cause much memory to be allocated.
+
+```tsx twoslash title="Decode an audio track"
+import {parseMedia} from '@remotion/media-parser';
+import {createAudioDecoder} from '@remotion/webcodecs';
+
+await parseMedia({
+  src: 'https://parser.media/video.mp4',
+  onAudioTrack: ({track, container}) => {
+    const decoder = createAudioDecoder({
+      track,
+      onFrame: console.log,
+      onError: console.error,
+    });
+
+    return async (sample) => {
+      // Called on every sample
+      await decoder.waitForQueueToBeLessThan(10);
+      await decoder.decode(sample);
+
+      return async () => {
+        // Called when the track is done
+        await decoder.flush();
+        await decoder.waitForFinish();
+        decoder.close()
+      };
+    };
+  },
+});
+```
+
+
+## See also
+
+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/webcodecs/src/create-audio-decoder.ts)
+- [`@remotion/webcodecs`](/docs/webcodecs)

packages/docs/docs/webcodecs/create-video-decoder.mdx

@@ -0,0 +1,137 @@
+---
+image: /generated/articles-docs-webcodecs-create-video-decoder.png
+sidebar_label: createVideoDecoder()
+title: createVideoDecoder()
+slug: /webcodecs/create-video-decoder
+crumb: '@remotion/webcodecs'
+---
+
+import {UnstableDisclaimer} from './UnstableDisclaimer';
+
+:::warning
+
+<UnstableDisclaimer />
+:::
+
+# createVideoDecoder()<AvailableFrom v="4.0.307" />
+
+This function is a wrapper around the [`VideoDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder) Web API.
+
+```tsx twoslash title="Create a video decoder"
+import type {MediaParserVideoTrack} from '@remotion/media-parser';
+
+const track = {} as unknown as MediaParserVideoTrack;
+
+// ---cut---
+
+import {createVideoDecoder} from '@remotion/webcodecs';
+
+const decoder = createVideoDecoder({
+  track,
+  onFrame: console.log,
+  onError: console.error,
+});
+```
+
+## Differences to `VideoDecoder`
+
+- Two new methods are added: [`.waitForQueueToBeLessThan()`](#waitforqueuetobelessthanqueue-size-number) and [`.waitForFinish()`](#waitforfinish).
+- The [`dequeue`](https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder/dequeue_event) event is not supported as it is not reliable across browsers.
+- In addition to [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk), [`EncodedVideoChunkInit`](https://www.w3.org/TR/webcodecs/#dictdef-encodedvideochunkinit) objects are also accepted for [`.decode()`](#decode).
+- A [`webcodecsController()`](/docs/webcodecs/webcodecs-controller) instance can be passed in to the function, allowing for decoding to be paused, resumed and aborted.
+- [`.decode()`](#decode) is async, and returns a promise, allowing for a halt if the decoder is paused.
+- A [`logLevel`](#loglevel) can be passed in to the function, allowing the queue to be debugged.
+- The [`onFrame`](#onframe) callback is being awaited. When rejected, the error lands in the [`onError`](#onerror) callback. When resolved, only then the queue size counter will be decreased.
+
+## API
+
+Takes an object with the following properties:
+
+### `track`
+
+An [`VideoDecoderConfig`](https://www.w3.org/TR/webcodecs/#dictdef-videodecoderconfig) object.  
+You may pass a [`MediaParserVideoTrack`](/docs/media-parser/types#mediaparservideotrack) object from [`parseMedia()`](/docs/media-parser/parse-media), which also is an `VideoDecoderConfig` object.
+
+### `onFrame`
+
+A callback that is called when a frame is decoded.
+
+Takes a single argument, which is a [`VideoFrame`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame) object.
+
+Only once the callback resolves, the queue size counter will be decreased.
+
+### `onError`
+
+A callback that is called when an error occurs or the decode is aborted through the controller.
+
+Takes a single argument, which is an [`Error`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error) object.
+
+### `controller?`
+
+A [`webcodecsController()`](/docs/webcodecs/webcodecs-controller) instance.
+
+If provided, you can call [`.pause()`](/docs/webcodecs/webcodecs-controller#pause), [`.resume()`](/docs/webcodecs/webcodecs-controller#resume) and [`.abort()`](/docs/webcodecs/webcodecs-controller#abort) on the controller to pause, resume and abort the decoding.
+
+### `logLevel?`
+
+_string_ <TsType type="LogLevel" source="@remotion/media-parser"/>
+
+One of `"error"`, `"warn"`, `"info"`, `"debug"`, `"trace"`.  
+Default value: `"info"`, which logs only important information.
+
+## Return type
+
+Returns an object with the following properties:
+
+### `waitForQueueToBeLessThan(queueSize: number)`
+
+A promise that resolves when the queue size is less than the given number.  
+The queue is only decremented when the[ `onFrame`](#onframe) callback resolves.
+
+### `waitForFinish()`
+
+A promise that resolves when the decoder is finished.
+
+### `flush()`
+
+Flushes the decoder. Same as [`AudioDecoder.flush()`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder/flush).
+
+### `close()`
+
+Closes the decoder. Same as [`AudioDecoder.close()`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder/close).
+
+## Example usage with `@remotion/media-parser`
+
+```tsx twoslash title="Decode a video track"
+import {parseMedia} from '@remotion/media-parser';
+import {createVideoDecoder} from '@remotion/webcodecs';
+
+await parseMedia({
+  src: 'https://parser.media/video.mp4',
+  onVideoTrack: ({track, container}) => {
+    const decoder = createVideoDecoder({
+      track,
+      onFrame: console.log,
+      onError: console.error,
+    });
+
+    return async (sample) => {
+      // Called on every sample
+      await decoder.waitForQueueToBeLessThan(10);
+      await decoder.decode(sample);
+
+      return async () => {
+        // Called when the track is done
+        await decoder.flush();
+        await decoder.waitForFinish();
+        decoder.close();
+      };
+    };
+  },
+});
+```
+
+## See also
+
+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/webcodecs/src/create-video-decoder.ts)
+- [`@remotion/webcodecs`](/docs/webcodecs)

packages/docs/sidebars.js

@@ -683,6 +683,8 @@ module.exports = {
 				'webcodecs/get-available-audio-codecs',
 				'webcodecs/get-available-video-codecs',
 				'webcodecs/convert-audiodata',
+				'webcodecs/create-audio-decoder',
+				'webcodecs/create-video-decoder',
 			],
 		},
 		{

packages/docs/src/data/articles.ts

@@ -2888,6 +2888,15 @@ export const articles = [
 		noAi: false,
 		slug: 'media-parser/runtime-support',
 	},
+	{
+		id: 'samples',
+		title: 'Extract samples',
+		relativePath: 'docs/media-parser/samples.mdx',
+		compId: 'articles-docs-media-parser-samples',
+		crumb: '@remotion/media-parser',
+		noAi: false,
+		slug: 'media-parser/samples',
+	},
 	{
 		id: 'seeking-hints',
 		title: 'Seeking Hints',
@@ -6000,6 +6009,24 @@ export const articles = [
 		noAi: false,
 		slug: 'webcodecs/convert-media',
 	},
+	{
+		id: 'webcodecs/create-audio-decoder',
+		title: 'createAudioDecoder()',
+		relativePath: 'docs/webcodecs/create-audio-decoder.mdx',
+		compId: 'articles-docs-webcodecs-create-audio-decoder',
+		crumb: '@remotion/webcodecs',
+		noAi: false,
+		slug: 'webcodecs/create-audio-decoder',
+	},
+	{
+		id: 'webcodecs/create-video-decoder',
+		title: 'createVideoDecoder()',
+		relativePath: 'docs/webcodecs/create-video-decoder.mdx',
+		compId: 'articles-docs-webcodecs-create-video-decoder',
+		crumb: '@remotion/webcodecs',
+		noAi: false,
+		slug: 'webcodecs/create-video-decoder',
+	},
 	{
 		id: 'default-on-audio-track-handler',
 		title: 'defaultOnAudioTrackHandler()',

packages/media-parser/src/webcodec-sample-types.ts

@@ -3,11 +3,11 @@ import type {MediaParserContainer} from './options';
 
 export type MediaParserOnAudioSample = (
 	sample: MediaParserAudioSample,
-) => void | Promise<void> | OnTrackDoneCallback;
+) => void | Promise<void> | OnTrackDoneCallback | Promise<OnTrackDoneCallback>;
 
 export type MediaParserOnVideoSample = (
 	sample: MediaParserVideoSample,
-) => void | Promise<void> | OnTrackDoneCallback;
+) => void | Promise<void> | OnTrackDoneCallback | Promise<OnTrackDoneCallback>;
 
 export type OnTrackDoneCallback = () => void | Promise<void>;