Skip to content

Commit 48df2ee

Browse files
Mugen87Mugen87
authored
Introduce VideoFrameTexture. (#30270)
* VideoTexture: Introduce `setFrame()`. * Clean up. * Examples: More clean up. * Lib: Add source of demuxer_mp4.js. * Update VideoTexture.html * Introduce `VideoFrameTexture`. * VideoTexture: Clean up. * VideoFrameTexture: More clean up. * VideoTexture: Clean up. * VideoFrameTexture: Improve comment.
1 parent 73eddcb commit 48df2ee

File tree

11 files changed

+364
-2
lines changed

11 files changed

+364
-2
lines changed

‎.eslintrc.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
"CodeMirror": "readonly",
4242
"esprima": "readonly",
4343
"jsonlint": "readonly",
44-
"VideoFrame": "readonly"
44+
"VideoFrame": "readonly",
45+
"VideoDecoder": "readonly"
4546
},
4647
"rules": {
4748
"no-throw-literal": [

‎docs/api/en/textures/VideoFrameTexture.html‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="utf-8" />
5+
<base href="../../../" />
6+
<script src="page.js"></script>
7+
<link type="text/css" rel="stylesheet" href="page.css" />
8+
</head>
9+
<body>
10+
[page:VideoTexture] &rarr;
11+
12+
<h1>[name]</h1>
13+
14+
<p class="desc">
15+
This class can be used as an alternative way to define video data. Instead of using
16+
an instance of `HTMLVideoElement` like with `VideoTexture`, [name] expects each frame is
17+
defined manaully via [page:.setFrame setFrame](). A typical use case for this module is when
18+
video frames are decoded with the WebCodecs API.
19+
</p>
20+
21+
<h2>Code Example</h2>
22+
23+
<code>
24+
const texture = new THREE.VideoFrameTexture();
25+
texture.setFrame( frame );
26+
</code>
27+
28+
<h2>Examples</h2>
29+
30+
<p>
31+
[example:webgpu_video_frame video / frame]
32+
</p>
33+
34+
<h2>Constructor</h2>
35+
<h3>
36+
[name]( [param:Constant mapping], [param:Constant wrapS],
37+
[param:Constant wrapT], [param:Constant magFilter], [param:Constant minFilter],
38+
[param:Constant format], [param:Constant type], [param:Number anisotropy] )
39+
</h3>
40+
<p>
41+
[page:Constant mapping] -- How the image is applied to the object. An
42+
object type of [page:Textures THREE.UVMapping].
43+
See [page:Textures mapping constants] for other choices.<br />
44+
45+
[page:Constant wrapS] -- The default is [page:Textures THREE.ClampToEdgeWrapping].
46+
See [page:Textures wrap mode constants] for
47+
other choices.<br />
48+
49+
[page:Constant wrapT] -- The default is [page:Textures THREE.ClampToEdgeWrapping].
50+
See [page:Textures wrap mode constants] for
51+
other choices.<br />
52+
53+
[page:Constant magFilter] -- How the texture is sampled when a texel
54+
covers more than one pixel. The default is [page:Textures THREE.LinearFilter].
55+
See [page:Textures magnification filter constants]
56+
for other choices.<br />
57+
58+
[page:Constant minFilter] -- How the texture is sampled when a texel
59+
covers less than one pixel. The default is [page:Textures THREE.LinearFilter].
60+
See [page:Textures minification filter constants] for
61+
other choices.<br />
62+
63+
[page:Constant format] -- The default is [page:Textures THREE.RGBAFormat].
64+
See [page:Textures format constants] for other choices.<br />
65+
66+
[page:Constant type] -- Default is [page:Textures THREE.UnsignedByteType].
67+
See [page:Textures type constants] for other choices.<br />
68+
69+
[page:Number anisotropy] -- The number of samples taken along the axis
70+
through the pixel that has the highest density of texels. By default, this
71+
value is `1`. A higher value gives a less blurry result than a basic mipmap,
72+
at the cost of more texture samples being used. Use
73+
[page:WebGLrenderer.getMaxAnisotropy renderer.getMaxAnisotropy]() to find
74+
the maximum valid anisotropy value for the GPU; this value is usually a
75+
power of 2.<br /><br />
76+
</p>
77+
78+
<h2>Properties</h2>
79+
80+
<p>See the base [page:VideoTexture VideoTexture] class for common properties.</p>
81+
82+
<h2>Methods</h2>
83+
84+
<p>See the base [page:VideoTexture VideoTexture] class for common methods.</p>
85+
86+
<h3>[method:undefined setFrame]( [param:VideoFrame frame] )</h3>
87+
<p>
88+
Sets the current frame of the video. This will automatically update the texture
89+
so the data can be used for rendering.
90+
</p>
91+
92+
<h2>Source</h2>
93+
94+
<p>
95+
[link:https://github.com/mrdoob/three.js/blob/master/src/[path].js src/[path].js]
96+
</p>
97+
</body>
98+
</html>

‎docs/list.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,7 @@
318318
"FramebufferTexture": "api/en/textures/FramebufferTexture",
319319
"Source": "api/en/textures/Source",
320320
"Texture": "api/en/textures/Texture",
321+
"VideoFrameTexture": "api/en/textures/VideoFrameTexture",
321322
"VideoTexture": "api/en/textures/VideoTexture"
322323
}
323324

‎examples/files.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,7 @@
443443
"webgpu_tsl_vfx_flames",
444444
"webgpu_tsl_vfx_linkedparticles",
445445
"webgpu_tsl_vfx_tornado",
446+
"webgpu_video_frame",
446447
"webgpu_video_panorama",
447448
"webgpu_volume_cloud",
448449
"webgpu_volume_perlin",

‎examples/jsm/libs/demuxer_mp4.js‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import MP4Box from 'https://cdn.jsdelivr.net/npm/mp4box@0.5.3/+esm';
2+
3+
// From: https://w3c.github.io/webcodecs/samples/video-decode-display/
4+
5+
// Wraps an MP4Box File as a WritableStream underlying sink.
6+
class MP4FileSink {
7+
#setStatus = null;
8+
#file = null;
9+
#offset = 0;
10+
11+
constructor(file, setStatus) {
12+
this.#file = file;
13+
this.#setStatus = setStatus;
14+
}
15+
16+
write(chunk) {
17+
// MP4Box.js requires buffers to be ArrayBuffers, but we have a Uint8Array.
18+
const buffer = new ArrayBuffer(chunk.byteLength);
19+
new Uint8Array(buffer).set(chunk);
20+
21+
// Inform MP4Box where in the file this chunk is from.
22+
buffer.fileStart = this.#offset;
23+
this.#offset += buffer.byteLength;
24+
25+
// Append chunk.
26+
this.#setStatus("fetch", (this.#offset / (1024 ** 2)).toFixed(1) + " MiB");
27+
this.#file.appendBuffer(buffer);
28+
}
29+
30+
close() {
31+
this.#setStatus("fetch", "Done");
32+
this.#file.flush();
33+
}
34+
}
35+
36+
// Demuxes the first video track of an MP4 file using MP4Box, calling
37+
// `onConfig()` and `onChunk()` with appropriate WebCodecs objects.
38+
export class MP4Demuxer {
39+
#onConfig = null;
40+
#onChunk = null;
41+
#setStatus = null;
42+
#file = null;
43+
44+
constructor(uri, {onConfig, onChunk, setStatus}) {
45+
this.#onConfig = onConfig;
46+
this.#onChunk = onChunk;
47+
this.#setStatus = setStatus;
48+
49+
// Configure an MP4Box File for demuxing.
50+
this.#file = MP4Box.createFile();
51+
this.#file.onError = error => setStatus("demux", error);
52+
this.#file.onReady = this.#onReady.bind(this);
53+
this.#file.onSamples = this.#onSamples.bind(this);
54+
55+
// Fetch the file and pipe the data through.
56+
const fileSink = new MP4FileSink(this.#file, setStatus);
57+
fetch(uri).then(response => {
58+
// highWaterMark should be large enough for smooth streaming, but lower is
59+
// better for memory usage.
60+
response.body.pipeTo(new WritableStream(fileSink, {highWaterMark: 2}));
61+
});
62+
}
63+
64+
// Get the appropriate `description` for a specific track. Assumes that the
65+
// track is H.264, H.265, VP8, VP9, or AV1.
66+
#description(track) {
67+
const trak = this.#file.getTrackById(track.id);
68+
for (const entry of trak.mdia.minf.stbl.stsd.entries) {
69+
const box = entry.avcC || entry.hvcC || entry.vpcC || entry.av1C;
70+
if (box) {
71+
const stream = new MP4Box.DataStream(undefined, 0, MP4Box.DataStream.BIG_ENDIAN);
72+
box.write(stream);
73+
return new Uint8Array(stream.buffer, 8); // Remove the box header.
74+
}
75+
}
76+
throw new Error("avcC, hvcC, vpcC, or av1C box not found");
77+
}
78+
79+
#onReady(info) {
80+
this.#setStatus("demux", "Ready");
81+
const track = info.videoTracks[0];
82+
83+
// Generate and emit an appropriate VideoDecoderConfig.
84+
this.#onConfig({
85+
// Browser doesn't support parsing full vp8 codec (eg: `vp08.00.41.08`),
86+
// they only support `vp8`.
87+
codec: track.codec.startsWith('vp08') ? 'vp8' : track.codec,
88+
codedHeight: track.video.height,
89+
codedWidth: track.video.width,
90+
description: this.#description(track),
91+
});
92+
93+
// Start demuxing.
94+
this.#file.setExtractionOptions(track.id);
95+
this.#file.start();
96+
}
97+
98+
#onSamples(track_id, ref, samples) {
99+
// Generate and emit an EncodedVideoChunk for each demuxed sample.
100+
for (const sample of samples) {
101+
this.#onChunk(new EncodedVideoChunk({
102+
type: sample.is_sync ? "key" : "delta",
103+
timestamp: 1e6 * sample.cts / sample.timescale,
104+
duration: 1e6 * sample.duration / sample.timescale,
105+
data: sample.data
106+
}));
107+
}
108+
}
109+
}

‎examples/screenshots/webgpu_video_frame.jpg‎

3.33 KB
Loading

‎examples/tags.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,5 +149,6 @@
149149
"webgpu_sky": [ "sun" ],
150150
"webgpu_tonemapping": [ "gltf" ],
151151
"webgpu_tsl_compute_attractors_particles": [ "gpgpu" ],
152-
"webgpu_ocean": [ "water" ]
152+
"webgpu_ocean": [ "water" ],
153+
"webgpu_video_frame": [ "webcodecs" ]
153154
}

‎examples/webgpu_video_frame.html‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<title>three.js webgpu - video frames</title>
5+
<meta charset="utf-8">
6+
<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
7+
<link type="text/css" rel="stylesheet" href="main.css">
8+
</head>
9+
<body>
10+
<div id="info">
11+
<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> - video - frames<br/>
12+
Decodes all frames from a MP4 file and renders them onto a plane as fast as possible.<br/>
13+
<a href="https://github.com/gpac/mp4box.js/" target="_blank" rel="noopener">mp4box.js</a> used for MP4 parsing.
14+
</div>
15+
16+
<script type="importmap">
17+
{
18+
"imports": {
19+
"three": "../build/three.webgpu.js",
20+
"three/webgpu": "../build/three.webgpu.js",
21+
"three/tsl": "../build/three.tsl.js",
22+
"three/addons/": "./jsm/"
23+
}
24+
}
25+
</script>
26+
27+
<script type="module">
28+
29+
import * as THREE from 'three';
30+
31+
import { MP4Demuxer } from 'three/addons/libs/demuxer_mp4.js';
32+
33+
let camera, scene, renderer;
34+
35+
init();
36+
37+
function init() {
38+
39+
camera = new THREE.PerspectiveCamera( 75, window.innerWidth / window.innerHeight, .25, 10 );
40+
camera.position.set( 0, 0, 1 );
41+
42+
scene = new THREE.Scene();
43+
44+
const geometry = new THREE.PlaneGeometry();
45+
46+
const videoTexture = new THREE.VideoFrameTexture();
47+
videoTexture.colorSpace = THREE.SRGBColorSpace;
48+
49+
// eslint-disable-next-line compat/compat
50+
const decoder = new VideoDecoder( {
51+
output( frame ) {
52+
53+
videoTexture.setFrame( frame );
54+
55+
},
56+
error( e ) {
57+
58+
console.error( 'VideoDecoder:', e );
59+
60+
}
61+
} );
62+
63+
new MP4Demuxer( './textures/sintel.mp4', {
64+
onConfig( config ) {
65+
66+
decoder.configure( config );
67+
68+
},
69+
onChunk( chunk ) {
70+
71+
decoder.decode( chunk );
72+
73+
},
74+
setStatus( s ) {
75+
76+
console.info( 'MP4Demuxer:', s );
77+
78+
}
79+
} );
80+
81+
const material = new THREE.MeshBasicMaterial( { map: videoTexture } );
82+
83+
const mesh = new THREE.Mesh( geometry, material );
84+
scene.add( mesh );
85+
86+
renderer = new THREE.WebGPURenderer();
87+
renderer.setPixelRatio( window.devicePixelRatio );
88+
renderer.setSize( window.innerWidth, window.innerHeight );
89+
renderer.setAnimationLoop( animate );
90+
document.body.appendChild( renderer.domElement );
91+
92+
//
93+
94+
window.addEventListener( 'resize', onWindowResize );
95+
96+
}
97+
98+
function onWindowResize() {
99+
100+
camera.aspect = window.innerWidth / window.innerHeight;
101+
camera.updateProjectionMatrix();
102+
103+
renderer.setSize( window.innerWidth, window.innerHeight );
104+
105+
}
106+
107+
function animate() {
108+
109+
renderer.render( scene, camera );
110+
111+
}
112+
113+
114+
</script>
115+
</body>
116+
</html>

‎src/Three.Core.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ export { Line } from './objects/Line.js';
2121
export { Points } from './objects/Points.js';
2222
export { Group } from './objects/Group.js';
2323
export { VideoTexture } from './textures/VideoTexture.js';
24+
export { VideoFrameTexture } from './textures/VideoFrameTexture.js';
2425
export { FramebufferTexture } from './textures/FramebufferTexture.js';
2526
export { Source } from './textures/Source.js';
2627
export { DataTexture } from './textures/DataTexture.js';

0 commit comments

Comments
 (0)