From 416075bba528677a22cf0a2970beecae0b9a1524 Mon Sep 17 00:00:00 2001 From: konstantin-paulus Date: Sun, 20 Oct 2024 12:22:40 -0700 Subject: [PATCH 1/2] Made clip IDs writable --- src/models/transcript.ts | 2 +- src/services/serializer.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/transcript.ts b/src/models/transcript.ts index 739010a..e971c6e 100644 --- a/src/models/transcript.ts +++ b/src/models/transcript.ts @@ -17,7 +17,7 @@ import type { Captions } from '../types'; import type { Serializer } from '../services'; export class Transcript implements Serializer { - public readonly id = crypto.randomUUID(); + public id = crypto.randomUUID(); public language: Language = Language.en; public groups: WordGroup[] = []; diff --git a/src/services/serializer.ts b/src/services/serializer.ts index 4d742b7..fc553f5 100644 --- a/src/services/serializer.ts +++ b/src/services/serializer.ts @@ -4,7 +4,7 @@ export class Serializer { /** * Unique identifier of the object */ - public readonly id = crypto.randomUUID(); + public id = crypto.randomUUID(); toJSON(): any { const obj: any = {}; From 65af0409c17d1d65319b361e50b8538da08ce3c2 Mon Sep 17 00:00:00 2001 From: konstantin-paulus Date: Sun, 20 Oct 2024 12:23:52 -0700 Subject: [PATCH 2/2] ensured clip demuxing is performed on demand --- package-lock.json | 4 ++-- package.json | 2 +- src/clips/video/buffer.spec.ts | 6 +++--- src/clips/video/buffer.ts | 14 +++++++------- src/clips/video/video.spec.ts | 11 ++++++++++- src/clips/video/video.ts | 18 +++++++----------- src/encoders/encoder.spec.ts | 3 +-- src/encoders/encoder.ts | 1 - src/tracks/video/video.ts | 11 +++++++++++ 9 files changed, 42 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6f349fe..619238d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@diffusionstudio/core", - "version": "1.0.0-rc.8", + "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@diffusionstudio/core", - "version": "1.0.0-rc.8", + "version": "1.0.0", "license": "MPL-2.0", "dependencies": { "mp4-muxer": "^5.1.3" diff --git a/package.json b/package.json index f9e5f38..f3b401b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@diffusionstudio/core", "private": false, - "version": "1.0.0-rc.8", + "version": "1.0.0", "type": "module", "description": "Build bleeding edge video processing applications", "files": [ diff --git a/src/clips/video/buffer.spec.ts b/src/clips/video/buffer.spec.ts index e64d993..4bcfb72 100644 --- a/src/clips/video/buffer.spec.ts +++ b/src/clips/video/buffer.spec.ts @@ -27,8 +27,8 @@ describe('FrameBuffer', () => { frameBuffer.enqueue(mockVideoFrame); - expect(frameBuffer['buffer'].length).toBe(1); - expect(frameBuffer['buffer'][0]).toBe(mockVideoFrame); + expect(frameBuffer.frames.length).toBe(1); + expect(frameBuffer.frames[0]).toBe(mockVideoFrame); expect(mockOnEnqueue).toHaveBeenCalled(); }); @@ -44,7 +44,7 @@ describe('FrameBuffer', () => { expect(dequeuedFrame1).toBe(frame1); expect(dequeuedFrame2).toBe(frame2); - expect(frameBuffer['buffer'].length).toBe(0); + expect(frameBuffer.frames.length).toBe(0); }); it('should wait for a frame to be enqueued if buffer is empty and state is active', async () => { diff --git a/src/clips/video/buffer.ts b/src/clips/video/buffer.ts index a7da9f6..5c89d22 100644 --- a/src/clips/video/buffer.ts +++ b/src/clips/video/buffer.ts @@ -6,27 +6,27 @@ */ export class FrameBuffer { - private buffer: Array = []; - private state: 'active' | 'closed' = 'active'; + public frames: Array = []; + public state: 'active' | 'closed' = 'active'; public onenqueue?: () => void; public onclose?: () => void; public enqueue(data: VideoFrame) { - this.buffer.unshift(data); + this.frames.unshift(data); this.onenqueue?.(); } public async dequeue() { - if (this.buffer.length == 0 && this.state == 'active') { + if (this.frames.length == 0 && this.state == 'active') { await this.waitFor(20e3); } - if (this.buffer.length == 0 && this.state == 'closed') { + if (this.frames.length == 0 && this.state == 'closed') { return; } - return this.buffer.pop(); + return this.frames.pop(); } public close() { @@ -35,7 +35,7 @@ export class FrameBuffer { } public terminate() { - for (const frame of this.buffer) { + for (const frame of this.frames) { frame.close(); } } diff --git a/src/clips/video/video.spec.ts b/src/clips/video/video.spec.ts index a504397..e1ff2b3 100644 --- a/src/clips/video/video.spec.ts +++ b/src/clips/video/video.spec.ts @@ -264,6 +264,10 @@ describe('The Video Clip', () => { const composition = new Composition(); await composition.add(clip); + composition.computeFrame(); + + expect(clip.track?.view.children.length).toBe(1); + const buffer = new FrameBuffer(); Object.defineProperty(buffer, 'onenqueue', { @@ -274,8 +278,13 @@ describe('The Video Clip', () => { const decodeSpy = vi.spyOn(clip, 'decodeVideo').mockReturnValueOnce(buffer); composition.state = 'RENDER'; - await clip.seek(new Timestamp()); + await composition.seek(0); + + expect(clip.track?.view.children.length).toBe(0); + + await composition.computeFrame(); + expect(clip.track?.view.children.length).toBe(1); expect(decodeSpy).toBeCalledTimes(1); expect(seekFn.mock.calls[0][0]).toBe(0); }); diff --git a/src/clips/video/video.ts b/src/clips/video/video.ts index 035e1c8..9f73d26 100644 --- a/src/clips/video/video.ts +++ b/src/clips/video/video.ts @@ -110,6 +110,13 @@ export class VideoClip extends VisualMixin(MediaClip) { await this.seek(Timestamp.fromFrames(frame)); } + public enter() { + super.enter(); + if (this.track?.composition?.rendering && this.buffer?.state != 'active') { + this.decodeVideo(); + } + } + @visualize @textureSwap public update(_: Timestamp): void | Promise { @@ -139,17 +146,6 @@ export class VideoClip extends VisualMixin(MediaClip) { return clip; } - public async seek(time: Timestamp): Promise { - if (this.track?.composition?.rendering) { - const buffer = await this.decodeVideo(); - return new Promise((resolve) => { - buffer.onenqueue = () => resolve(); - }); - } - - return super.seek(time); - } - private async decodeVideo() { this.buffer = new FrameBuffer(); this.worker = new DecodeWorker(); diff --git a/src/encoders/encoder.spec.ts b/src/encoders/encoder.spec.ts index f069ea3..d97b1bd 100644 --- a/src/encoders/encoder.spec.ts +++ b/src/encoders/encoder.spec.ts @@ -49,8 +49,7 @@ describe('The Encoder', () => { await encoder.render(); expect(pauseSpy).toBeCalledTimes(1); - // before and after - expect(seekSpy).toBeCalledTimes(2); + expect(seekSpy).toBeCalledTimes(1); }); it('should not render when the composition renderer is not defined', async () => { diff --git a/src/encoders/encoder.ts b/src/encoders/encoder.ts index 2f7b1f3..b482238 100644 --- a/src/encoders/encoder.ts +++ b/src/encoders/encoder.ts @@ -96,7 +96,6 @@ export class Encoder extends WebcodecsVideoEncoder { this.composition.state = 'IDLE'; this.composition.renderer.resolution = 1; this.composition.fps = FPS_DEFAULT; - this.composition.seek(0); } } diff --git a/src/tracks/video/video.ts b/src/tracks/video/video.ts index fc46295..6c7412b 100644 --- a/src/tracks/video/video.ts +++ b/src/tracks/video/video.ts @@ -6,8 +6,19 @@ */ import { MediaTrack } from '../media'; +import { Timestamp } from '../../models'; + import type { VideoClip } from '../../clips'; export class VideoTrack extends MediaTrack { public readonly type = 'video'; + + public async seek(time: Timestamp): Promise { + if (this.composition?.rendering) { + // ensures that 'enter' method will be called again + this.view.removeChildren(); + } else { + super.seek(time); + } + } }