feat(video): add support for webcodec video

package.json

@@ -89,6 +89,7 @@
     "sass-loader": "^7.3.1",
     "text-loader": "0.0.1",
     "vue-cli-plugin-electron-builder": "^2.0.0",
+    "vue-template-babel-compiler": "^2.0.0",
     "vue-template-compiler": "^2.6.10",
     "vuex-localstorage": "^1.0.0",
     "worker-loader": "^2.0.0"

src/application/createWebcodecVideo.js

@@ -0,0 +1,42 @@
+export function createWebcodecVideo({ id, url, textureDefinition }) {
+  return new Promise(async (resolve, reject) => {
+    const video = document.createElement("video");
+    video.setAttribute("crossorigin", "anonymous");
+    video.setAttribute("loop", true);
+    video.onerror = reject;
+    video.muted = true;
+
+    video.onloadedmetadata = async () => {
+      const stream = video.captureStream();
+      const [track] = stream.getVideoTracks();
+
+      // eslint-disable-next-line
+      const processor = new MediaStreamTrackProcessor(track);
+      const frameStream = processor.readable;
+
+      // Transfer the readable stream to the worker.
+      // NOTE: transferring frameStream and reading it in the worker is more
+      // efficient than reading frameStream here and transferring VideoFrames individually.
+      this.store.dispatch(
+        "videos/assignVideoStream",
+        {
+          id,
+          stream: frameStream,
+          width: video.videoWidth || 256,
+          height: video.videoHeight || 256
+        },
+        [frameStream]
+      );
+
+      resolve({ id, video, stream });
+    };
+
+    video.setAttribute("src", url);
+    video.playbackRate = textureDefinition?.options?.playbackrate;
+    if (textureDefinition?.options?.paused) {
+      video.pause();
+    } else {
+      video.play();
+    }
+  });
+}

src/application/index.js

@@ -16,6 +16,7 @@ import PromiseWorker from "promise-worker-transferable";
 import Vue from "vue";
 import { ipcRenderer } from "electron";
 import { app } from "@electron/remote";
+import { createWebcodecVideo } from "./createWebcodecVideo";
 import { GROUP_ENABLED } from "./constants";
 
 let imageBitmap;
@@ -29,6 +30,7 @@ export default class ModV {
   setupBeatDetektor = setupBeatDetektor;
   setupMidi = setupMidi;
   windowHandler = windowHandler;
+  createWebcodecVideo = createWebcodecVideo;
   use = use;
   debug = false;
   features = Vue.observable({
@@ -45,6 +47,7 @@ export default class ModV {
     perceptualSpread: 0,
     perceptualSharpness: 0
   });
+  videos = {};
 
   _store = store;
   store = {
@@ -64,7 +67,7 @@ export default class ModV {
       payload: app.getAppPath()
     });
 
-    this.$worker.addEventListener("message", e => {
+    this.$worker.addEventListener("message", async e => {
       const message = e.data;
       const { type } = message;
 
@@ -78,6 +81,19 @@ export default class ModV {
       //   console.log(`⚙️%c ${type}`, "color: red");
       // }
 
+      if (type === "createWebcodecVideo") {
+        const videoContext = await this.createWebcodecVideo(message);
+        this.videos[videoContext.id] = videoContext;
+      }
+
+      if (type === "removeWebcodecVideo") {
+        const { video, stream } = this.videos[message.id];
+        video.src = "";
+        // eslint-disable-next-line no-for-each/no-for-each
+        stream.getTracks().forEach(track => track.stop());
+        delete this.videos[message.id];
+      }
+
       if (e.data.type === "tick" && this.ready) {
         this.tick(e.data.payload);
         return;

src/application/sample-modules/Texture2d.js

@@ -32,24 +32,75 @@ export default {
       min: -100,
       max: 100,
       step: 1
+    },
+    constrain: {
+      label: "Constrain",
+      type: "enum",
+      default: "none",
+      enum: [
+        { label: "None (Scale)", value: "none" },
+        { label: "Contain", value: "contain" },
+        { label: "Cover", value: "cover" }
+      ]
     }
   },
   draw({ canvas: { width, height }, context, props }) {
-    const { scale, offsetX, offsetY } = props;
+    const { constrain, offsetX, offsetY, scale, texture } = props;
+
+    if (texture.value) {
+      let { width: imageWidth, height: imageHeight } = texture.value;
+
+      let x;
+      let y;
+
+      if (constrain === "contain") {
+        imageHeight = (imageHeight / imageWidth) * width;
+        imageWidth = width;
+
+        y = (height - imageHeight) / 2;
+        x = 0;
+
+        if (imageHeight > height) {
+          imageWidth = (imageWidth / imageHeight) * height;
+          imageHeight = height;
+
+          y = 0;
+          x = (width - imageWidth) / 2;
+        }
+      } else if (constrain === "cover") {
+        const imageRatio = imageHeight / imageWidth;
+        const canvasRatio = height / width;
+
+        if (imageRatio < canvasRatio) {
+          imageWidth = (width * canvasRatio) / imageRatio;
+          imageHeight = height;
+
+          x = (width - imageWidth) / 2;
+          y = 0;
+        } else {
+          imageHeight = width * imageRatio;
+          imageWidth = width;
+
+          x = 0;
+          y = (height - imageHeight * scale) / 2;
+        }
+      } else {
+        imageWidth = imageWidth * scale;
+        imageHeight = imageHeight * scale;
+
+        x = (width - imageWidth) / 2;
+        y = (height - imageHeight) / 2;
+      }
 
-    if (props.texture.value) {
-      const { width: imageWidth, height: imageHeight } = props.texture.value;
-      const x = (width - imageWidth * scale) / 2;
-      const y = (height - imageHeight * scale) / 2;
       const calculatedOffsetX = (width / 100) * offsetX;
       const calculatedOffsetY = (height / 100) * offsetY;
 
       context.drawImage(
-        props.texture.value,
+        texture.value,
         x + calculatedOffsetX,
         y + calculatedOffsetY,
-        imageWidth * scale,
-        imageHeight * scale
+        imageWidth,
+        imageHeight
       );
     }
   }

src/application/utils/get-prop-default.js

@@ -22,7 +22,8 @@ export default async function getPropDefault(
     if (store.state.dataTypes[type].create) {
       return await store.state.dataTypes[type].create(
         propData,
-        module.meta.isGallery
+        module.meta.isGallery,
+        useExistingData
       );
     }
   }

src/application/worker/store/modules/dataTypes.js

@@ -28,10 +28,10 @@ const state = {
     inputs: () => ({ r: 0, g: 0, b: 0, a: 0 })
   },
   texture: {
-    async create(textureDefinition = {}, isGallery) {
+    async create(textureDefinition = {}, isGallery, useExistingData = false) {
       const { type, options } = textureDefinition;
-      textureDefinition.location = "";
-      textureDefinition.id = "";
+      textureDefinition.location = textureDefinition.location ?? "";
+      // textureDefinition.id = textureDefinition.id ?? "";
 
       if (type === "image") {
         const { path } = options;
@@ -48,7 +48,22 @@ const state = {
         textureDefinition.id = id;
       }
 
-      if (type === "canvas" || type == "group") {
+      if (type === "video" && (useExistingData || !textureDefinition.id)) {
+        let id;
+        try {
+          ({ id } = await store.dispatch(
+            "videos/createVideoFromPath",
+            textureDefinition
+          ));
+        } catch (e) {
+          console.error(e);
+        }
+
+        textureDefinition.location = "videos/video";
+        textureDefinition.id = id;
+      }
+
+      if (type === "canvas" || type === "group") {
         const { id } = options;
         textureDefinition.location = "outputs/auxillaryCanvas";
         textureDefinition.id = id;
@@ -63,6 +78,15 @@ const state = {
         }
       });
     },
+    async destroy(textureDefinition) {
+      const { type, id } = textureDefinition;
+
+      if (type === "video") {
+        await store.dispatch("videos/removeVideoById", {
+          id
+        });
+      }
+    },
     get: textureDefinition => {
       if (!textureDefinition.location.length) {
         return false;

src/application/worker/store/modules/modules.js

@@ -646,12 +646,24 @@ const actions = {
       meta.compositeOperationInputId,
       meta.enabledInputId
     ];
-    const moduleProperties = Object.values(module.$props).map(prop => ({
-      id: prop.id,
-      type: prop.type
-    }));
+    const moduleProperties = Object.entries(module.$props).map(
+      ([key, prop]) => ({
+        key,
+        id: prop.id,
+        type: prop.type
+      })
+    );
     const inputIds = [...moduleProperties, ...metaInputIds.map(id => ({ id }))];
 
+    for (let i = 0, len = moduleProperties.length; i < len; i++) {
+      const { key, type: propType } = moduleProperties[i];
+
+      // destroy anything created by datatypes we don't need anymore
+      if (store.state.dataTypes[propType].destroy) {
+        store.state.dataTypes[propType].destroy(module.props[key]);
+      }
+    }
+
     for (let i = 0, len = inputIds.length; i < len; i++) {
       const { id: inputId, type: propType } = inputIds[i];