diff --git a/html5/index.html b/html5/index.html
index 639b81d7..869c75f0 100644
--- a/html5/index.html
+++ b/html5/index.html
@@ -918,6 +918,10 @@
Xpra Bug Report
if (video) {
// broadway h264 decoder:
client.check_encodings.push("h264");
+ // VPX checks. Will only succeed if we can use VideoDecoder.js
+ client.check_encodings.push("vp8");
+ client.check_encodings.push("vp9");
+
if (JSMpeg && JSMpeg.Renderer && JSMpeg.Decoder) {
//TODO: should be moved to 'check_encodings'
//and added to the decode worker:
diff --git a/html5/js/Client.js b/html5/js/Client.js
index 2d64e616..2e6d547c 100644
--- a/html5/js/Client.js
+++ b/html5/js/Client.js
@@ -1584,6 +1584,8 @@ class XpraClient {
"YUV422P",
"YUV444P",
],
+ vp8: ["YUV420P"],
+ vp9: ["YUV420P", "YUV444P", "YUV444P10"],
},
//this is a workaround for server versions between 2.5.0 to 2.5.2 only:
"encoding.x264.YUV420P.profile": "baseline",
@@ -1594,12 +1596,14 @@ class XpraClient {
"encoding.h264.fast-decode": true,
"encoding.h264+mp4.YUV420P.profile": "baseline",
"encoding.h264+mp4.YUV420P.level": "3.0",
- //prefer native video in mp4/webm container to broadway plain h264:
- "encoding.h264.score-delta": -20,
+ //prefer unmuxed VPX
+ "encoding.vp8.score-delta": 70,
+ "encoding.vp9.score-delta": 60,
"encoding.h264+mp4.score-delta": 50,
"encoding.h264+mp4.": 50,
"encoding.mpeg4+mp4.score-delta": 40,
"encoding.vp8+webm.score-delta": 40,
+ "encoding.h264.score-delta": -20,
"sound.receive": true,
"sound.send": false,
diff --git a/html5/js/OffscreenDecodeWorker.js b/html5/js/OffscreenDecodeWorker.js
index 9550977f..d17fcd68 100644
--- a/html5/js/OffscreenDecodeWorker.js
+++ b/html5/js/OffscreenDecodeWorker.js
@@ -34,8 +34,10 @@ const image_coding = [
];
const video_coding = [];
if (XpraVideoDecoderLoader.hasNativeDecoder) {
- // We can support native H264 decoding
+ // We can support native H264 & VP8 decoding
video_coding.push("h264");
+ video_coding.push("vp8");
+ video_coding.push("vp9");
} else {
console.warn(
"Offscreen decoding is available for images only. Please consider using Google Chrome 94+ in a secure (SSL or localhost) context h264 offscreen decoding support."
@@ -112,10 +114,14 @@ class WindowDecoder {
} else if (image_coding.includes(coding)) {
await this.image_decoder.convertToBitmap(packet);
} else if (video_coding.includes(coding)) {
+ if (coding == "vp9") {
+ // Prepare the VP9 codec (if needed)
+ const csc = packet[10]["csc"];
+ this.video_decoder.prepareVP9params(csc);
+ }
+
if (!this.video_decoder.initialized) {
- // Init with width and heigth of this packet.
- // TODO: Use video max-size? It does not seem to matter.
- this.video_decoder.init(packet[4], packet[5]);
+ this.video_decoder.init(coding);
}
await this.video_decoder.queue_frame(packet);
} else {
diff --git a/html5/js/VideoDecoder.js b/html5/js/VideoDecoder.js
index 290518db..39cb648d 100644
--- a/html5/js/VideoDecoder.js
+++ b/html5/js/VideoDecoder.js
@@ -29,30 +29,61 @@ class XpraVideoDecoder {
this.decoder_queue = [];
this.decoded_frames = [];
+ this.erroneous_frame = false;
+
+ this.codec = null;
+ this.vp9_params = null;
+ this.frameWaitTimeout = 1; // Interval for wait loop while frame is being decoded
this.frame_threshold = 250;
- this.on_frame_error = (packet, error) => {
- console.error("VideoDecoder error on packet", packet, ":", error);
- };
+
}
- init(width, height) {
+ prepareVP9params(csc) {
+ console.log(csc);
+ // Check if we have the right VP9 params before init.
+ // This is needed because we can only set the params when the video decoder is created.
+ // We close the decoder when the coding changes.
+ if (csc == "YUV444P" && this.vp9_params != ".01.10.08") {
+ this.vp9_params = ".01.10.08";
+ this._close();
+ } else if (csc == "YUV444P10" && this.vp9_params != ".03.10.10") {
+ this.vp9_params = ".03.10.10";
+ this._close();
+ } else if (this.vp9_params != ".00.20.08.01.02.02") {
+ // chroma shift for YUV420P, both X and Y are downscaled by 2^1
+ this.vp9_params = ".00.20.08.01.02.02";
+ this._close();
+ }
+ }
+
+ init(coding) {
+ this.draining = false;
+
+ this.codec = this.resolveCodec(coding);
this.videoDecoder = new VideoDecoder({
output: this._on_decoded_frame.bind(this),
error: this._on_decoder_error.bind(this),
});
+ // ToDo: hardwareAcceleration can be "no-preference" / "prefer-hardware" / "prefer-software"
+ // Figure out when "prefer-hardware" is the right choise and when not (ie: no GPU, remote session like RDP?)
this.videoDecoder.configure({
- codec: "avc1.42C01E",
- // hardwareAcceleration: "prefer-hardware",
+ codec: this.codec,
+ hardwareAcceleration: "no-preference",
optimizeForLatency: true,
- codedWidth: width,
- codedHeight: height,
});
this.last_timestamp = 0;
this.initialized = true;
}
+ resolveCodec(coding) {
+ if (coding == "h264") return "avc1.42C01E";
+ if (coding == "vp8") return "vp8";
+ if (coding == "vp9") return "vp09" + this.vp9_params;
+ throw `No codec defined for coding ${coding}`;
+ }
+
_on_decoded_frame(videoFrame) {
if (this.decoder_queue.length === 0) {
videoFrame.close();
@@ -110,8 +141,9 @@ class XpraVideoDecoder {
}
_on_decoder_error(error) {
- // TODO: Handle err? Or just assume we will catch up?
- this._close();
+ // TODO: Handle error? Or just assume we will catch up?
+ console.error(`Error decoding frame: ${error}`);
+ this.erroneous_frame = true;
}
queue_frame(packet) {
@@ -121,12 +153,11 @@ class XpraVideoDecoder {
const data = packet[7];
const packet_sequence = packet[8];
- if (!this.had_first_key && options["type"] != "IDR") {
- reject(
- new Error(
- `first frame must be a key frame but packet ${packet_sequence} is not.`
- )
- );
+ // H264 (avc1.42C01E) needs key frames
+ if (this.codec.startsWith("avc1")
+ && !this.had_first_key
+ && options["type"] != "IDR") {
+ reject(`first frame must be a key frame but packet ${packet_sequence} is not.`);
return;
}
@@ -154,8 +185,18 @@ class XpraVideoDecoder {
);
while (frame_out.length === 0) {
// Await our frame
- await new Promise((r) => setTimeout(r, 5));
- frame_out = this.decoded_frames.filter((p) => p[8] == packet_sequence);
+ await new Promise(r => setTimeout(r, this.frameWaitTimeout));
+ if (this.erroneous_frame) {
+ // The last frame was erroneous, break the wait loop
+ break;
+ }
+ frame_out = this.decoded_frames.filter(p => p[8] == packet_sequence);
+ }
+
+ if (this.erroneous_frame) {
+ // Last frame was erroneous. Reject the promise and reset the state.
+ this.erroneous_frame = false;
+ reject("decoder error");
}
// Remove the frame from decoded frames list
this.decoded_frames = this.decoded_frames.filter(
@@ -171,17 +212,8 @@ class XpraVideoDecoder {
this.videoDecoder.close();
}
this.had_first_key = false;
-
- // Callback on all frames (bail out)
this.draining = true;
- const drain_queue = this.decoder_queue;
this.decoder_queue = [];
-
- for (const frame of drain_queue) {
- const packet = frame.p;
- this.on_frame_error(packet, "video decoder is draining");
- }
- this.draining = false;
}
this.initialized = false;
}