diff --git a/html5/index.html b/html5/index.html index 639b81d7..869c75f0 100644 --- a/html5/index.html +++ b/html5/index.html @@ -918,6 +918,10 @@

Xpra Bug Report

if (video) { // broadway h264 decoder: client.check_encodings.push("h264"); + // VPX checks. Will only succeed if we can use VideoDecoder.js + client.check_encodings.push("vp8"); + client.check_encodings.push("vp9"); + if (JSMpeg && JSMpeg.Renderer && JSMpeg.Decoder) { //TODO: should be moved to 'check_encodings' //and added to the decode worker: diff --git a/html5/js/Client.js b/html5/js/Client.js index 2d64e616..2e6d547c 100644 --- a/html5/js/Client.js +++ b/html5/js/Client.js @@ -1584,6 +1584,8 @@ class XpraClient { "YUV422P", "YUV444P", ], + vp8: ["YUV420P"], + vp9: ["YUV420P", "YUV444P", "YUV444P10"], }, //this is a workaround for server versions between 2.5.0 to 2.5.2 only: "encoding.x264.YUV420P.profile": "baseline", @@ -1594,12 +1596,14 @@ class XpraClient { "encoding.h264.fast-decode": true, "encoding.h264+mp4.YUV420P.profile": "baseline", "encoding.h264+mp4.YUV420P.level": "3.0", - //prefer native video in mp4/webm container to broadway plain h264: - "encoding.h264.score-delta": -20, + //prefer unmuxed VPX + "encoding.vp8.score-delta": 70, + "encoding.vp9.score-delta": 60, "encoding.h264+mp4.score-delta": 50, "encoding.h264+mp4.": 50, "encoding.mpeg4+mp4.score-delta": 40, "encoding.vp8+webm.score-delta": 40, + "encoding.h264.score-delta": -20, "sound.receive": true, "sound.send": false, diff --git a/html5/js/OffscreenDecodeWorker.js b/html5/js/OffscreenDecodeWorker.js index 9550977f..d17fcd68 100644 --- a/html5/js/OffscreenDecodeWorker.js +++ b/html5/js/OffscreenDecodeWorker.js @@ -34,8 +34,10 @@ const image_coding = [ ]; const video_coding = []; if (XpraVideoDecoderLoader.hasNativeDecoder) { - // We can support native H264 decoding + // We can support native H264 & VP8 decoding video_coding.push("h264"); + video_coding.push("vp8"); + video_coding.push("vp9"); } else { console.warn( "Offscreen decoding is available for images only. Please consider using Google Chrome 94+ in a secure (SSL or localhost) context h264 offscreen decoding support." @@ -112,10 +114,14 @@ class WindowDecoder { } else if (image_coding.includes(coding)) { await this.image_decoder.convertToBitmap(packet); } else if (video_coding.includes(coding)) { + if (coding == "vp9") { + // Prepare the VP9 codec (if needed) + const csc = packet[10]["csc"]; + this.video_decoder.prepareVP9params(csc); + } + if (!this.video_decoder.initialized) { - // Init with width and heigth of this packet. - // TODO: Use video max-size? It does not seem to matter. - this.video_decoder.init(packet[4], packet[5]); + this.video_decoder.init(coding); } await this.video_decoder.queue_frame(packet); } else { diff --git a/html5/js/VideoDecoder.js b/html5/js/VideoDecoder.js index 290518db..39cb648d 100644 --- a/html5/js/VideoDecoder.js +++ b/html5/js/VideoDecoder.js @@ -29,30 +29,61 @@ class XpraVideoDecoder { this.decoder_queue = []; this.decoded_frames = []; + this.erroneous_frame = false; + + this.codec = null; + this.vp9_params = null; + this.frameWaitTimeout = 1; // Interval for wait loop while frame is being decoded this.frame_threshold = 250; - this.on_frame_error = (packet, error) => { - console.error("VideoDecoder error on packet", packet, ":", error); - }; + } - init(width, height) { + prepareVP9params(csc) { + console.log(csc); + // Check if we have the right VP9 params before init. + // This is needed because we can only set the params when the video decoder is created. + // We close the decoder when the coding changes. + if (csc == "YUV444P" && this.vp9_params != ".01.10.08") { + this.vp9_params = ".01.10.08"; + this._close(); + } else if (csc == "YUV444P10" && this.vp9_params != ".03.10.10") { + this.vp9_params = ".03.10.10"; + this._close(); + } else if (this.vp9_params != ".00.20.08.01.02.02") { + // chroma shift for YUV420P, both X and Y are downscaled by 2^1 + this.vp9_params = ".00.20.08.01.02.02"; + this._close(); + } + } + + init(coding) { + this.draining = false; + + this.codec = this.resolveCodec(coding); this.videoDecoder = new VideoDecoder({ output: this._on_decoded_frame.bind(this), error: this._on_decoder_error.bind(this), }); + // ToDo: hardwareAcceleration can be "no-preference" / "prefer-hardware" / "prefer-software" + // Figure out when "prefer-hardware" is the right choise and when not (ie: no GPU, remote session like RDP?) this.videoDecoder.configure({ - codec: "avc1.42C01E", - // hardwareAcceleration: "prefer-hardware", + codec: this.codec, + hardwareAcceleration: "no-preference", optimizeForLatency: true, - codedWidth: width, - codedHeight: height, }); this.last_timestamp = 0; this.initialized = true; } + resolveCodec(coding) { + if (coding == "h264") return "avc1.42C01E"; + if (coding == "vp8") return "vp8"; + if (coding == "vp9") return "vp09" + this.vp9_params; + throw `No codec defined for coding ${coding}`; + } + _on_decoded_frame(videoFrame) { if (this.decoder_queue.length === 0) { videoFrame.close(); @@ -110,8 +141,9 @@ class XpraVideoDecoder { } _on_decoder_error(error) { - // TODO: Handle err? Or just assume we will catch up? - this._close(); + // TODO: Handle error? Or just assume we will catch up? + console.error(`Error decoding frame: ${error}`); + this.erroneous_frame = true; } queue_frame(packet) { @@ -121,12 +153,11 @@ class XpraVideoDecoder { const data = packet[7]; const packet_sequence = packet[8]; - if (!this.had_first_key && options["type"] != "IDR") { - reject( - new Error( - `first frame must be a key frame but packet ${packet_sequence} is not.` - ) - ); + // H264 (avc1.42C01E) needs key frames + if (this.codec.startsWith("avc1") + && !this.had_first_key + && options["type"] != "IDR") { + reject(`first frame must be a key frame but packet ${packet_sequence} is not.`); return; } @@ -154,8 +185,18 @@ class XpraVideoDecoder { ); while (frame_out.length === 0) { // Await our frame - await new Promise((r) => setTimeout(r, 5)); - frame_out = this.decoded_frames.filter((p) => p[8] == packet_sequence); + await new Promise(r => setTimeout(r, this.frameWaitTimeout)); + if (this.erroneous_frame) { + // The last frame was erroneous, break the wait loop + break; + } + frame_out = this.decoded_frames.filter(p => p[8] == packet_sequence); + } + + if (this.erroneous_frame) { + // Last frame was erroneous. Reject the promise and reset the state. + this.erroneous_frame = false; + reject("decoder error"); } // Remove the frame from decoded frames list this.decoded_frames = this.decoded_frames.filter( @@ -171,17 +212,8 @@ class XpraVideoDecoder { this.videoDecoder.close(); } this.had_first_key = false; - - // Callback on all frames (bail out) this.draining = true; - const drain_queue = this.decoder_queue; this.decoder_queue = []; - - for (const frame of drain_queue) { - const packet = frame.p; - this.on_frame_error(packet, "video decoder is draining"); - } - this.draining = false; } this.initialized = false; }