From a0f50fd7121a246722666314e331210dfc6435df Mon Sep 17 00:00:00 2001 From: totaam Date: Sun, 15 Aug 2021 23:03:58 +0700 Subject: [PATCH] #20 decode h264 using broadway in the worker --- html5/js/Client.js | 11 +++++- html5/js/DecodeWorker.js | 74 ++++++++++++++++++++++++++++++++++++++-- html5/js/Window.js | 22 +++++++----- 3 files changed, 94 insertions(+), 13 deletions(-) diff --git a/html5/js/Client.js b/html5/js/Client.js index 16a51f89..2d93f025 100644 --- a/html5/js/Client.js +++ b/html5/js/Client.js @@ -2535,8 +2535,15 @@ XpraClient.prototype._process_lost_window = function(packet, ctx) { //it had focus, find the next highest: ctx.auto_focus(); } + ctx.decode_worker_eos(wid); }; +XpraClient.prototype.decode_worker_eos = function(wid) { + if (this.decode_worker) { + this.decode_worker.postMessage({'cmd': 'eos', 'wid' : wid}); + } +} + XpraClient.prototype.auto_focus = function() { let highest_window = null; let highest_stacking = -1; @@ -2795,7 +2802,9 @@ XpraClient.prototype.do_process_draw = function(packet) { }; XpraClient.prototype._process_eos = function(packet, ctx) { - ctx._process_draw(packet, ctx); + ctx.do_process_draw(packet); + const wid = packet[1]; + ctx.decode_worker_eos(wid); }; diff --git a/html5/js/DecodeWorker.js b/html5/js/DecodeWorker.js index 3800f654..07d2b394 100644 --- a/html5/js/DecodeWorker.js +++ b/html5/js/DecodeWorker.js @@ -5,6 +5,7 @@ //importScripts("./Decode.js"); importScripts("./lib/zlib.js"); importScripts("./lib/lz4.js"); +importScripts("./lib/broadway/Decoder.js"); // initialise LZ4 library var Buffer = require('buffer').Buffer; @@ -118,7 +119,16 @@ function decode_rgb(packet) { //packet[6] = "bitmap:"+coding; //packet[7] = bitmap; //console.log("converted", orig_data, "to", data); - return packet; +} + +const broadway_decoders = {}; +function close_broadway(wid) { + try { + delete broadway_decoders[wid]; + } + catch (e) { + //not much we can do + } } onmessage = function(e) { @@ -127,6 +137,11 @@ onmessage = function(e) { case 'check': self.postMessage({'result': true}); break; + case 'eos': + const wid = data.wid; + close_broadway(wid); + //console.log("decode worker eos for wid", wid); + break; case 'decode': const packet = data.packet; //console.log("packet to decode:", data.packet); @@ -139,8 +154,8 @@ onmessage = function(e) { try { const coding = packet[6]; if (coding=="rgb24" || coding=="rgb32") { - const decoded = decode_rgb(packet) - send_back(decoded, [decoded[7].buffer]); + decode_rgb(packet) + send_back(packet, [packet[7].buffer]); } else if (coding=="png" || coding=="jpeg" || coding=="webp") { const data = packet[7]; @@ -151,6 +166,59 @@ onmessage = function(e) { send_back(packet, [bitmap]); }, decode_error); } + else if (coding=="h264") { + const wid = packet[1]; + let options = {}; + if (packet.length>10) + options = packet[10]; + let enc_width = packet[4]; + let enc_height = packet[5]; + const data = packet[7]; + const scaled_size = options["scaled_size"]; + if (scaled_size) { + enc_width = scaled_size[0]; + enc_height = scaled_size[1]; + } + const frame = options["frame"] || 0; + if (frame==0) { + close_broadway(); + } + let decoder = broadway_decoders[wid]; + if (decoder && (decoder._enc_size[0]!=enc_width || decoder._enc_size[1]!=enc_height)) { + close_broadway(); + decoder = null; + } + //console.log("decoder=", decoder); + if (!decoder) { + decoder = new Decoder({ + "rgb": true, + "size": { "width" : enc_width, "height" : enc_height }, + }); + decoder._enc_size = [enc_width, enc_height]; + broadway_decoders[wid] = decoder; + } + let count = 0; + decoder.onPictureDecoded = function(buffer, p_width, p_height, infos) { + //console.log("broadway frame: enc size=", enc_width, enc_height, ", decode size=", p_width, p_height); + count++; + //forward it as rgb32: + packet[6] = "rgb32"; + packet[7] = buffer; + options["scaled_size"] = [p_width, p_height]; + send_back(packet, [packet[7].buffer]); + }; + // we can pass a buffer full of NALs to decode() directly + // as long as they are framed properly with the NAL header + if (!Array.isArray(data)) { + img_data = Array.from(data); + } + decoder.decode(data); + // broadway decoding is actually synchronous + // and onPictureDecoded is called from decode(data) above. + if (count==0) { + decode_error("no "+coding+" picture decoded"); + } + } else { //pass-through: send_back(packet, []); diff --git a/html5/js/Window.js b/html5/js/Window.js index eb0fc9de..4ebf91fd 100644 --- a/html5/js/Window.js +++ b/html5/js/Window.js @@ -1439,14 +1439,14 @@ XpraWindow.prototype.do_paint = function paint(x, y, width, height, coding, img_ img_data = inflated.slice(0, uncompressedSize); } } - let target_stride = width*4; + let target_stride = enc_width*4; this.debug("draw", "got ", img_data.length, "bytes of", coding, "to paint with stride", rowstride, ", target stride", target_stride); if (coding=="rgb24") { - const uint = new Uint8Array(target_stride*height); + const uint = new Uint8Array(target_stride*enc_height); let i = 0, j = 0, l = img_data.length; - if (rowstride==width*3) { + if (rowstride==enc_width*3) { //faster path, single loop: while (itarget_stride) { - img = this.offscreen_canvas_ctx.createImageData(Math.round(rowstride/4), height); + img = this.offscreen_canvas_ctx.createImageData(Math.round(rowstride/4), enc_height); } else { - img = this.offscreen_canvas_ctx.createImageData(width, height); + img = this.offscreen_canvas_ctx.createImageData(enc_width, enc_height); } img.data.set(img_data); - this.offscreen_canvas_ctx.clearRect(x, y, width, height); - this.offscreen_canvas_ctx.putImageData(img, x, y, 0, 0, width, height); + //this.offscreen_canvas_ctx.clearRect(x, y, width, height); + this.offscreen_canvas_ctx.putImageData(img, x, y, 0, 0, enc_width, enc_height); + if (enc_width!=width || enc_height!=height) { + //scale it: + this.offscreen_canvas_ctx.drawImage(this.offscreen_canvas, x, y, enc_width, enc_height, x, y, width, height); + } painted(); this.may_paint_now(); }