#20 decode h264 using broadway in the worker

html5/js/Client.js

@@ -2535,8 +2535,15 @@ XpraClient.prototype._process_lost_window = function(packet, ctx) {
 		//it had focus, find the next highest:
 		ctx.auto_focus();
 	}
+	ctx.decode_worker_eos(wid);
 };
 
+XpraClient.prototype.decode_worker_eos = function(wid) {
+	if (this.decode_worker) {
+		this.decode_worker.postMessage({'cmd': 'eos', 'wid' : wid});
+	}
+}
+
 XpraClient.prototype.auto_focus = function() {
 	let highest_window = null;
 	let highest_stacking = -1;
@@ -2795,7 +2802,9 @@ XpraClient.prototype.do_process_draw = function(packet) {
 };
 
 XpraClient.prototype._process_eos = function(packet, ctx) {
-	ctx._process_draw(packet, ctx);
+	ctx.do_process_draw(packet);
+	const wid = packet[1];
+	ctx.decode_worker_eos(wid);
 };
 
 

html5/js/DecodeWorker.js

@@ -5,6 +5,7 @@
 //importScripts("./Decode.js");
 importScripts("./lib/zlib.js");
 importScripts("./lib/lz4.js");
+importScripts("./lib/broadway/Decoder.js");
 
 // initialise LZ4 library
 var Buffer = require('buffer').Buffer;
@@ -118,7 +119,16 @@ function decode_rgb(packet) {
 	//packet[6] = "bitmap:"+coding;
 	//packet[7] = bitmap;
 	//console.log("converted", orig_data, "to", data);
-	return packet;
+}
+
+const broadway_decoders = {};
+function close_broadway(wid) {
+	try {
+		delete broadway_decoders[wid];
+	}
+	catch (e) {
+		//not much we can do
+	}
 }
 
 onmessage = function(e) {
@@ -127,6 +137,11 @@ onmessage = function(e) {
 	case 'check':
 		self.postMessage({'result': true});
 		break;
+	case 'eos':
+		const wid = data.wid;
+		close_broadway(wid);
+		//console.log("decode worker eos for wid", wid);
+		break;
 	case 'decode':
 		const packet = data.packet;
 		//console.log("packet to decode:", data.packet);
@@ -139,8 +154,8 @@ onmessage = function(e) {
 		try {
 			const coding = packet[6];
 			if (coding=="rgb24" || coding=="rgb32") {
-				const decoded = decode_rgb(packet)
-				send_back(decoded, [decoded[7].buffer]);
+				decode_rgb(packet)
+				send_back(packet, [packet[7].buffer]);
 			}
 			else if (coding=="png" || coding=="jpeg" || coding=="webp") {
 				const data = packet[7];
@@ -151,6 +166,59 @@ onmessage = function(e) {
 					send_back(packet, [bitmap]);
 				}, decode_error);
 			}
+			else if (coding=="h264") {
+				const wid = packet[1];
+				let options = {};
+				if (packet.length>10)
+					options = packet[10];
+				let enc_width = packet[4];
+				let enc_height = packet[5];
+				const data = packet[7];
+				const scaled_size = options["scaled_size"];
+				if (scaled_size) {
+					enc_width = scaled_size[0];
+					enc_height = scaled_size[1];
+				}
+				const frame = options["frame"] || 0;
+				if (frame==0) {
+					close_broadway();
+				}
+				let decoder = broadway_decoders[wid];
+				if (decoder && (decoder._enc_size[0]!=enc_width || decoder._enc_size[1]!=enc_height)) {
+					close_broadway();
+					decoder = null;
+				}
+				//console.log("decoder=", decoder);
+				if (!decoder) {
+					decoder = new Decoder({
+						"rgb": 	true,
+						"size": { "width" : enc_width, "height" : enc_height },
+					});
+					decoder._enc_size = [enc_width, enc_height];
+					broadway_decoders[wid] = decoder;
+				}
+				let count = 0;
+				decoder.onPictureDecoded = function(buffer, p_width, p_height, infos) {
+					//console.log("broadway frame: enc size=", enc_width, enc_height, ", decode size=", p_width, p_height);
+					count++;
+					//forward it as rgb32:
+					packet[6] = "rgb32";
+					packet[7] = buffer;
+					options["scaled_size"] = [p_width, p_height];
+					send_back(packet, [packet[7].buffer]);
+				};
+				// we can pass a buffer full of NALs to decode() directly
+				// as long as they are framed properly with the NAL header
+				if (!Array.isArray(data)) {
+					img_data = Array.from(data);
+				}
+				decoder.decode(data);
+				// broadway decoding is actually synchronous
+				// and onPictureDecoded is called from decode(data) above.
+				if (count==0) {
+					decode_error("no "+coding+" picture decoded");
+				}
+			}
 			else {
 				//pass-through:
 				send_back(packet, []);

html5/js/Window.js

@@ -1439,14 +1439,14 @@ XpraWindow.prototype.do_paint = function paint(x, y, width, height, coding, img_
 					img_data = inflated.slice(0, uncompressedSize);
 				}
 			}
-			let target_stride = width*4;
+			let target_stride = enc_width*4;
 			this.debug("draw", "got ", img_data.length, "bytes of", coding, "to paint with stride", rowstride, ", target stride", target_stride);
 			if (coding=="rgb24") {
-				const uint = new Uint8Array(target_stride*height);
+				const uint = new Uint8Array(target_stride*enc_height);
 				let i = 0,
 					j = 0,
 					l = img_data.length;
-				if (rowstride==width*3) {
+				if (rowstride==enc_width*3) {
 					//faster path, single loop:
 					while (i<l) {
 						uint[j++] = img_data[i++];
@@ -1458,9 +1458,9 @@ XpraWindow.prototype.do_paint = function paint(x, y, width, height, coding, img_
 				else {
 					let psrc = 0,
 						pdst = 0;
-					for (i=0; i<height; i++) {
+					for (i=0; i<enc_height; i++) {
 						psrc = i*rowstride;
-						for (j=0; j<width; j++) {
+						for (j=0; j<enc_width; j++) {
 							uint[pdst++] = img_data[psrc++];
 							uint[pdst++] = img_data[psrc++];
 							uint[pdst++] = img_data[psrc++];
@@ -1473,14 +1473,18 @@ XpraWindow.prototype.do_paint = function paint(x, y, width, height, coding, img_
 			}
 			let img = null;
 			if (rowstride>target_stride) {
-				img = this.offscreen_canvas_ctx.createImageData(Math.round(rowstride/4), height);
+				img = this.offscreen_canvas_ctx.createImageData(Math.round(rowstride/4), enc_height);
 			}
 			else {
-				img = this.offscreen_canvas_ctx.createImageData(width, height);
+				img = this.offscreen_canvas_ctx.createImageData(enc_width, enc_height);
 			}
 			img.data.set(img_data);
-			this.offscreen_canvas_ctx.clearRect(x, y, width, height);
-			this.offscreen_canvas_ctx.putImageData(img, x, y, 0, 0, width, height);
+			//this.offscreen_canvas_ctx.clearRect(x, y, width, height);
+			this.offscreen_canvas_ctx.putImageData(img, x, y, 0, 0, enc_width, enc_height);
+			if (enc_width!=width || enc_height!=height) {
+				//scale it:
+				this.offscreen_canvas_ctx.drawImage(this.offscreen_canvas, x, y, enc_width, enc_height, x, y, width, height);
+			}
 			painted();
 			this.may_paint_now();
 		}