From 96347b7a9aa3931faddba11ee623ef7ba3c5e0b5 Mon Sep 17 00:00:00 2001
From: totaam <antoine@xpra.org>
Date: Tue, 17 Aug 2021 14:24:58 +0700
Subject: [PATCH] preserve paint order when using the decode worker (#20)

---
 html5/js/Client.js       | 125 ++++++++++++++++++++-------------------
 html5/js/DecodeWorker.js |  69 +++++++++++++++++++--
 html5/js/Window.js       |   7 +++
 3 files changed, 134 insertions(+), 67 deletions(-)

diff --git a/html5/js/Client.js b/html5/js/Client.js
index 1a4a8572..077bf42c 100644
--- a/html5/js/Client.js
+++ b/html5/js/Client.js
@@ -380,72 +380,75 @@ XpraClient.prototype.connect = function() {
 			return;
 		}
 	}
+	this.initialize_workers();
+}
+
+XpraClient.prototype.initialize_workers = function() {
 	// detect websocket in webworker support and degrade gracefully
-	if(window.Worker) {
-		this.clog("we have webworker support");
-		// spawn worker that checks for a websocket
-		const me = this;
-		const worker = new Worker('js/lib/wsworker_check.js');
-		worker.addEventListener('message', function(e) {
-			const data = e.data;
-			switch (data['result']) {
-			case true:
-				// yey, we can use websocket in worker!
-				me.clog("we can use websocket in webworker");
-				me._do_connect(true);
-				break;
-			case false:
-				me.clog("we can't use websocket in webworker, won't use webworkers");
-				me._do_connect(false);
-				break;
-			default:
-				me.clog("client got unknown message from worker");
-				me._do_connect(false);
-			}
-		}, false);
-		// ask the worker to check for websocket support, when we receive a reply
-		// through the eventlistener above, _do_connect() will finish the job
-		worker.postMessage({'cmd': 'check'});
-
-		const decode_worker = new Worker('js/DecodeWorker.js');
-		decode_worker.addEventListener('message', function(e) {
-			const data = e.data;
-			if (data['draw']) {
-				me.do_process_draw(data['draw']);
-				return;
-			}
-			if (data['error']) {
-				const msg = data['error'],
-					packet = data['packet'],
-					wid = packet[1],
-					width = packet[2],
-					height = packet[3],
-					coding = packet[6],
-					packet_sequence = packet[8];
-				me.clog("decode error on ", coding, "packet sequence", packet_sequence, ":", msg);
-				me.do_send_damage_sequence(packet_sequence, wid, width, height, -1, msg);
-				return;
-			}
-			switch (data['result']) {
-			case true:
-				me.clog("we can decode using a worker");
-				me.decode_worker = decode_worker;
-				break;
-			case false:
-				me.clog("we can't decode using a worker");
-				break;
-			default:
-				me.clog("client got unknown message from the decode worker");
-			}
-		}, false);
-		// ask the worker to check for websocket support, when we receive a reply
-		// through the eventlistener above, _do_connect() will finish the job
-		decode_worker.postMessage({'cmd': 'check'});
-	} else {
+	if (!window.Worker) {
 		// no webworker support
 		this.clog("no webworker support at all.");
 		this._do_connect(false);
 	}
+	this.clog("we have webworker support");
+	// spawn worker that checks for a websocket
+	const me = this;
+	const worker = new Worker('js/lib/wsworker_check.js');
+	worker.addEventListener('message', function(e) {
+		const data = e.data;
+		switch (data['result']) {
+		case true:
+			// yey, we can use websocket in worker!
+			me.clog("we can use websocket in webworker");
+			me._do_connect(true);
+			break;
+		case false:
+			me.clog("we can't use websocket in webworker, won't use webworkers");
+			me._do_connect(false);
+			break;
+		default:
+			me.clog("client got unknown message from worker");
+			me._do_connect(false);
+		}
+	}, false);
+	// ask the worker to check for websocket support, when we receive a reply
+	// through the eventlistener above, _do_connect() will finish the job
+	worker.postMessage({'cmd': 'check'});
+
+	const decode_worker = new Worker('js/DecodeWorker.js');
+	decode_worker.addEventListener('message', function(e) {
+		const data = e.data;
+		if (data['draw']) {
+			me.do_process_draw(data['draw']);
+			return;
+		}
+		if (data['error']) {
+			const msg = data['error'],
+				packet = data['packet'],
+				wid = packet[1],
+				width = packet[2],
+				height = packet[3],
+				coding = packet[6],
+				packet_sequence = packet[8];
+			me.clog("decode error on ", coding, "packet sequence", packet_sequence, ":", msg);
+			me.do_send_damage_sequence(packet_sequence, wid, width, height, -1, msg);
+			return;
+		}
+		switch (data['result']) {
+		case true:
+			me.clog("we can decode using a worker");
+			me.decode_worker = decode_worker;
+			break;
+		case false:
+			me.clog("we can't decode using a worker");
+			break;
+		default:
+			me.clog("client got unknown message from the decode worker");
+		}
+	}, false);
+	// ask the worker to check for websocket support, when we receive a reply
+	// through the eventlistener above, _do_connect() will finish the job
+	decode_worker.postMessage({'cmd': 'check'});
 };
 
 XpraClient.prototype._do_connect = function(with_worker) {
diff --git a/html5/js/DecodeWorker.js b/html5/js/DecodeWorker.js
index f359becd..78ffd534 100644
--- a/html5/js/DecodeWorker.js
+++ b/html5/js/DecodeWorker.js
@@ -101,6 +101,8 @@ function close_broadway(wid) {
 	}
 }
 
+const on_hold = new Map();
+
 onmessage = function(e) {
 	var data = e.data;
 	switch (data.cmd) {
@@ -108,21 +110,46 @@ onmessage = function(e) {
 		self.postMessage({'result': true});
 		break;
 	case 'eos':
-		const wid = data.wid;
-		close_broadway(wid);
+		close_broadway(data.wid);
+		if (data.wid in on_hold) {
+			on_hold.remove(data.wid);
+		}
 		//console.log("decode worker eos for wid", wid);
 		break;
 	case 'decode':
-		const packet = data.packet;
+		const packet = data.packet,
+			wid = packet[1],
+			coding = packet[6],
+			packet_sequence = packet[8];
+		let wid_hold = on_hold.get(wid);
 		//console.log("packet to decode:", data.packet);
 		function send_back(raw_buffers) {
+			//console.log("send_back: wid_hold=", wid_hold);
+			if (wid_hold) {
+				//find the highest sequence number which is still lower than this packet
+				let seq_holding = 0;
+				for (let seq of wid_hold.keys()) {
+					if (seq>seq_holding && seq<packet_sequence) {
+						seq_holding = seq;
+					}
+				}
+				if (seq_holding) {
+					const held = wid_hold.get(seq_holding);
+					if (held) {
+						held.push([packet, raw_buffers]);
+						return;
+					}
+				}
+			}
 			self.postMessage({'draw': packet}, raw_buffers);
 		}
+		function do_send_back(p, raw_buffers) {
+			self.postMessage({'draw': p}, raw_buffers);
+		}
 		function decode_error(msg) {
 			self.postMessage({'error': msg, 'packet' : packet});
 		}
 		try {
-			const coding = packet[6];
 			if (coding=="rgb24" || coding=="rgb32") {
 				decode_rgb(packet)
 				send_back([packet[7].buffer]);
@@ -130,14 +157,44 @@ onmessage = function(e) {
 			else if (coding=="png" || coding=="jpeg" || coding=="webp") {
 				const data = packet[7];
 				const blob = new Blob([data.buffer]);
+				//we're loading asynchronously
+				//so ensure that any packet sequence arriving after this one will be put on hold
+				//until we have finished decoding this one:
+				if (!wid_hold) {
+					wid_hold = new Map();
+					on_hold.set(wid, wid_hold);
+				}
+				//console.log("holding=", packet_sequence);
+				wid_hold[packet_sequence] = [];
+				function release() {
+					//release any packets held back by this image:
+					const held = wid_hold[packet_sequence];
+					//console.log("release held=", held);
+					if (held) {
+						let i;
+						for (i=0; i<held.length; i++) {
+							const held_packet = held[i][0];
+							const held_raw_buffers = held[i][1];
+							do_send_back(held_packet, held_raw_buffers);
+						}
+						wid_hold.delete(packet_sequence);
+						//console.log("wid_hold=", wid_hold, "on_hold=", on_hold);
+						if (wid_hold.size==0 && on_hold.has(wid)) {
+							on_hold.delete(wid);
+						}
+					}
+				}
 				createImageBitmap(blob).then(function(bitmap) {
 					packet[6] = "bitmap:"+coding;
 					packet[7] = bitmap;
 					send_back([bitmap]);
-				}, decode_error);
+					release();
+				}, function(e) {
+					decode_error(e);
+					release();
+				});
 			}
 			else if (coding=="h264") {
-				const wid = packet[1];
 				let options = {};
 				if (packet.length>10)
 					options = packet[10];
diff --git a/html5/js/Window.js b/html5/js/Window.js
index 6d754bb2..ba287c6b 100644
--- a/html5/js/Window.js
+++ b/html5/js/Window.js
@@ -1303,6 +1303,13 @@ XpraWindow.prototype._non_video_paint = function(coding) {
  * The image is painted into off-screen canvas.
  */
 XpraWindow.prototype.paint = function paint() {
+	if (this.client.decode_worker) {
+		//no need to synchronize paint packets here
+		//the decode worker ensures that we get the packets
+		//in the correct order, ready to update the canvas
+		XpraWindow.prototype.do_paint.apply(this, arguments);
+		return;
+	}
 	//process all paint request in order using the paint_queue:
 	const item = Array.prototype.slice.call(arguments);
 	this.paint_queue.push(item);