Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Native VPX decoding in HTML5 #200

Merged
merged 5 commits into from
Aug 1, 2022
Merged

[WIP] Native VPX decoding in HTML5 #200

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3e040de
VPX
Jul 21, 2022
37b9cc2
trailing comma
Aug 1, 2022
9dc4746
downmerg
Aug 1, 2022
aa207a5
Merge branch 'vp8' into vp8_downmerge
TijZwa Aug 1, 2022
6a1b986
Merge pull request #2 from Xpra-org/vp8_downmerge
TijZwa Aug 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions html5/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,10 @@ <h2>Xpra Bug Report</h2>
if (video) {
// broadway h264 decoder:
client.check_encodings.push("h264");
// VPX checks. Will only succeed if we can use VideoDecoder.js
client.check_encodings.push("vp8");
client.check_encodings.push("vp9");

if (JSMpeg && JSMpeg.Renderer && JSMpeg.Decoder) {
//TODO: should be moved to 'check_encodings'
//and added to the decode worker:
Expand Down
18 changes: 13 additions & 5 deletions html5/js/Client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class XpraClient {
"rgb32",
"rgb24",
"scroll",
"void",
"void"
TijZwa marked this conversation as resolved.
Show resolved Hide resolved
];
//extra encodings we enable if validated via the decode worker:
//(we also validate jpeg and png as a sanity check)
Expand All @@ -94,7 +94,7 @@ class XpraClient {
"scroll",
"webp",
"void",
"avif",
"avif"
TijZwa marked this conversation as resolved.
Show resolved Hide resolved
];
this.debug_categories = [];
this.start_new_session = null;
Expand Down Expand Up @@ -1583,6 +1583,8 @@ class XpraClient {
"YUV422P",
"YUV444P",
],
vp8: ["YUV420P"],
vp9: ["YUV420P", "YUV444P", "YUV444P10"],
},
//this is a workaround for server versions between 2.5.0 to 2.5.2 only:
"encoding.x264.YUV420P.profile": "baseline",
Expand All @@ -1593,12 +1595,14 @@ class XpraClient {
"encoding.h264.fast-decode": true,
"encoding.h264+mp4.YUV420P.profile": "baseline",
"encoding.h264+mp4.YUV420P.level": "3.0",
//prefer native video in mp4/webm container to broadway plain h264:
"encoding.h264.score-delta": -20,
//prefer unmuxed VPX
"encoding.vp8.score-delta": 70,
"encoding.vp9.score-delta": 60,
"encoding.h264+mp4.score-delta": 50,
"encoding.h264+mp4.": 50,
"encoding.mpeg4+mp4.score-delta": 40,
"encoding.vp8+webm.score-delta": 40,
"encoding.h264.score-delta": -20,

"sound.receive": true,
"sound.send": false,
Expand Down Expand Up @@ -1703,7 +1707,11 @@ class XpraClient {
}

on_mousemove(e, window) {
if (this.server_readonly || this.mouse_grabbed || !this.connected) {
if (this.mouse_grabbed) {
return true;
}

if (this.server_readonly || !this.connected) {
totaam marked this conversation as resolved.
Show resolved Hide resolved
return window == undefined;
}
const mouse = this.getMouse(e);
Expand Down
15 changes: 10 additions & 5 deletions html5/js/OffscreenDecodeWorker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,10 @@ const image_coding = [
];
const video_coding = [];
if (XpraVideoDecoderLoader.hasNativeDecoder) {
// We can support native H264 decoding
// We can support native H264 & VP8 decoding
video_coding.push("h264");
video_coding.push("vp8");
video_coding.push("vp9");
} else {
console.warn("Offscreen decoding is available for images only. Please consider using Google Chrome 94+ in a secure (SSL or localhost) context h264 offscreen decoding support.");
}
Expand Down Expand Up @@ -103,7 +105,6 @@ class WindowDecoder {

async proccess_packet(packet) {
let coding = packet[6];

const start = performance.now();
if (coding == "eos" && this.video_decoder) {
this.video_decoder._close();
Expand All @@ -112,10 +113,14 @@ class WindowDecoder {
else if (image_coding.includes(coding) && !(coding == "scroll" || coding == "void")) {
await this.image_decoder.convertToBitmap(packet);
} else if (video_coding.includes(coding)) {
if (coding == "vp9") {
// Prepare the VP9 codec (if needed)
const csc = packet[10]["csc"];
this.video_decoder.prepareVP9params(csc);
}

if (!this.video_decoder.initialized) {
// Init with width and heigth of this packet.
// TODO: Use video max-size? It does not seem to matter.
this.video_decoder.init(packet[4], packet[5]);
this.video_decoder.init(coding);
}
await this.video_decoder.queue_frame(packet);
} else {
Expand Down
79 changes: 58 additions & 21 deletions html5/js/VideoDecoder.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,30 +29,61 @@ class XpraVideoDecoder {

this.decoder_queue = [];
this.decoded_frames = [];
this.erroneous_frame = false;

this.codec = null;
this.vp9_params = null;
this.frameWaitTimeout = 1; // Interval for wait loop while frame is being decoded

this.frame_threshold = 250;
this.on_frame_error = (packet, error) => {
console.error("VideoDecoder error on packet", packet, ":", error);
};

}

prepareVP9params(csc) {
console.log(csc);
// Check if we have the right VP9 params before init.
// This is needed because we can only set the params when the video decoder is created.
// We close the decoder when the coding changes.
if (csc == "YUV444P" && this.vp9_params != ".01.10.08") {
this.vp9_params = ".01.10.08";
this._close();
} else if (csc == "YUV444P10" && this.vp9_params != ".03.10.10") {
this.vp9_params = ".03.10.10";
this._close();
} else if (this.vp9_params != ".00.20.08.01.02.02") {
// chroma shift for YUV420P, both X and Y are downscaled by 2^1
this.vp9_params = ".00.20.08.01.02.02";
this._close();
}
}

init(width, height) {
init(coding) {
this.draining = false;

this.codec = this.resolveCodec(coding);
this.videoDecoder = new VideoDecoder({
output: this._on_decoded_frame.bind(this),
error: this._on_decoder_error.bind(this),
});

// ToDo: hardwareAcceleration can be "no-preference" / "prefer-hardware" / "prefer-software"
// Figure out when "prefer-hardware" is the right choise and when not (ie: no GPU, remote session like RDP?)
this.videoDecoder.configure({
codec: "avc1.42C01E",
// hardwareAcceleration: "prefer-hardware",
codec: this.codec,
hardwareAcceleration: "no-preference",
optimizeForLatency: true,
codedWidth: width,
codedHeight: height,
});
this.last_timestamp = 0;
this.initialized = true;
}

resolveCodec(coding) {
if (coding == "h264") return "avc1.42C01E";
if (coding == "vp8") return "vp8";
if (coding == "vp9") return "vp09" + this.vp9_params;
throw `No codec defined for coding ${coding}`;
}

_on_decoded_frame(videoFrame) {
if (this.decoder_queue.length === 0) {
videoFrame.close();
Expand Down Expand Up @@ -111,8 +142,9 @@ class XpraVideoDecoder {
}

_on_decoder_error(error) {
// TODO: Handle err? Or just assume we will catch up?
this._close();
// TODO: Handle error? Or just assume we will catch up?
console.error(`Error decoding frame: ${error}`);
this.erroneous_frame = true;
}

queue_frame(packet) {
Expand All @@ -121,7 +153,10 @@ class XpraVideoDecoder {
const data = packet[7];
const packet_sequence = packet[8];

if (!this.had_first_key && options["type"] != "IDR") {
// H264 (avc1.42C01E) needs key frames
if (this.codec.startsWith("avc1")
&& !this.had_first_key
&& options["type"] != "IDR") {
reject(`first frame must be a key frame but packet ${packet_sequence} is not.`);
return;
}
Expand All @@ -148,9 +183,20 @@ class XpraVideoDecoder {
let frame_out = this.decoded_frames.filter(p => p[8] == packet_sequence);
while (frame_out.length == 0) {
// Await our frame
await new Promise(r => setTimeout(r, 5));
await new Promise(r => setTimeout(r, this.frameWaitTimeout));
if (this.erroneous_frame) {
// The last frame was erroneous, break the wait loop
break;
}
frame_out = this.decoded_frames.filter(p => p[8] == packet_sequence);
}

if (this.erroneous_frame) {
// Last frame was erroneous. Reject the promise and reset the state.
this.erroneous_frame = false;
reject("decoder error");
}

// Remove the frame from decoded frames list
this.decoded_frames = this.decoded_frames.filter(p => p[8] != packet_sequence);

Expand All @@ -164,17 +210,8 @@ class XpraVideoDecoder {
this.videoDecoder.close();
}
this.had_first_key = false;

// Callback on all frames (bail out)
this.draining = true;
const drain_queue = this.decoder_queue;
this.decoder_queue = [];

for (const frame of drain_queue) {
const packet = frame.p;
this.on_frame_error(packet, "video decoder is draining");
}
this.draining = false;
}
this.initialized = false;
}
Expand Down