Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix: HA & Demux - Audio Increment Not Respecting Position Diff-Threshold #301

Merged
merged 4 commits into from
Nov 1, 2023
Merged

Fix: HA & Demux - Audio Increment Not Respecting Position Diff-Threshold #301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b022db8
chore: first version of seperate fn for audio increment + unit tests
Nfrederiksen Oct 27, 2023
c21f006
fix: HA demux audio incr not in sync
Nfrederiksen Nov 1, 2023
5089dde
test: for new increment function
Nfrederiksen Nov 1, 2023
22c9df2
test: add more cases
Nfrederiksen Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 76 additions & 40 deletions engine/session.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -834,51 +834,51 @@ class Session {
}
} else {
sessionState.vodMediaSeqVideo = await this._sessionState.increment("vodMediaSeqVideo", 1);
let audioIncrement;
let playheadPosVideoMs;
let playheadAudio;
let playheadSubtitle;
if (this.use_demuxed_audio || this.use_vtt_subtitles) {
playheadPosVideoMs = (await this._getCurrentPlayheadPosition()) * 1000;
}

if (this.use_demuxed_audio) {
let positionV = 0;
let positionA = 0;
const position = (await this._getCurrentPlayheadPosition()) * 1000;
positionV = position ? position / 1000 : 0;
let currentVod = await this._sessionState.getCurrentVod();
const sessionState = await this._sessionState.getValues(["vodMediaSeqAudio"]);
let posDiff;
let incrementValue = 1;
let index = 0;
const audioSeqLastIdx = currentVod.getLiveMediaSequencesCount("audio") - 1;
const thresh = 0.5;
const maxAcceptableDiff = 0.001;
debug(`[${this._sessionId}]: About to determine audio increment`);
do {
const audioPosition = (await this._getAudioPlayheadPosition(sessionState.vodMediaSeqAudio + index)) * 1000;
positionA = audioPosition ? audioPosition / 1000 : 0;
posDiff = (positionV - positionA).toFixed(3);
debug(`[${this._sessionId}]: positionV=${positionV};positionA=${positionA};posDiff=${posDiff}`);
if (posDiff <= maxAcceptableDiff) {
break;
}
if (posDiff > thresh) {
index++;
incrementValue++;
} else if (posDiff > maxAcceptableDiff) {
index = incrementValue;
debug(`[${this._sessionId}]: Audio Stepping index set to = ${index}`);
break;
}
if (sessionState.vodMediaSeqAudio + index > audioSeqLastIdx) {
break;
}
} while (!(-thresh < posDiff && posDiff < thresh) && !isNaN(posDiff));
audioIncrement = index;
debug(`[${this._sessionId}]: Current VOD Playhead Positions are to be: [${positionV.toFixed(3)}][${positionA.toFixed(3)}] (${posDiff})`);
const sessionStateObj = await this._sessionState.getValues(["vodMediaSeqAudio"]);
playheadAudio = await this._determineExtraMediaIncrement(
"audio",
playheadPosVideoMs,
audioSeqLastIdx,
sessionStateObj.vodMediaSeqAudio,
this._getAudioPlayheadPosition.bind(this)
);
// Perform the Increment
debug(`[${this._sessionId}]: Will increment audio with ${playheadAudio.increment}`);
sessionState.vodMediaSeqAudio = await this._sessionState.increment("vodMediaSeqAudio", playheadAudio.increment);
}
debug(`[${this._sessionId}]: Will increment audio with ${audioIncrement}`);
sessionState.vodMediaSeqAudio = await this._sessionState.increment("vodMediaSeqAudio", audioIncrement);

if (this.use_vtt_subtitles) {
debug(`[${this._sessionId}]: Will increment subtitle with 1`);
sessionState.vodMediaSeqSubtitle = await this._sessionState.increment("vodMediaSeqSubtitle", 1);
const playheadPosVideo = playheadPosVideo || (await this._getCurrentPlayheadPosition()) * 1000;
const subtitleSeqLastIdx = currentVod.getLiveMediaSequencesCount("subtitle") - 1;
const sessionStateObj = await this._sessionState.getValues(["vodMediaSeqSubtitle"]);
playheadSubtitle = await this._determineExtraMediaIncrement(
"subtitle",
playheadPosVideoMs,
subtitleSeqLastIdx,
sessionStateObj.vodMediaSeqSubtitle,
this._getSubtitlePlayheadPosition.bind(this)
);
// Perform the Increment
debug(`[${this._sessionId}]: Will increment subtitle with ${playheadSubtitle.increment}`);
sessionState.vodMediaSeqSubtitle = await this._sessionState.increment("vodMediaSeqSubtitle", playheadSubtitle.increment);
}
debug(`[${this._sessionId}]: Current VOD Playhead Positions are to be V[${(playheadPosVideoMs/1000).toFixed(3)}]${
playheadAudio ? `A[${(playheadAudio.position).toFixed(3)}]` : ""
}${
playheadSubtitle ? `S[${(playheadSubtitle.position).toFixed(3)})]` : ""
}${
playheadAudio ? ` (${playheadAudio.diff})` : ""
}${
playheadSubtitle ? ` (${playheadSubtitle.diff})` : ""
}`);
}

let newSessionState = {};
Expand Down Expand Up @@ -1388,6 +1388,7 @@ class Session {
debug(`[${this._sessionId}]: ${currentVod.getDeltaTimes()}`);
debug(`[${this._sessionId}]: playhead positions [V]=${currentVod.getPlayheadPositions("video")}`);
debug(`[${this._sessionId}]: playhead positions [A]=${currentVod.getPlayheadPositions("audio")}`);
debug(`[${this._sessionId}]: playhead positions [S]=${currentVod.getPlayheadPositions("subtitle")}`);
//debug(newVod);
const updatedSessionState = await this._sessionState.setValues({
"mediaSeq": 0,
Expand Down Expand Up @@ -1584,6 +1585,7 @@ class Session {
debug(`[${this._sessionId}]: next VOD loaded (${newVod.getDeltaTimes()})`);
debug(`[${this._sessionId}]: playhead positions [V]=${newVod.getPlayheadPositions("video")}`);
debug(`[${this._sessionId}]: playhead positions [A]=${newVod.getPlayheadPositions("audio")}`);
debug(`[${this._sessionId}]: playhead positions [S]=${newVod.getPlayheadPositions("subtitle")}`);
currentVod = newVod;
debug(`[${this._sessionId}]: msequences=${currentVod.getLiveMediaSequencesCount()}; audio msequences=${currentVod.getLiveMediaSequencesCount("audio")}; subtitle msequences=${currentVod.getLiveMediaSequencesCount("subtitle")}`);
sessionState.currentVod = await this._sessionState.setCurrentVod(currentVod, { ttl: currentVod.getDuration() * 1000 });
Expand Down Expand Up @@ -2089,6 +2091,40 @@ class Session {
}
return false;
}

async _determineExtraMediaIncrement(_extraType, _currentPosVideo, _extraSeqFinalIndex, _vodMediaSeqExtra, _getExtraPlayheadPositionAsyncFn) {
debug(`[${this._sessionId}]: About to determine ${_extraType} increment. Video increment has already been executed.`);
let extraMediaIncrement = 0;
let positionV = _currentPosVideo ? _currentPosVideo / 1000 : 0;
let positionX;
let posDiff;
const threshold = 0.250;
while (extraMediaIncrement < _extraSeqFinalIndex) {
const currentPosExtraMedia = (await _getExtraPlayheadPositionAsyncFn(_vodMediaSeqExtra + extraMediaIncrement)) * 1000;
positionX = currentPosExtraMedia ? currentPosExtraMedia / 1000 : 0;
posDiff = (positionV - positionX).toFixed(3);
debug(`[${this._sessionId}]: positionV=${positionV};position${_extraType === "audio" ? "A" : "S"}=${positionX};posDiff=${posDiff}`);
if (isNaN(posDiff)) {
break;
}
if (positionX >= positionV) {
break;
}
const difference = Math.abs(posDiff);
if (difference > threshold && difference > Number.EPSILON) {
// Video position ahead of audio|sub position, further increment needed...
extraMediaIncrement++;
} else {
debug(`[${this._sessionId}]: Difference(${difference}) is acceptable; IncrementValue=${extraMediaIncrement}`);
break;
}
}
return {
increment: extraMediaIncrement,
position: positionX,
diff: posDiff
};
}
}

module.exports = Session;
112 changes: 104 additions & 8 deletions spec/engine/session_spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,116 @@
const Session = require('../../engine/session.js');
const Session = require("../../engine/session.js");

const { SessionStateStore } = require('../../engine/session_state.js');
const { PlayheadStateStore } = require('../../engine/playhead_state.js');
const { SessionStateStore } = require("../../engine/session_state.js");
const { PlayheadStateStore } = require("../../engine/playhead_state.js");

describe("Session", () => {
let sessionLiveStore = undefined;
beforeEach(() => {
sessionLiveStore = {
sessionStateStore: new SessionStateStore(),
playheadStateStore: new PlayheadStateStore()
};
playheadStateStore: new PlayheadStateStore(),
};
});

it("creates a unique session ID", () => {
const id1 = new Session('dummy', null, sessionLiveStore).sessionId;
const id2 = new Session('dummy', null, sessionLiveStore).sessionId;
const id1 = new Session("dummy", null, sessionLiveStore).sessionId;
const id2 = new Session("dummy", null, sessionLiveStore).sessionId;
expect(id1).not.toEqual(id2);
});
});

it("for demuxed, returns the appropriate audio increment value when desync is within acceptable limit, case I", async () => {
const session = new Session("dummy", null, sessionLiveStore);
const mockFinalAudioIdx = 50; // current Vod has 50 media sequences to serve.
const mockCurrentVideoPosition = 200.0 * 1000; // Video is 200s deep into its content.
const mockMseqAudio = 25; // current mseq for audio on vod, 25 out of 50.
const mock_getAudioPlayheadPosition = async (pos_n_current) => {
const mockPositions = [196.0, 199.84, 203.68, 207.52];
return mockPositions[pos_n_current - mockMseqAudio];
};
const output = await session._determineExtraMediaIncrement(
"audio",
mockCurrentVideoPosition,
mockFinalAudioIdx,
mockMseqAudio,
mock_getAudioPlayheadPosition,
24
);
expect(output.increment).toBe(1);
});

it("for demuxed, returns the appropriate audio increment value when desync is within acceptable limit, case II", async () => {
const session = new Session("dummy", null, sessionLiveStore);
const mockFinalAudioIdx = 50; // current Vod has 50 media sequences to serve.
const mockCurrentVideoPosition = 200.0 * 1000; // Video is 200s deep into its content.
const mockMseqAudio = 25; // current mseq for audio on vod, 25 out of 50.
const mock_getAudioPlayheadPosition = async (pos_n_current) => {
const mockPositions = [192.16, 196.0, 199.84, 203.68, 207.52];
return mockPositions[pos_n_current - mockMseqAudio];
};
const output = await session._determineExtraMediaIncrement(
"audio",
mockCurrentVideoPosition,
mockFinalAudioIdx,
mockMseqAudio,
mock_getAudioPlayheadPosition,
24
);
expect(output.increment).toBe(2);
});

it("for demuxed, returns the appropriate audio increment value when they are in sync but there is a floating point error", async () => {
const session = new Session("dummy", null, sessionLiveStore);
const mockFinalAudioIdx = 50;
const mockCurrentVideoPosition = 441.7599999999981697 * 1000;
const mockMseqAudio = 25;
const mock_getAudioPlayheadPosition = async (pos_n_current) => {
const mockPositions = [437.919999999999, 441.75999999999897];
return mockPositions[pos_n_current - mockMseqAudio];
};
const output = await session._determineExtraMediaIncrement(
"audio",
mockCurrentVideoPosition,
mockFinalAudioIdx,
mockMseqAudio,
mock_getAudioPlayheadPosition,
24
);
expect(output.increment).toBe(1);
});
it("for demuxed, returns the appropriate audio increment value, normal case I", async () => {
const session = new Session("dummy", null, sessionLiveStore);
const mockFinalAudioIdx = 50;
const mockCurrentVideoPosition = 3.840 * 8 * 1000;
const mockMseqAudio = 5;
const mock_getAudioPlayheadPosition = async (pos_n_current) => {
const mockPositions = [0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52];
return mockPositions[pos_n_current];
};
const output = await session._determineExtraMediaIncrement(
"audio",
mockCurrentVideoPosition,
mockFinalAudioIdx,
mockMseqAudio,
mock_getAudioPlayheadPosition
);
expect(output.increment).toBe(3);
});
it("for demuxed, returns the appropriate audio increment value, normal case II", async () => {
const session = new Session("dummy", null, sessionLiveStore);
const mockFinalAudioIdx = 50;
const mockCurrentVideoPosition = 14 * 1000;
const mockMseqAudio = 0;
const mock_getAudioPlayheadPosition = async (pos_n_current) => {
const mockPositions = [0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52];
return mockPositions[pos_n_current];
};
const output = await session._determineExtraMediaIncrement(
"subtitle",
mockCurrentVideoPosition,
mockFinalAudioIdx,
mockMseqAudio,
mock_getAudioPlayheadPosition
);
expect(output.increment).toBe(4);
});
});