Skip to content

Commit

Permalink
Live player fixes (#13143)
Browse files Browse the repository at this point in the history
* Jump to live when exceeding buffer time threshold in MSE player

* clean up

* Try adjusting playback rate instead of jumping to live

* clean up

* fallback to webrtc if enabled before jsmpeg

* baseline

* clean up

* remove comments

* adaptive playback rate and intelligent switching improvements

* increase logging and reset live mode after camera is no longer active on dashboard only

* jump to live on safari/iOS

* clean up

* clean up

* refactor camera live mode hook

* remove key listener

* resolve conflicts
  • Loading branch information
hawkeye217 authored and NickM-27 committed Aug 30, 2024
1 parent 758b0f9 commit ef46451
Show file tree
Hide file tree
Showing 7 changed files with 229 additions and 121 deletions.
14 changes: 7 additions & 7 deletions web/src/components/player/LivePlayer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import {
LivePlayerMode,
VideoResolutionType,
} from "@/types/live";
import useCameraLiveMode from "@/hooks/use-camera-live-mode";
import { getIconForLabel } from "@/utils/iconUtil";
import Chip from "../indicators/Chip";
import { capitalizeFirstLetter } from "@/utils/stringUtil";
Expand All @@ -25,7 +24,7 @@ type LivePlayerProps = {
containerRef?: React.MutableRefObject<HTMLDivElement | null>;
className?: string;
cameraConfig: CameraConfig;
preferredLiveMode?: LivePlayerMode;
preferredLiveMode: LivePlayerMode;
showStillWithoutActivity?: boolean;
windowVisible?: boolean;
playAudio?: boolean;
Expand All @@ -36,6 +35,7 @@ type LivePlayerProps = {
onClick?: () => void;
setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
onError?: (error: LivePlayerError) => void;
onResetLiveMode?: () => void;
};

export default function LivePlayer({
Expand All @@ -54,6 +54,7 @@ export default function LivePlayer({
onClick,
setFullResolution,
onError,
onResetLiveMode,
}: LivePlayerProps) {
const internalContainerRef = useRef<HTMLDivElement | null>(null);
// camera activity
Expand All @@ -70,8 +71,6 @@ export default function LivePlayer({

// camera live state

const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode);

const [liveReady, setLiveReady] = useState(false);

const liveReadyRef = useRef(liveReady);
Expand All @@ -91,6 +90,7 @@ export default function LivePlayer({
const timer = setTimeout(() => {
if (liveReadyRef.current && !cameraActiveRef.current) {
setLiveReady(false);
onResetLiveMode?.();
}
}, 500);

Expand Down Expand Up @@ -152,7 +152,7 @@ export default function LivePlayer({
let player;
if (!autoLive) {
player = null;
} else if (liveMode == "webrtc") {
} else if (preferredLiveMode == "webrtc") {
player = (
<WebRtcPlayer
className={`size-full rounded-lg md:rounded-2xl ${liveReady ? "" : "hidden"}`}
Expand All @@ -166,7 +166,7 @@ export default function LivePlayer({
onError={onError}
/>
);
} else if (liveMode == "mse") {
} else if (preferredLiveMode == "mse") {
if ("MediaSource" in window || "ManagedMediaSource" in window) {
player = (
<MSEPlayer
Expand All @@ -187,7 +187,7 @@ export default function LivePlayer({
</div>
);
}
} else if (liveMode == "jsmpeg") {
} else if (preferredLiveMode == "jsmpeg") {
if (cameraActive || !showStillWithoutActivity || liveReady) {
player = (
<JSMpegPlayer
Expand Down
138 changes: 130 additions & 8 deletions web/src/components/player/MsePlayer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ function MSEPlayer({
onError,
}: MSEPlayerProps) {
const RECONNECT_TIMEOUT: number = 10000;
const BUFFERING_COOLDOWN_TIMEOUT: number = 5000;

const CODECS: string[] = [
"avc1.640029", // H.264 high 4.1 (Chromecast 1st and 2nd Gen)
Expand All @@ -46,6 +47,11 @@ function MSEPlayer({

const visibilityCheck: boolean = !pip;
const [isPlaying, setIsPlaying] = useState(false);
const lastJumpTimeRef = useRef(0);

const MAX_BUFFER_ENTRIES = 10; // Size of the rolling window of buffered times
const bufferTimes = useRef<number[]>([]);
const bufferIndex = useRef(0);

const [wsState, setWsState] = useState<number>(WebSocket.CLOSED);
const [connectTS, setConnectTS] = useState<number>(0);
Expand Down Expand Up @@ -133,6 +139,13 @@ function MSEPlayer({
}
}, [bufferTimeout]);

const handlePause = useCallback(() => {
// don't let the user pause the live stream
if (isPlaying && playbackEnabled) {
videoRef.current?.play();
}
}, [isPlaying, playbackEnabled]);

const onOpen = () => {
setWsState(WebSocket.OPEN);

Expand Down Expand Up @@ -193,6 +206,7 @@ function MSEPlayer({

const onMse = () => {
if ("ManagedMediaSource" in window) {
// safari
const MediaSource = window.ManagedMediaSource;

msRef.current?.addEventListener(
Expand Down Expand Up @@ -224,6 +238,7 @@ function MSEPlayer({
videoRef.current.srcObject = msRef.current;
}
} else {
// non safari
msRef.current?.addEventListener(
"sourceopen",
() => {
Expand All @@ -247,15 +262,35 @@ function MSEPlayer({
},
{ once: true },
);
videoRef.current!.src = URL.createObjectURL(msRef.current!);
videoRef.current!.srcObject = null;
if (videoRef.current && msRef.current) {
videoRef.current.src = URL.createObjectURL(msRef.current);
videoRef.current.srcObject = null;
}
}
play();

onmessageRef.current["mse"] = (msg) => {
if (msg.type !== "mse") return;

const sb = msRef.current?.addSourceBuffer(msg.value);
let sb: SourceBuffer | undefined;
try {
sb = msRef.current?.addSourceBuffer(msg.value);
if (sb?.mode) {
sb.mode = "segments";
}
} catch (e) {
// Safari sometimes throws this error
if (e instanceof DOMException && e.name === "InvalidStateError") {
if (wsRef.current) {
onDisconnect();
}
onError?.("mse-decode");
return;
} else {
throw e; // Re-throw if it's not the error we're handling
}
}

sb?.addEventListener("updateend", () => {
if (sb.updating) return;

Expand Down Expand Up @@ -302,6 +337,43 @@ function MSEPlayer({
return video.buffered.end(video.buffered.length - 1) - video.currentTime;
};

const jumpToLive = () => {
if (!videoRef.current) return;

const buffered = videoRef.current.buffered;
if (buffered.length > 0) {
const liveEdge = buffered.end(buffered.length - 1);
// Jump to the live edge
videoRef.current.currentTime = liveEdge - 0.75;
lastJumpTimeRef.current = Date.now();
}
};

const calculateAdaptiveBufferThreshold = () => {
const filledEntries = bufferTimes.current.length;
const sum = bufferTimes.current.reduce((a, b) => a + b, 0);
const averageBufferTime = filledEntries ? sum / filledEntries : 0;
return averageBufferTime * (isSafari || isIOS ? 3 : 1.5);
};

const calculateAdaptivePlaybackRate = (
bufferTime: number,
bufferThreshold: number,
) => {
const alpha = 0.2; // aggressiveness of playback rate increase
const beta = 0.5; // steepness of exponential growth

// don't adjust playback rate if we're close enough to live
if (
(bufferTime <= bufferThreshold && bufferThreshold < 3) ||
bufferTime < 3
) {
return 1;
}
const rate = 1 + alpha * Math.exp(beta * bufferTime - bufferThreshold);
return Math.min(rate, 2);
};

useEffect(() => {
if (!playbackEnabled) {
return;
Expand Down Expand Up @@ -386,21 +458,71 @@ function MSEPlayer({
handleLoadedMetadata?.();
onPlaying?.();
setIsPlaying(true);
lastJumpTimeRef.current = Date.now();
}}
muted={!audioEnabled}
onPause={() => videoRef.current?.play()}
onPause={handlePause}
onProgress={() => {
const bufferTime = getBufferedTime(videoRef.current);

if (
videoRef.current &&
(videoRef.current.playbackRate === 1 || bufferTime < 3)
) {
if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) {
bufferTimes.current.push(bufferTime);
} else {
bufferTimes.current[bufferIndex.current] = bufferTime;
bufferIndex.current =
(bufferIndex.current + 1) % MAX_BUFFER_ENTRIES;
}
}

const bufferThreshold = calculateAdaptiveBufferThreshold();

// if we have > 3 seconds of buffered data and we're still not playing,
// something might be wrong - maybe codec issue, no audio, etc
// so mark the player as playing so that error handlers will fire
if (!isPlaying && playbackEnabled && bufferTime > 3) {
setIsPlaying(true);
lastJumpTimeRef.current = Date.now();
onPlaying?.();
}

// if we have more than 10 seconds of buffer, something's wrong so error out
if (
!isPlaying &&
isPlaying &&
playbackEnabled &&
getBufferedTime(videoRef.current) > 3
(bufferThreshold > 10 || bufferTime > 10)
) {
setIsPlaying(true);
onPlaying?.();
onDisconnect();
onError?.("stalled");
}

const playbackRate = calculateAdaptivePlaybackRate(
bufferTime,
bufferThreshold,
);

// if we're above our rolling average threshold or have > 3 seconds of
// buffered data and we're playing, we may have drifted from actual live
// time, so increase playback rate to compensate - non safari/ios only
if (
videoRef.current &&
isPlaying &&
playbackEnabled &&
Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT
) {
// Jump to live on Safari/iOS due to a change of playback rate causing re-buffering
if (isSafari || isIOS) {
if (bufferTime > 3) {
jumpToLive();
}
} else {
videoRef.current.playbackRate = playbackRate;
}
}

if (onError != undefined) {
if (videoRef.current?.paused) {
return;
Expand Down
90 changes: 53 additions & 37 deletions web/src/hooks/use-camera-live-mode.ts
Original file line number Diff line number Diff line change
@@ -1,49 +1,65 @@
import { CameraConfig, FrigateConfig } from "@/types/frigateConfig";
import { useMemo } from "react";
import { useCallback, useEffect, useState } from "react";
import useSWR from "swr";
import { usePersistence } from "./use-persistence";
import { LivePlayerMode } from "@/types/live";

export default function useCameraLiveMode(
cameraConfig: CameraConfig,
preferredMode?: LivePlayerMode,
): LivePlayerMode | undefined {
cameras: CameraConfig[],
windowVisible: boolean,
) {
const { data: config } = useSWR<FrigateConfig>("config");
const [preferredLiveModes, setPreferredLiveModes] = useState<{
[key: string]: LivePlayerMode;
}>({});

const restreamEnabled = useMemo(() => {
if (!config) {
return false;
}
useEffect(() => {
if (!cameras) return;

return (
cameraConfig &&
Object.keys(config.go2rtc.streams || {}).includes(
cameraConfig.live.stream_name,
)
const mseSupported =
"MediaSource" in window || "ManagedMediaSource" in window;

const newPreferredLiveModes = cameras.reduce(
(acc, camera) => {
const isRestreamed =
config &&
Object.keys(config.go2rtc.streams || {}).includes(
camera.live.stream_name,
);

if (!mseSupported) {
acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg";
} else {
acc[camera.name] = isRestreamed ? "mse" : "jsmpeg";
}
return acc;
},
{} as { [key: string]: LivePlayerMode },
);
}, [config, cameraConfig]);
const defaultLiveMode = useMemo<LivePlayerMode | undefined>(() => {
if (config) {
if (restreamEnabled) {
return preferredMode || "mse";
}

return "jsmpeg";
}

return undefined;
}, [config, preferredMode, restreamEnabled]);
const [viewSource] = usePersistence<LivePlayerMode>(
`${cameraConfig.name}-source`,
defaultLiveMode,

setPreferredLiveModes(newPreferredLiveModes);
}, [cameras, config, windowVisible]);

const resetPreferredLiveMode = useCallback(
(cameraName: string) => {
const mseSupported =
"MediaSource" in window || "ManagedMediaSource" in window;
const isRestreamed =
config && Object.keys(config.go2rtc.streams || {}).includes(cameraName);

setPreferredLiveModes((prevModes) => {
const newModes = { ...prevModes };

if (!mseSupported) {
newModes[cameraName] = isRestreamed ? "webrtc" : "jsmpeg";
} else {
newModes[cameraName] = isRestreamed ? "mse" : "jsmpeg";
}

return newModes;
});
},
[config],
);

if (
restreamEnabled &&
(preferredMode == "mse" || preferredMode == "webrtc")
) {
return preferredMode;
} else {
return viewSource;
}
return { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode };
}
Loading

0 comments on commit ef46451

Please sign in to comment.