Add fps-awareness to DefaultTrackSelector

This change aims to prioritise tracks that have a 'smooth enough for video' frame rate, without always selecting the track with the highest frame rate. In particular MP4 files extracted from motion photos sometimes have two HEVC tracks, with the higher-res one having a very low frame rate (not intended for use in video playback). Before this change `DefaultTrackSelector` would pick the low-fps, high-res track. This change adds a somewhat arbitrary 10fps threshold for "smooth video playback", meaning any tracks above this threshold are selected in preference to tracks below it. Within the tracks above the threshold other attributes are used to select the preferred track. We deliberately don't pick the highest-fps track (over pixel count and bitrate), because most users would prefer to see a 30fps 4k track over a 60fps 720p track. This change also includes a test MP4 file, extracted from the existing `jpeg/pixel-motion-photo-2-hevc-tracks.jpg` file by logging `mp4StartPosition` in [`MotionPhotoDescription.getMotionPhotoMetadata`](https://github.com/androidx/media/blob/b930b40a16c06318e43c81771fa2b1024bdb3f29/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/MotionPhotoDescription.java#L123) and then using `dd`: ``` mp4StartPosition=2603594 $ dd if=jpeg/pixel-motion-photo-2-hevc-tracks.jpg \ of=mp4/pixel-motion-photo-2-hevc-tracks.mp4 \ bs=1 \ skip=2603594 ``` ---- This solution is in addition to the `JpegMotionPhotoExtractor` change made specifically for these two-track motion photos in 5266c71. We will keep both changes, even though that change is not strictly needed after this one, because adding the role flags helps to communicate more clearly the intended usage of these tracks. This change to consider FPS seems like a generally useful improvement to `DefaultTrackSelector`, since it seems unlikely we would prefer a 5fps video track over a 30fps one. Issue: #1051 PiperOrigin-RevId: 611015459 (cherry picked from commit c7e00b1)
androidx · Apr 2, 2024 · 1d2116c · 1d2116c
1 parent 3521ccd
commit 1d2116c
Show file tree

Hide file tree

Showing 6 changed files with 521 additions and 3 deletions.
diff --git a/RELEASENOTES.md b/RELEASENOTES.md
@@ -10,7 +10,14 @@
 *   Transformer:
     *   Add workaround for exception thrown due to `MediaMuxer` not supporting
         negative presentation timestamps before API 30.
-*   Audio:
+*   Track Selection:
+    *   `DefaultTrackSelector`: Prefer video tracks with a 'reasonable' frame
+        rate (>=10fps) over those with a lower or unset frame rate. This ensures
+        the player selects the 'real' video track in MP4s extracted from motion
+        photos that can contain two HEVC tracks where one has a higher
+        resolution but a very small number of frames
+        ([#1051](https://github.com/androidx/media/issues/1051)).  
+* Audio:
     *   Allow renderer recovery by disabling offload if audio track fails to
         initialize in offload mode.
 *   Video:

diff --git a/...xoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java b/...xoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java
@@ -3516,6 +3516,12 @@ public TrackInfo(int rendererIndex, TrackGroup trackGroup, int trackIndex) {
 
   private static final class VideoTrackInfo extends TrackInfo<VideoTrackInfo> {
 
+    /**
+     * Frame rate below which video playback will definitely not be considered smooth by the human
+     * eye.
+     */
+    private static final float MIN_REASONABLE_FRAME_RATE = 10;
+
     public static ImmutableList<VideoTrackInfo> createForTrackGroup(
         int rendererIndex,
         TrackGroup trackGroup,
@@ -3551,6 +3557,12 @@ public static ImmutableList<VideoTrackInfo> createForTrackGroup(
     private final Parameters parameters;
     private final boolean isWithinMinConstraints;
     private final boolean isWithinRendererCapabilities;
+
+    /**
+     * True if {@link Format#frameRate} is set and is at least {@link #MIN_REASONABLE_FRAME_RATE}.
+     */
+    private final boolean hasReasonableFrameRate;
+
     private final int bitrate;
     private final int pixelCount;
     private final int preferredMimeTypeMatchIndex;
@@ -3599,6 +3611,8 @@ public VideoTrackInfo(
                   || format.bitrate >= parameters.minVideoBitrate);
       isWithinRendererCapabilities =
           isSupported(formatSupport, /* allowExceedsCapabilities= */ false);
+      hasReasonableFrameRate =
+          format.frameRate != Format.NO_VALUE && format.frameRate >= MIN_REASONABLE_FRAME_RATE;
       bitrate = format.bitrate;
       pixelCount = format.getPixelCount();
       preferredRoleFlagsScore =
@@ -3669,16 +3683,19 @@ private static int compareNonQualityPreferences(VideoTrackInfo info1, VideoTrack
               .compare(info1.preferredRoleFlagsScore, info2.preferredRoleFlagsScore)
               // 2. Compare match with implicit content preferences set by the media.
               .compareFalseFirst(info1.hasMainOrNoRoleFlag, info2.hasMainOrNoRoleFlag)
-              // 3. Compare match with technical preferences set by the parameters.
+              // 3. Compare match with 'reasonable' frame rate threshold.
+              .compareFalseFirst(info1.hasReasonableFrameRate, info2.hasReasonableFrameRate)
+              // 4. Compare match with technical preferences set by the parameters.
               .compareFalseFirst(info1.isWithinMaxConstraints, info2.isWithinMaxConstraints)
               .compareFalseFirst(info1.isWithinMinConstraints, info2.isWithinMinConstraints)
               .compare(
                   info1.preferredMimeTypeMatchIndex,
                   info2.preferredMimeTypeMatchIndex,
                   Ordering.natural().reverse())
-              // 4. Compare match with renderer capability preferences.
+              // 5. Compare match with renderer capability preferences.
               .compareFalseFirst(info1.usesPrimaryDecoder, info2.usesPrimaryDecoder)
               .compareFalseFirst(info1.usesHardwareAcceleration, info2.usesHardwareAcceleration);
+
       if (info1.usesPrimaryDecoder && info1.usesHardwareAcceleration) {
         chain = chain.compare(info1.codecPreferenceScore, info2.codecPreferenceScore);
       }

diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/Mp4PlaybackTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/Mp4PlaybackTest.java
@@ -46,6 +46,7 @@ public static ImmutableList<String> mediaSamples() {
         "midroll-5s.mp4",
         "postroll-5s.mp4",
         "preroll-5s.mp4",
+        "pixel-motion-photo-2-hevc-tracks.mp4",
         "sample_ac3_fragmented.mp4",
         "sample_ac3.mp4",
         "sample_ac4_fragmented.mp4",

diff --git a/...ayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java b/...ayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java
@@ -2821,6 +2821,84 @@ public void selectTracks_withPreferredAudioMimeTypes_selectsTrackWithPreferredMi
     assertFixedSelection(result.selections[0], trackGroups, formatAac);
   }
 
+  /**
+   * Tests that the track selector will select a group with a single video track with a 'reasonable'
+   * frame rate instead of a larger groups of tracks all with lower frame rates (the larger group of
+   * tracks would normally be preferred).
+   */
+  @Test
+  public void selectTracks_reasonableFrameRatePreferredOverTrackCount() throws Exception {
+    Format.Builder formatBuilder = VIDEO_FORMAT.buildUpon();
+    Format frameRateTooLow = formatBuilder.setFrameRate(5).build();
+    Format frameRateAlsoTooLow = formatBuilder.setFrameRate(6).build();
+    Format highEnoughFrameRate = formatBuilder.setFrameRate(30).build();
+    // Use an adaptive group to check that frame rate has higher priority than number of tracks.
+    TrackGroup adaptiveFrameRateTooLowGroup = new TrackGroup(frameRateTooLow, frameRateAlsoTooLow);
+    TrackGroupArray trackGroups =
+        new TrackGroupArray(adaptiveFrameRateTooLowGroup, new TrackGroup(highEnoughFrameRate));
+
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {VIDEO_CAPABILITIES}, trackGroups, periodId, TIMELINE);
+
+    assertFixedSelection(result.selections[0], trackGroups, highEnoughFrameRate);
+  }
+
+  /**
+   * Tests that the track selector will select the video track with a 'reasonable' frame rate that
+   * has the best match on other attributes, instead of an otherwise preferred track with a lower
+   * frame rate.
+   */
+  @Test
+  public void selectTracks_reasonableFrameRatePreferredButNotHighestFrameRate() throws Exception {
+    Format.Builder formatBuilder = VIDEO_FORMAT.buildUpon();
+    Format frameRateUnsetHighRes =
+        formatBuilder.setFrameRate(Format.NO_VALUE).setWidth(3840).setHeight(2160).build();
+    Format frameRateTooLowHighRes =
+        formatBuilder.setFrameRate(5).setWidth(3840).setHeight(2160).build();
+    Format highEnoughFrameRateHighRes =
+        formatBuilder.setFrameRate(30).setWidth(1920).setHeight(1080).build();
+    Format highestFrameRateLowRes =
+        formatBuilder.setFrameRate(60).setWidth(1280).setHeight(720).build();
+    TrackGroupArray trackGroups =
+        new TrackGroupArray(
+            new TrackGroup(frameRateUnsetHighRes),
+            new TrackGroup(frameRateTooLowHighRes),
+            new TrackGroup(highestFrameRateLowRes),
+            new TrackGroup(highEnoughFrameRateHighRes));
+
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {VIDEO_CAPABILITIES}, trackGroups, periodId, TIMELINE);
+
+    assertFixedSelection(result.selections[0], trackGroups, highEnoughFrameRateHighRes);
+  }
+
+  /**
+   * Tests that the track selector will select a track with {@link C#ROLE_FLAG_MAIN} with an
+   * 'unreasonably low' frame rate, if the other track with a 'reasonable' frame rate is marked with
+   * {@link C#ROLE_FLAG_ALTERNATE}. These role flags show an explicit signal from the media, so they
+   * should be respected.
+   */
+  @Test
+  public void selectTracks_roleFlagsOverrideReasonableFrameRate() throws Exception {
+    Format.Builder formatBuilder = VIDEO_FORMAT.buildUpon();
+    Format mainTrackWithLowFrameRate =
+        formatBuilder.setFrameRate(3).setRoleFlags(C.ROLE_FLAG_MAIN).build();
+    Format alternateTrackWithHighFrameRate =
+        formatBuilder.setFrameRate(30).setRoleFlags(C.ROLE_FLAG_ALTERNATE).build();
+    TrackGroupArray trackGroups =
+        new TrackGroupArray(
+            new TrackGroup(mainTrackWithLowFrameRate),
+            new TrackGroup(alternateTrackWithHighFrameRate));
+
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {VIDEO_CAPABILITIES}, trackGroups, periodId, TIMELINE);
+
+    assertFixedSelection(result.selections[0], trackGroups, mainTrackWithLowFrameRate);
+  }
+
   /** Tests audio track selection when there are multiple audio renderers. */
   @Test
   public void selectTracks_multipleRenderer_allSelected() throws Exception {

diff --git a/libraries/test_data/src/test/assets/media/mp4/pixel-motion-photo-2-hevc-tracks.mp4 b/libraries/test_data/src/test/assets/media/mp4/pixel-motion-photo-2-hevc-tracks.mp4