Skip to content

Commit a5dd314

Browse files
matanlureymatanlurey
authored
Migrate video_player/android from SurfaceTexture->SurfaceProducer. (#6456)
_**WIP**: We do not plan to land this PR until the next stable release (>= April 3rd 2024)_. Work towards flutter/flutter#145930. ## Details Migrates uses of `createSurfaceTexture` to `createSurfaceProducer`, which is intended to have no change in behavior, but _does_ change the backend rendering path, so it will require more testing (and we're also open to minor API renames or changes before it becomes stable). ## Background Android plugins previously requested a `SurfaceTexture` from the Android embedder, and used that to produce a `Surface` to render external textures on (i.e. `video_player`). This worked because 100% of Flutter applications on Android used OpenGLES (via our Skia backend), and `SurfaceTexture` is actually an (opaque) OpenGLES-texture. Starting soon (roughly ~Q3, this is not a guarantee and just an estimate), Flutter on Android will start to use our new Impeller graphics backend, which on newer devices (`>= API_VERSION_28`), will default to the Vulkan, _not_ OpenGLES. In other words, `SurfaceTexture` will cease to work (it is possible, but non-trivial, to map an OpenGLES texture over to Vulkan). After consultation with the Android team, they helped us understand that vending `SurfaceTexture` (the _consumer-side_ API) was never the right abstraction, and we should have been vending the _producer-side_ API, or `Surface` directly. The new `SurfaceProducer` API is exactly that - it generates a `Surface`, and similar to our platform view strategy, picks the "right" _consumer-side_ implementation details _for_ the user/plugin packages. The new `SurfaceProducer` API has 2 possible rendering types (as an implementation detail): - `SurfaceTexture`, for older OpenGLES devices, which works exactly as it does today. - `ImageReader`, for newer OpenGLES _or_ Vulkan devices. These are some subtle nuances in how these two APIs work differently (one example: flutter/flutter#144407), but our theory at this point is we don't expect these changes to be observed by any users, and we have other ideas if necessary. > [!NOTE] > These invariants are [tested on CI in `flutter/engine`](https://github.com/flutter/engine/tree/main/testing/scenario_app/android#ci-configuration). Points of contact: - @matanlurey or @jonahwilliams (Flutter Engine) - @johnmccutchan or @reidbaker (Flutter on Android)
1 parent 28e8afd commit a5dd314

File tree

6 files changed

+136
-38
lines changed

6 files changed

+136
-38
lines changed

‎packages/video_player/video_player_android/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## 2.4.16
2+
3+
* [Supports Impeller](https://docs.flutter.dev/release/breaking-changes/android-surface-plugins).
4+
15
## 2.4.15
26

37
* Updates minimum supported SDK version to Flutter 3.22/Dart 3.4.

‎packages/video_player/video_player_android/android/src/main/java/io/flutter/plugins/videoplayer/VideoPlayer.java‎

Lines changed: 49 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99

1010
import android.content.Context;
1111
import android.net.Uri;
12-
import android.view.Surface;
1312
import androidx.annotation.NonNull;
1413
import androidx.annotation.VisibleForTesting;
1514
import com.google.android.exoplayer2.C;
@@ -48,32 +47,37 @@ final class VideoPlayer {
4847

4948
private ExoPlayer exoPlayer;
5049

51-
private Surface surface;
52-
53-
private final TextureRegistry.SurfaceTextureEntry textureEntry;
50+
private TextureRegistry.SurfaceProducer surfaceProducer;
5451

5552
private QueuingEventSink eventSink;
5653

5754
private final EventChannel eventChannel;
5855

5956
private static final String USER_AGENT = "User-Agent";
6057

58+
private MediaSource mediaSource;
59+
6160
@VisibleForTesting boolean isInitialized = false;
6261

62+
// State that must be reset when the surface is re-created.
6363
private final VideoPlayerOptions options;
64+
private long restoreVideoLocation = 0;
65+
private int restoreRepeatMode = 0;
66+
private float restoreVolume = 0;
67+
private PlaybackParameters restorePlaybackParameters;
6468

6569
private DefaultHttpDataSource.Factory httpDataSourceFactory = new DefaultHttpDataSource.Factory();
6670

6771
VideoPlayer(
6872
Context context,
6973
EventChannel eventChannel,
70-
TextureRegistry.SurfaceTextureEntry textureEntry,
74+
TextureRegistry.SurfaceProducer surfaceProducer,
7175
String dataSource,
7276
String formatHint,
7377
@NonNull Map<String, String> httpHeaders,
7478
VideoPlayerOptions options) {
7579
this.eventChannel = eventChannel;
76-
this.textureEntry = textureEntry;
80+
this.surfaceProducer = surfaceProducer;
7781
this.options = options;
7882

7983
ExoPlayer exoPlayer = new ExoPlayer.Builder(context).build();
@@ -83,7 +87,7 @@ final class VideoPlayer {
8387
DataSource.Factory dataSourceFactory =
8488
new DefaultDataSource.Factory(context, httpDataSourceFactory);
8589

86-
MediaSource mediaSource = buildMediaSource(uri, dataSourceFactory, formatHint);
90+
mediaSource = buildMediaSource(uri, dataSourceFactory, formatHint);
8791

8892
exoPlayer.setMediaSource(mediaSource);
8993
exoPlayer.prepare();
@@ -96,12 +100,12 @@ final class VideoPlayer {
96100
VideoPlayer(
97101
ExoPlayer exoPlayer,
98102
EventChannel eventChannel,
99-
TextureRegistry.SurfaceTextureEntry textureEntry,
103+
TextureRegistry.SurfaceProducer surfaceProducer,
100104
VideoPlayerOptions options,
101105
QueuingEventSink eventSink,
102106
DefaultHttpDataSource.Factory httpDataSourceFactory) {
103107
this.eventChannel = eventChannel;
104-
this.textureEntry = textureEntry;
108+
this.surfaceProducer = surfaceProducer;
105109
this.options = options;
106110
this.httpDataSourceFactory = httpDataSourceFactory;
107111

@@ -169,6 +173,40 @@ private MediaSource buildMediaSource(
169173
}
170174
}
171175

176+
public void recreateSurface(Context context) {
177+
ExoPlayer exoPlayer = new ExoPlayer.Builder(context).build();
178+
179+
exoPlayer.setMediaSource(mediaSource);
180+
exoPlayer.prepare();
181+
182+
setUpVideoPlayer(exoPlayer, new QueuingEventSink());
183+
exoPlayer.setVideoSurface(surfaceProducer.getSurface());
184+
exoPlayer.seekTo(restoreVideoLocation);
185+
exoPlayer.setRepeatMode(restoreRepeatMode);
186+
exoPlayer.setVolume(restoreVolume);
187+
if (restorePlaybackParameters != null) {
188+
exoPlayer.setPlaybackParameters(restorePlaybackParameters);
189+
}
190+
}
191+
192+
public void pauseSurface() {
193+
if (!isInitialized) {
194+
return;
195+
}
196+
restoreVideoLocation = exoPlayer.getCurrentPosition();
197+
restoreRepeatMode = exoPlayer.getRepeatMode();
198+
restoreVolume = exoPlayer.getVolume();
199+
restorePlaybackParameters = exoPlayer.getPlaybackParameters();
200+
eventChannel.setStreamHandler(null);
201+
if (isInitialized) {
202+
exoPlayer.stop();
203+
}
204+
if (exoPlayer != null) {
205+
exoPlayer.release();
206+
}
207+
isInitialized = false;
208+
}
209+
172210
private void setUpVideoPlayer(ExoPlayer exoPlayer, QueuingEventSink eventSink) {
173211
this.exoPlayer = exoPlayer;
174212
this.eventSink = eventSink;
@@ -186,8 +224,7 @@ public void onCancel(Object o) {
186224
}
187225
});
188226

189-
surface = new Surface(textureEntry.surfaceTexture());
190-
exoPlayer.setVideoSurface(surface);
227+
exoPlayer.setVideoSurface(surfaceProducer.getSurface());
191228
setAudioAttributes(exoPlayer, options.mixWithOthers);
192229

193230
exoPlayer.addListener(
@@ -334,11 +371,8 @@ void dispose() {
334371
if (isInitialized) {
335372
exoPlayer.stop();
336373
}
337-
textureEntry.release();
374+
surfaceProducer.release();
338375
eventChannel.setStreamHandler(null);
339-
if (surface != null) {
340-
surface.release();
341-
}
342376
if (exoPlayer != null) {
343377
exoPlayer.release();
344378
}

‎packages/video_player/video_player_android/android/src/main/java/io/flutter/plugins/videoplayer/VideoPlayerPlugin.java‎

Lines changed: 69 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,16 @@
88
import android.os.Build;
99
import android.util.LongSparseArray;
1010
import androidx.annotation.NonNull;
11+
import androidx.annotation.Nullable;
12+
import androidx.lifecycle.DefaultLifecycleObserver;
13+
import androidx.lifecycle.Lifecycle;
14+
import androidx.lifecycle.LifecycleOwner;
1115
import io.flutter.FlutterInjector;
1216
import io.flutter.Log;
1317
import io.flutter.embedding.engine.plugins.FlutterPlugin;
18+
import io.flutter.embedding.engine.plugins.activity.ActivityAware;
19+
import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding;
20+
import io.flutter.embedding.engine.plugins.lifecycle.FlutterLifecycleAdapter;
1421
import io.flutter.plugin.common.BinaryMessenger;
1522
import io.flutter.plugin.common.EventChannel;
1623
import io.flutter.plugins.videoplayer.Messages.AndroidVideoPlayerApi;
@@ -29,11 +36,13 @@
2936
import javax.net.ssl.HttpsURLConnection;
3037

3138
/** Android platform implementation of the VideoPlayerPlugin. */
32-
public class VideoPlayerPlugin implements FlutterPlugin, AndroidVideoPlayerApi {
39+
public class VideoPlayerPlugin
40+
implements FlutterPlugin, AndroidVideoPlayerApi, DefaultLifecycleObserver, ActivityAware {
3341
private static final String TAG = "VideoPlayerPlugin";
3442
private final LongSparseArray<VideoPlayer> videoPlayers = new LongSparseArray<>();
3543
private FlutterState flutterState;
3644
private final VideoPlayerOptions options = new VideoPlayerOptions();
45+
@Nullable Lifecycle lifecycle;
3746

3847
/** Register this with the v2 embedding for the plugin to respond to lifecycle callbacks. */
3948
public VideoPlayerPlugin() {}
@@ -83,7 +92,7 @@ public void onDetachedFromEngine(@NonNull FlutterPluginBinding binding) {
8392
}
8493
flutterState.stopListening(binding.getBinaryMessenger());
8594
flutterState = null;
86-
onDestroy();
95+
performDestroy();
8796
}
8897

8998
private void disposeAllPlayers() {
@@ -93,7 +102,7 @@ private void disposeAllPlayers() {
93102
videoPlayers.clear();
94103
}
95104

96-
public void onDestroy() {
105+
public void performDestroy() {
97106
// The whole FlutterView is being destroyed. Here we release resources acquired for all
98107
// instances
99108
// of VideoPlayer. Once https://github.com/flutter/flutter/issues/19358 is resolved this may
@@ -107,8 +116,7 @@ public void initialize() {
107116
}
108117

109118
public @NonNull TextureMessage create(@NonNull CreateMessage arg) {
110-
TextureRegistry.SurfaceTextureEntry handle =
111-
flutterState.textureRegistry.createSurfaceTexture();
119+
TextureRegistry.SurfaceProducer handle = flutterState.textureRegistry.createSurfaceProducer();
112120
EventChannel eventChannel =
113121
new EventChannel(
114122
flutterState.binaryMessenger, "flutter.io/videoPlayer/videoEvents" + handle.id());
@@ -144,7 +152,6 @@ public void initialize() {
144152
options);
145153
}
146154
videoPlayers.put(handle.id(), player);
147-
148155
return new TextureMessage.Builder().setTextureId(handle.id()).build();
149156
}
150157

@@ -200,6 +207,62 @@ public void setMixWithOthers(@NonNull MixWithOthersMessage arg) {
200207
options.mixWithOthers = arg.getMixWithOthers();
201208
}
202209

210+
// Activity Aware
211+
212+
@Override
213+
public void onAttachedToActivity(@NonNull ActivityPluginBinding binding) {
214+
lifecycle = FlutterLifecycleAdapter.getActivityLifecycle(binding);
215+
lifecycle.addObserver(this);
216+
}
217+
218+
@Override
219+
public void onDetachedFromActivity() {}
220+
221+
@Override
222+
public void onReattachedToActivityForConfigChanges(@NonNull ActivityPluginBinding binding) {
223+
onAttachedToActivity(binding);
224+
}
225+
226+
@Override
227+
public void onDetachedFromActivityForConfigChanges() {
228+
onDetachedFromActivity();
229+
}
230+
231+
// DefaultLifecycleObserver
232+
@Override
233+
public void onResume(@NonNull LifecycleOwner owner) {
234+
recreateAllSurfaces();
235+
}
236+
237+
@Override
238+
public void onPause(@NonNull LifecycleOwner owner) {
239+
destroyAllSurfaces();
240+
}
241+
242+
@Override
243+
public void onStop(@NonNull LifecycleOwner owner) {
244+
destroyAllSurfaces();
245+
}
246+
247+
@Override
248+
public void onDestroy(@NonNull LifecycleOwner owner) {
249+
if (lifecycle != null) {
250+
lifecycle.removeObserver(this);
251+
}
252+
}
253+
254+
private void destroyAllSurfaces() {
255+
for (int i = 0; i < videoPlayers.size(); i++) {
256+
videoPlayers.valueAt(i).pauseSurface();
257+
}
258+
}
259+
260+
private void recreateAllSurfaces() {
261+
for (int i = 0; i < videoPlayers.size(); i++) {
262+
videoPlayers.valueAt(i).recreateSurface(flutterState.applicationContext);
263+
}
264+
}
265+
203266
private interface KeyForAssetFn {
204267
String get(String asset);
205268
}

‎packages/video_player/video_player_android/android/src/test/java/io/flutter/plugins/videoplayer/VideoPlayerTest.java‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import static org.mockito.Mockito.spy;
1414
import static org.mockito.Mockito.times;
1515

16-
import android.graphics.SurfaceTexture;
1716
import com.google.android.exoplayer2.ExoPlayer;
1817
import com.google.android.exoplayer2.Format;
1918
import com.google.android.exoplayer2.PlaybackException;
@@ -38,8 +37,7 @@
3837
public class VideoPlayerTest {
3938
private ExoPlayer fakeExoPlayer;
4039
private EventChannel fakeEventChannel;
41-
private TextureRegistry.SurfaceTextureEntry fakeSurfaceTextureEntry;
42-
private SurfaceTexture fakeSurfaceTexture;
40+
private TextureRegistry.SurfaceProducer fakeSurfaceProducer;
4341
private VideoPlayerOptions fakeVideoPlayerOptions;
4442
private QueuingEventSink fakeEventSink;
4543
private DefaultHttpDataSource.Factory httpDataSourceFactorySpy;
@@ -52,9 +50,7 @@ public void before() {
5250

5351
fakeExoPlayer = mock(ExoPlayer.class);
5452
fakeEventChannel = mock(EventChannel.class);
55-
fakeSurfaceTextureEntry = mock(TextureRegistry.SurfaceTextureEntry.class);
56-
fakeSurfaceTexture = mock(SurfaceTexture.class);
57-
when(fakeSurfaceTextureEntry.surfaceTexture()).thenReturn(fakeSurfaceTexture);
53+
fakeSurfaceProducer = mock(TextureRegistry.SurfaceProducer.class);
5854
fakeVideoPlayerOptions = mock(VideoPlayerOptions.class);
5955
fakeEventSink = mock(QueuingEventSink.class);
6056
httpDataSourceFactorySpy = spy(new DefaultHttpDataSource.Factory());
@@ -66,7 +62,7 @@ public void videoPlayer_buildsHttpDataSourceFactoryProperlyWhenHttpHeadersNull()
6662
new VideoPlayer(
6763
fakeExoPlayer,
6864
fakeEventChannel,
69-
fakeSurfaceTextureEntry,
65+
fakeSurfaceProducer,
7066
fakeVideoPlayerOptions,
7167
fakeEventSink,
7268
httpDataSourceFactorySpy);
@@ -85,7 +81,7 @@ public void videoPlayer_buildsHttpDataSourceFactoryProperlyWhenHttpHeadersNull()
8581
new VideoPlayer(
8682
fakeExoPlayer,
8783
fakeEventChannel,
88-
fakeSurfaceTextureEntry,
84+
fakeSurfaceProducer,
8985
fakeVideoPlayerOptions,
9086
fakeEventSink,
9187
httpDataSourceFactorySpy);
@@ -111,7 +107,7 @@ public void videoPlayer_buildsHttpDataSourceFactoryProperlyWhenHttpHeadersNull()
111107
new VideoPlayer(
112108
fakeExoPlayer,
113109
fakeEventChannel,
114-
fakeSurfaceTextureEntry,
110+
fakeSurfaceProducer,
115111
fakeVideoPlayerOptions,
116112
fakeEventSink,
117113
httpDataSourceFactorySpy);
@@ -135,7 +131,7 @@ public void sendInitializedSendsExpectedEvent_90RotationDegrees() {
135131
new VideoPlayer(
136132
fakeExoPlayer,
137133
fakeEventChannel,
138-
fakeSurfaceTextureEntry,
134+
fakeSurfaceProducer,
139135
fakeVideoPlayerOptions,
140136
fakeEventSink,
141137
httpDataSourceFactorySpy);
@@ -164,7 +160,7 @@ public void sendInitializedSendsExpectedEvent_270RotationDegrees() {
164160
new VideoPlayer(
165161
fakeExoPlayer,
166162
fakeEventChannel,
167-
fakeSurfaceTextureEntry,
163+
fakeSurfaceProducer,
168164
fakeVideoPlayerOptions,
169165
fakeEventSink,
170166
httpDataSourceFactorySpy);
@@ -193,7 +189,7 @@ public void sendInitializedSendsExpectedEvent_0RotationDegrees() {
193189
new VideoPlayer(
194190
fakeExoPlayer,
195191
fakeEventChannel,
196-
fakeSurfaceTextureEntry,
192+
fakeSurfaceProducer,
197193
fakeVideoPlayerOptions,
198194
fakeEventSink,
199195
httpDataSourceFactorySpy);
@@ -222,7 +218,7 @@ public void sendInitializedSendsExpectedEvent_180RotationDegrees() {
222218
new VideoPlayer(
223219
fakeExoPlayer,
224220
fakeEventChannel,
225-
fakeSurfaceTextureEntry,
221+
fakeSurfaceProducer,
226222
fakeVideoPlayerOptions,
227223
fakeEventSink,
228224
httpDataSourceFactorySpy);
@@ -251,7 +247,7 @@ public void onIsPlayingChangedSendsExpectedEvent() {
251247
new VideoPlayer(
252248
fakeExoPlayer,
253249
fakeEventChannel,
254-
fakeSurfaceTextureEntry,
250+
fakeSurfaceProducer,
255251
fakeVideoPlayerOptions,
256252
fakeEventSink,
257253
httpDataSourceFactorySpy);
@@ -296,7 +292,7 @@ public void behindLiveWindowErrorResetsPlayerToDefaultPosition() {
296292
new VideoPlayer(
297293
fakeExoPlayer,
298294
fakeEventChannel,
299-
fakeSurfaceTextureEntry,
295+
fakeSurfaceProducer,
300296
fakeVideoPlayerOptions,
301297
fakeEventSink,
302298
httpDataSourceFactorySpy);

‎packages/video_player/video_player_android/example/android/app/src/test/java/io/flutter/plugins/videoplayerexample/FlutterActivityTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,6 @@ public void disposeAllPlayers() {
4545

4646
engine.destroy();
4747
verify(videoPlayerPlugin, times(1)).onDetachedFromEngine(pluginBindingCaptor.capture());
48-
verify(videoPlayerPlugin, times(1)).onDestroy();
48+
verify(videoPlayerPlugin, times(1)).performDestroy();
4949
}
5050
}

0 commit comments

Comments
 (0)