Skip to content

[Android] Add a way to request new Surfaces from SurfaceProducer and avoid SurfaceProducer returning invalid Surface #169899

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Jun 20, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -555,10 +555,10 @@ void close() {
return (double) deltaNanos / 1000000.0;
}

PerImageReader getOrCreatePerImageReader(ImageReader reader) {
private PerImageReader getOrCreatePerImageReader(ImageReader reader) {
PerImageReader r = perImageReaders.get(reader);
if (r == null) {
r = new PerImageReader(reader);
r = createPerImageReader(reader);
perImageReaders.put(reader, r);
imageReaderQueue.add(r);
if (VERBOSE_LOGS) {
Expand All @@ -568,6 +568,11 @@ PerImageReader getOrCreatePerImageReader(ImageReader reader) {
return r;
}

@VisibleForTesting
public PerImageReader createPerImageReader(ImageReader reader) {
return new PerImageReader(reader);
}

void pruneImageReaderQueue() {
boolean change = false;
// Prune nodes from the head of the ImageReader queue.
Expand Down Expand Up @@ -829,6 +834,12 @@ public Surface getSurface() {
return pir.reader.getSurface();
}

@Override
public Surface getForcedNewSurface() {
createNewReader = true;
return getSurface();
}

@Override
public void scheduleFrame() {
if (VERBOSE_LOGS) {
Expand All @@ -855,17 +866,23 @@ public Image acquireLatestImage() {

private PerImageReader getActiveReader() {
synchronized (lock) {
if (createNewReader) {
createNewReader = false;
// Create a new ImageReader and add it to the queue.
ImageReader reader = createImageReader();
if (VERBOSE_LOGS) {
Log.i(
TAG, reader.hashCode() + " created w=" + requestedWidth + " h=" + requestedHeight);
if (!createNewReader) {
// Verify we don't need a new ImageReader anyway because its Surface has been invalidated.
PerImageReader lastPerImageReader = imageReaderQueue.peekLast();
Surface lastImageReaderSurface = lastPerImageReader.reader.getSurface();
boolean lastImageReaderHasValidSurface = lastImageReaderSurface.isValid();
if (lastImageReaderHasValidSurface) {
return lastPerImageReader;
}
return getOrCreatePerImageReader(reader);
}
return imageReaderQueue.peekLast();

createNewReader = false;
// Create a new ImageReader and add it to the queue.
ImageReader reader = createImageReader();
if (VERBOSE_LOGS) {
Log.i(TAG, reader.hashCode() + " created w=" + requestedWidth + " h=" + requestedHeight);
}
return getOrCreatePerImageReader(reader);
}
}

Expand Down Expand Up @@ -909,7 +926,8 @@ private ImageReader createImageReader29() {
HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
}

private ImageReader createImageReader() {
@VisibleForTesting
public ImageReader createImageReader() {
if (Build.VERSION.SDK_INT >= API_LEVELS.API_33) {
return createImageReader33();
} else if (Build.VERSION.SDK_INT >= API_LEVELS.API_29) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import android.view.Surface;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import io.flutter.embedding.engine.FlutterJNI;
import io.flutter.view.TextureRegistry;

Expand Down Expand Up @@ -92,12 +93,23 @@ public int getHeight() {

@Override
public Surface getSurface() {
if (surface == null) {
surface = new Surface(texture.surfaceTexture());
if (surface == null || !surface.isValid()) {
surface = createSurface(texture.surfaceTexture());
}
return surface;
}

@Override
public Surface getForcedNewSurface() {
surface = null;
return getSurface();
}

@VisibleForTesting
public Surface createSurface(SurfaceTexture surfaceTexture) {
return new Surface(surfaceTexture);
}

@Override
public void scheduleFrame() {
flutterJNI.markTextureFrameAvailable(id);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,24 @@ interface SurfaceProducer extends TextureEntry {
*/
Surface getSurface();

/**
* Direct access to a surface, which will be newly created (and thus, different from any surface
* objects returned from previous calls to {@link #getSurface()} or {@link
* #getForcedNewSurface()}.
*
* <p>When using this API, you will usually need to implement {@link SurfaceProducer.Callback}
* and provide it to {@link #setCallback(Callback)} in order to be notified when an existing
* surface has been destroyed (such as when the application goes to the background) or a new
* surface has been created (such as when the application is resumed back to the foreground).
*
* <p>NOTE: You should not cache the returned surface but instead invoke {@code getSurface} each
* time you need to draw. The surface may change when the texture is resized or has its format
* changed.
*
* @return a Surface to use for a drawing target for various APIs.
*/
Surface getForcedNewSurface();

/**
* Sets a callback that is notified when a previously created {@link Surface} returned by {@link
* SurfaceProducer#getSurface()} is no longer valid due to being destroyed, or a new surface is
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
Expand All @@ -21,12 +22,14 @@
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.robolectric.Shadows.shadowOf;

import android.graphics.Canvas;
import android.graphics.Rect;
import android.graphics.SurfaceTexture;
import android.media.Image;
import android.media.ImageReader;
import android.os.Looper;
import android.view.Surface;
import androidx.lifecycle.Lifecycle;
Expand Down Expand Up @@ -991,4 +994,72 @@ public void ImageReaderSurfaceProducerSchedulesFrameIfQueueNotEmpty() throws Exc
// is now empty.
verify(flutterRenderer, times(3)).scheduleEngineFrame();
}

@Test
public void getSurface_doesNotReturnInvalidSurface() {
FlutterRenderer flutterRenderer = spy(engineRule.getFlutterEngine().getRenderer());
TextureRegistry.SurfaceProducer producer = flutterRenderer.createSurfaceProducer();
FlutterRenderer.ImageReaderSurfaceProducer spyImageReaderSurfaceProducer =
spy((FlutterRenderer.ImageReaderSurfaceProducer) producer);
ImageReader mockImageReader = mock(ImageReader.class);
ImageReader mockSecondImageReader = mock(ImageReader.class);
Surface firstMockSurface = mock(Surface.class);
Surface secondMockSurface = mock(Surface.class);

when(mockImageReader.getSurface()).thenReturn(firstMockSurface);
when(mockSecondImageReader.getSurface()).thenReturn(secondMockSurface);
when(firstMockSurface.isValid()).thenReturn(false);
when(spyImageReaderSurfaceProducer.createImageReader())
.thenReturn(mockImageReader)
.thenReturn(mockSecondImageReader);

Surface firstSurface = spyImageReaderSurfaceProducer.getSurface();
Surface secondSurface = spyImageReaderSurfaceProducer.getSurface();

assertNotEquals(firstSurface, secondSurface);
assertEquals(firstSurface, firstMockSurface);
assertEquals(secondSurface, secondMockSurface);
}

@Test
public void getSurface_consecutiveCallsReturnSameSurfaceIfStillValid() {
FlutterRenderer flutterRenderer = spy(engineRule.getFlutterEngine().getRenderer());
TextureRegistry.SurfaceProducer producer = flutterRenderer.createSurfaceProducer();
FlutterRenderer.ImageReaderSurfaceProducer spyImageReaderSurfaceProducer =
spy((FlutterRenderer.ImageReaderSurfaceProducer) producer);
ImageReader mockImageReader = mock(ImageReader.class);
Surface mockSurface = mock(Surface.class);

when(mockSurface.isValid()).thenReturn(true);
when(mockImageReader.getSurface()).thenReturn(mockSurface);
when(spyImageReaderSurfaceProducer.createImageReader()).thenReturn(mockImageReader);

Surface firstSurface = spyImageReaderSurfaceProducer.getSurface();
Surface secondSurface = spyImageReaderSurfaceProducer.getSurface();

assertEquals(firstSurface, secondSurface);
assertEquals(firstSurface, mockSurface);
}

@Test
public void getForcedNewSurface_returnsNewSurface() {
FlutterRenderer flutterRenderer = spy(engineRule.getFlutterEngine().getRenderer());
TextureRegistry.SurfaceProducer producer = flutterRenderer.createSurfaceProducer();

Surface firstSurface = producer.getSurface();
Surface secondSurface = producer.getForcedNewSurface();

assertNotEquals(firstSurface, secondSurface);
}

@Test
public void getSurface_doesNotReturnNewSurface() {
FlutterRenderer flutterRenderer = spy(engineRule.getFlutterEngine().getRenderer());
TextureRegistry.SurfaceProducer producer = flutterRenderer.createSurfaceProducer();

Surface firstSurface = producer.getSurface();
Surface secondSurface = producer.getSurface();

assertEquals(firstSurface, secondSurface);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@
import static io.flutter.Build.API_LEVELS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.robolectric.Shadows.shadowOf;

import android.annotation.TargetApi;
Expand All @@ -19,6 +22,7 @@
import android.view.Surface;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import io.flutter.embedding.engine.FlutterJNI;
import io.flutter.view.TextureRegistry;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand Down Expand Up @@ -84,4 +88,77 @@ public void releaseWillReleaseSurface() {
producer.release();
assertFalse(surface.isValid());
}

@Test
public void getSurface_doesNotReturnInvalidSurface() {
final FlutterRenderer flutterRenderer = new FlutterRenderer(fakeJNI);
final Handler handler = new Handler(Looper.getMainLooper());
final SurfaceTexture mockSurfaceTexture = mock(SurfaceTexture.class);
final TextureRegistry.SurfaceTextureEntry spyTexture =
spy(flutterRenderer.registerSurfaceTexture(mockSurfaceTexture));
final SurfaceTextureSurfaceProducer producerSpy =
spy(new SurfaceTextureSurfaceProducer(0, handler, fakeJNI, spyTexture));
final Surface firstMockSurface = mock(Surface.class);
final Surface secondMockSurface = mock(Surface.class);

when(spyTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
when(firstMockSurface.isValid()).thenReturn(false);
when(producerSpy.createSurface(mockSurfaceTexture))
.thenReturn(firstMockSurface)
.thenReturn(secondMockSurface);

final Surface firstSurface = producerSpy.getSurface();
final Surface secondSurface = producerSpy.getSurface();

assertNotEquals(firstSurface, secondSurface);
}

@Test
public void getSurface_consecutiveCallsReturnSameSurfaceIfStillValid() {
final FlutterRenderer flutterRenderer = new FlutterRenderer(fakeJNI);
final Handler handler = new Handler(Looper.getMainLooper());
final SurfaceTexture mockSurfaceTexture = mock(SurfaceTexture.class);
final TextureRegistry.SurfaceTextureEntry spyTexture =
spy(flutterRenderer.registerSurfaceTexture(mockSurfaceTexture));
final SurfaceTextureSurfaceProducer producerSpy =
spy(new SurfaceTextureSurfaceProducer(0, handler, fakeJNI, spyTexture));
final Surface mockSurface = mock(Surface.class);

when(spyTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
when(mockSurface.isValid()).thenReturn(true);
when(producerSpy.createSurface(mockSurfaceTexture)).thenReturn(mockSurface);

final Surface firstSurface = producerSpy.getSurface();
final Surface secondSurface = producerSpy.getSurface();

assertEquals(firstSurface, secondSurface);
}

@Test
public void getForcedNewSurface_returnsNewSurface() {
final FlutterRenderer flutterRenderer = new FlutterRenderer(fakeJNI);
final Handler handler = new Handler(Looper.getMainLooper());
final SurfaceTextureSurfaceProducer producer =
new SurfaceTextureSurfaceProducer(
0, handler, fakeJNI, flutterRenderer.registerSurfaceTexture(new SurfaceTexture(0)));

final Surface firstSurface = producer.getSurface();
final Surface secondSurface = producer.getForcedNewSurface();

assertNotEquals(firstSurface, secondSurface);
}

@Test
public void getSurface_doesNotReturnNewSurface() {
final FlutterRenderer flutterRenderer = new FlutterRenderer(fakeJNI);
final Handler handler = new Handler(Looper.getMainLooper());
final SurfaceTextureSurfaceProducer producer =
new SurfaceTextureSurfaceProducer(
0, handler, fakeJNI, flutterRenderer.registerSurfaceTexture(new SurfaceTexture(0)));

Surface firstSurface = producer.getSurface();
Surface secondSurface = producer.getSurface();

assertEquals(firstSurface, secondSurface);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1689,6 +1689,11 @@ public Surface getSurface() {
return null;
}

@Override
public Surface getForcedNewSurface() {
return null;
}

@Override
public boolean handlesCropAndRotation() {
return false;
Expand Down