From a123723cb51fed70ccc25cb6b02765e84389819b Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 3 Aug 2023 08:11:57 -0700 Subject: [PATCH 1/9] [Flutter GPU] Export symbols from engine, stub for testing on CI. (#44280) Part of https://github.com/flutter/flutter/issues/131346 Stubs a minimal test of the FFI utilities that `dart:ui` uses, but using public symbols exported from the engine library. If this goes well, I'll move the stuff from `dart:ui` into here and begin landing parts of the API with test coverage. --- ci/licenses_golden/licenses_flutter | 6 +++ common/exported_symbols.sym | 2 + common/exported_symbols_mac.sym | 2 + lib/gpu/BUILD.gn | 51 +++++++++++++++++++++++ lib/gpu/gpu.cc | 60 ++++++++++++++++++++++++++++ lib/gpu/gpu.dart | 39 ++++++++++++++++++ lib/gpu/gpu.h | 57 ++++++++++++++++++++++++++ runtime/BUILD.gn | 1 + testing/dart/BUILD.gn | 1 + testing/dart/gpu_test.dart | 24 +++++++++++ testing/symbols/verify_exported.dart | 13 ++++-- 11 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 lib/gpu/BUILD.gn create mode 100644 lib/gpu/gpu.cc create mode 100644 lib/gpu/gpu.dart create mode 100644 lib/gpu/gpu.h create mode 100644 testing/dart/gpu_test.dart diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 0d16668ffb5f8..9207b90c4caae 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1716,6 +1716,9 @@ ORIGIN: ../../../flutter/impeller/typographer/text_run.cc + ../../../flutter/LIC ORIGIN: ../../../flutter/impeller/typographer/text_run.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/typographer/typeface.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/typographer/typeface.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/gpu/gpu.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/gpu/gpu.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/gpu/gpu.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/io/dart_io.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/io/dart_io.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/snapshot/snapshot.h + ../../../flutter/LICENSE @@ -4415,6 +4418,9 @@ FILE: ../../../flutter/impeller/typographer/text_run.cc FILE: ../../../flutter/impeller/typographer/text_run.h FILE: ../../../flutter/impeller/typographer/typeface.cc FILE: ../../../flutter/impeller/typographer/typeface.h +FILE: ../../../flutter/lib/gpu/gpu.cc +FILE: ../../../flutter/lib/gpu/gpu.dart +FILE: ../../../flutter/lib/gpu/gpu.h FILE: ../../../flutter/lib/io/dart_io.cc FILE: ../../../flutter/lib/io/dart_io.h FILE: ../../../flutter/lib/snapshot/libraries_experimental.json diff --git a/common/exported_symbols.sym b/common/exported_symbols.sym index d85c253e3c07b..c9ad42cd3350e 100644 --- a/common/exported_symbols.sym +++ b/common/exported_symbols.sym @@ -5,4 +5,6 @@ kDartVmSnapshotInstructions; kDartIsolateSnapshotData; kDartIsolateSnapshotInstructions; + InternalFlutterGpu*; + kInternalFlutterGpu*; }; diff --git a/common/exported_symbols_mac.sym b/common/exported_symbols_mac.sym index c65b740925271..09687a75427f5 100644 --- a/common/exported_symbols_mac.sym +++ b/common/exported_symbols_mac.sym @@ -4,3 +4,5 @@ _kDartVmSnapshotData _kDartVmSnapshotInstructions _kDartIsolateSnapshotData _kDartIsolateSnapshotInstructions +_InternalFlutterGpu* +_kInternalFlutterGpu* diff --git a/lib/gpu/BUILD.gn b/lib/gpu/BUILD.gn new file mode 100644 index 0000000000000..7b24bf4007907 --- /dev/null +++ b/lib/gpu/BUILD.gn @@ -0,0 +1,51 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//flutter/common/config.gni") +import("//flutter/impeller/tools/impeller.gni") +import("//flutter/testing/testing.gni") + +source_set("gpu") { + cflags = [ + # Dart gives us doubles. Skia and Impeller work in floats. + # If Dart gives us a double > FLT_MAX or < -FLT_MAX, implicit conversion + # will convert it to either inf/-inf or FLT_MAX/-FLT_MAX (undefined + # behavior). This can result in surprising and difficult to debug behavior + # for Flutter application developers, so it should be explicitly handled + # via SafeNarrow. + "-Wimplicit-float-conversion", + ] + + if (is_win) { + # This causes build failures in third_party dependencies on Windows. + cflags += [ "-Wno-implicit-int-float-conversion" ] + } + + public_configs = [ "//flutter:config" ] + + public_deps = [] + + if (!defined(defines)) { + defines = [] + } + + sources = [ + "gpu.cc", + "gpu.h", + ] + deps = [ + "//flutter/impeller", + "//flutter/impeller/display_list:skia_conversions", + "//flutter/lib/ui", + "//flutter/third_party/tonic", + ] + + if (is_win) { + # Required for M_PI and others. + defines += [ "_USE_MATH_DEFINES" ] + } +} + +if (enable_unittests) { +} diff --git a/lib/gpu/gpu.cc b/lib/gpu/gpu.cc new file mode 100644 index 0000000000000..4d6bf37bbcfb2 --- /dev/null +++ b/lib/gpu/gpu.cc @@ -0,0 +1,60 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/lib/gpu/gpu.h" +#include "flutter/fml/memory/ref_ptr.h" +#include "flutter/lib/ui/dart_wrapper.h" +#include "flutter/lib/ui/ui_dart_state.h" +#include "third_party/dart/runtime/include/dart_api.h" +#include "third_party/tonic/converter/dart_converter.h" +#include "third_party/tonic/dart_state.h" +#include "third_party/tonic/dart_wrappable.h" +#include "third_party/tonic/dart_wrapper_info.h" +#include "third_party/tonic/logging/dart_invoke.h" + +namespace flutter { + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +IMPLEMENT_WRAPPERTYPEINFO(gpu, FlutterGpuTestClass); + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +FlutterGpuTestClass::~FlutterGpuTestClass() = default; + +} // namespace flutter + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +uint32_t InternalFlutterGpuTestProc() { + return 1; +} + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +Dart_Handle InternalFlutterGpuTestProcWithCallback(Dart_Handle callback) { + flutter::UIDartState::ThrowIfUIOperationsProhibited(); + if (!Dart_IsClosure(callback)) { + return tonic::ToDart("Callback must be a function"); + } + + tonic::DartInvoke(callback, {tonic::ToDart(1234)}); + + return Dart_Null(); +} + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +void InternalFlutterGpuTestClass_Create(Dart_Handle wrapper) { + auto res = fml::MakeRefCounted(); + res->AssociateWithDartWrapper(wrapper); + FML_LOG(INFO) << "FlutterGpuTestClass Wrapped."; +} + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +void InternalFlutterGpuTestClass_Method(flutter::FlutterGpuTestClass* self, + int something) { + FML_LOG(INFO) << "Something: " << something; +} + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +void FlutterGpuTestClass_Dispose(flutter::FlutterGpuTestClass* self, + int something) { + self->ClearDartWrapper(); +} diff --git a/lib/gpu/gpu.dart b/lib/gpu/gpu.dart new file mode 100644 index 0000000000000..039d11585ecb2 --- /dev/null +++ b/lib/gpu/gpu.dart @@ -0,0 +1,39 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// ignore_for_file: public_member_api_docs + +import 'dart:ffi'; +import 'dart:nativewrappers'; + +/// This is a simple test fuction. +@Native(symbol: 'InternalFlutterGpuTestProc') +external int testProc(); + +/// This is a test callback that follows the same pattern as much of dart:ui -- +/// immediately returning an error string and supplying an asynchronous result +/// via callback later. +typedef Callback = void Function(T result); +@Native( + symbol: 'InternalFlutterGpuTestProcWithCallback') +external String? testProcWithCallback(Callback callback); + +/// This is a test of NativeFieldWrapperClass1, which is commonly used in +/// dart:ui to enable Dart to dictate the lifetime of a C counterpart. +base class FlutterGpuTestClass extends NativeFieldWrapperClass1 { + FlutterGpuTestClass() { + _constructor(); + } + + /// This "constructor" is used to instantiate and wrap the C counterpart. + /// This is a common pattern in dart:ui. + @Native(symbol: 'InternalFlutterGpuTestClass_Create') + external void _constructor(); + + /// This is a method that will supply a pointer to the C data counterpart when + /// calling the gunction + @Native, Int)>( + symbol: 'InternalFlutterGpuTestClass_Method') + external void coolMethod(int something); +} diff --git a/lib/gpu/gpu.h b/lib/gpu/gpu.h new file mode 100644 index 0000000000000..0007845a3460a --- /dev/null +++ b/lib/gpu/gpu.h @@ -0,0 +1,57 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_LIB_GPU_GPU_H_ +#define FLUTTER_LIB_GPU_GPU_H_ + +#include + +#include "flutter/lib/ui/dart_wrapper.h" +#include "third_party/dart/runtime/include/dart_api.h" +#include "third_party/tonic/dart_state.h" +#include "third_party/tonic/dart_wrapper_info.h" + +#if FML_OS_WIN +#define FLUTTER_EXPORT __declspec(dllexport) +#else // FML_OS_WIN +#define FLUTTER_EXPORT __attribute__((visibility("default"))) +#endif // FML_OS_WIN + +namespace flutter { + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +class FlutterGpuTestClass + : public RefCountedDartWrappable { + DEFINE_WRAPPERTYPEINFO(); + FML_FRIEND_MAKE_REF_COUNTED(FlutterGpuTestClass); + + public: + ~FlutterGpuTestClass() override; +}; + +} // namespace flutter + +extern "C" { + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +FLUTTER_EXPORT +extern uint32_t InternalFlutterGpuTestProc(); + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +FLUTTER_EXPORT +extern Dart_Handle InternalFlutterGpuTestProcWithCallback(Dart_Handle callback); + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +FLUTTER_EXPORT +extern void InternalFlutterGpuTestClass_Create(Dart_Handle wrapper); + +// TODO(131346): Remove this once we migrate the Dart GPU API into this space. +FLUTTER_EXPORT +extern void InternalFlutterGpuTestClass_Method( + flutter::FlutterGpuTestClass* self, + int something); + +} // extern "C" + +#endif // FLUTTER_LIB_GPU_GPU_H_ diff --git a/runtime/BUILD.gn b/runtime/BUILD.gn index 70e9172d51dd3..8d07eb8e1dfde 100644 --- a/runtime/BUILD.gn +++ b/runtime/BUILD.gn @@ -90,6 +90,7 @@ source_set("runtime") { } public_deps = [ + "//flutter/lib/gpu", "//flutter/lib/ui", "//third_party/rapidjson", ] diff --git a/testing/dart/BUILD.gn b/testing/dart/BUILD.gn index 1c372c5ed9ea1..2e1bb1ad38f67 100644 --- a/testing/dart/BUILD.gn +++ b/testing/dart/BUILD.gn @@ -17,6 +17,7 @@ tests = [ "fragment_shader_test.dart", "geometry_test.dart", "gesture_settings_test.dart", + "gpu_test.dart", "gradient_test.dart", "http_allow_http_connections_test.dart", "http_disallow_http_connections_test.dart", diff --git a/testing/dart/gpu_test.dart b/testing/dart/gpu_test.dart new file mode 100644 index 0000000000000..e6fab0247831d --- /dev/null +++ b/testing/dart/gpu_test.dart @@ -0,0 +1,24 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// ignore_for_file: avoid_relative_lib_imports + +import 'package:litetest/litetest.dart'; +import '../../lib/gpu/gpu.dart' as gpu; + +void main() { + // TODO(131346): Remove this once we migrate the Dart GPU API into this space. + test('no crash', () async { + final int result = gpu.testProc(); + expect(result, 1); + + final String? message = gpu.testProcWithCallback((int result) { + expect(result, 1234); + }); + expect(message, null); + + final gpu.FlutterGpuTestClass a = gpu.FlutterGpuTestClass(); + a.coolMethod(9847); + }); +} diff --git a/testing/symbols/verify_exported.dart b/testing/symbols/verify_exported.dart index 2e97f3e755219..22e9736aff9cb 100644 --- a/testing/symbols/verify_exported.dart +++ b/testing/symbols/verify_exported.dart @@ -94,9 +94,12 @@ int _checkIos(String outPath, String nmPath, Iterable builds) { continue; } final Iterable unexpectedEntries = NmEntry.parse(nmResult.stdout as String).where((NmEntry entry) { - return !(((entry.type == '(__DATA,__common)' || entry.type == '(__DATA,__const)') && entry.name.startsWith('_Flutter')) - || (entry.type == '(__DATA,__objc_data)' - && (entry.name.startsWith(r'_OBJC_METACLASS_$_Flutter') || entry.name.startsWith(r'_OBJC_CLASS_$_Flutter')))); + final bool cSymbol = (entry.type == '(__DATA,__common)' || entry.type == '(__DATA,__const)') + && entry.name.startsWith('_Flutter'); + final bool cInternalSymbol = entry.type == '(__TEXT,__text)' && entry.name.startsWith('_InternalFlutter'); + final bool objcSymbol = entry.type == '(__DATA,__objc_data)' + && (entry.name.startsWith(r'_OBJC_METACLASS_$_Flutter') || entry.name.startsWith(r'_OBJC_CLASS_$_Flutter')); + return !(cSymbol || cInternalSymbol || objcSymbol); }); if (unexpectedEntries.isNotEmpty) { print('ERROR: $libFlutter exports unexpected symbols:'); @@ -275,7 +278,9 @@ int _checkLinux(String outPath, String nmPath, Iterable builds) { } if (!(entry.name.startsWith('Flutter') || entry.name.startsWith('__Flutter') - || entry.name.startsWith('kFlutter'))) { + || entry.name.startsWith('kFlutter') + || entry.name.startsWith('InternalFlutter') + || entry.name.startsWith('kInternalFlutter'))) { print('ERROR: $libFlutter exports an unexpected symbol name: ($entry)'); print(' Library has $entries.'); failures++; From e589d597c317f8ec549fe486ab8ce6e2590bf7d9 Mon Sep 17 00:00:00 2001 From: skia-flutter-autoroll Date: Thu, 3 Aug 2023 11:55:55 -0400 Subject: [PATCH 2/9] Roll Skia from 8a377a9545b8 to 768050436c0a (1 revision) (#44320) https://skia.googlesource.com/skia.git/+log/8a377a9545b8..768050436c0a 2023-08-03 johnstiles@google.com Remove or replace GCE Perf tests. If this roll has caused a breakage, revert this CL and stop the roller using the controls here: https://autoroll.skia.org/r/skia-flutter-autoroll Please CC brianosman@google.com,jacksongardner@google.com,jvanverth@google.com,rmistry@google.com on the revert to ensure that a human is aware of the problem. To file a bug in Skia: https://bugs.chromium.org/p/skia/issues/entry To file a bug in Flutter: https://github.com/flutter/flutter/issues/new/choose To report a problem with the AutoRoller itself, please file a bug: https://bugs.chromium.org/p/skia/issues/entry?template=Autoroller+Bug Documentation for the AutoRoller is here: https://skia.googlesource.com/buildbot/+doc/main/autoroll/README.md --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index bc6379b48ac5a..8469aab4ce005 100644 --- a/DEPS +++ b/DEPS @@ -18,7 +18,7 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', # OCMock is for testing only so there is no google clone 'ocmock_git': 'https://github.com/erikdoe/ocmock.git', - 'skia_revision': '8a377a9545b8d7f274ceec1eb9e6fc816ff0425c', + 'skia_revision': '768050436c0abe18284c9d0e9371ab102854fba3', # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. From 0832dfa3c7e00316e16c94ada41d9ce412fceebe Mon Sep 17 00:00:00 2001 From: John McCutchan Date: Thu, 3 Aug 2023 09:49:55 -0700 Subject: [PATCH 3/9] Re-Re-Land Support for rendering Android Platform Views into a HardwareBuffer backed texture (#44326) Introduce TextureRegistry.ImageTexture and related machinery. Introduce ImageReaderPlatformViewRenderTarget. Introduce HardwareBufferExternalTextureGL and related machinery. NOTE: ImageReaderPlatformViewRenderTarget requires Android 26. NOTE: This CL does not enable using ImageReaderPlatformViewRenderTarget yet. Related https://github.com/flutter/flutter/issues/130892 --------- Co-authored-by: Jason Simmons --- ci/licenses_golden/licenses_flutter | 17 ++ impeller/toolkit/egl/BUILD.gn | 2 + impeller/toolkit/egl/egl.h | 2 + impeller/toolkit/egl/image.cc | 3 + impeller/toolkit/egl/image.h | 74 ++++++++ impeller/toolkit/gles/BUILD.gn | 20 ++ impeller/toolkit/gles/gles.h | 9 + impeller/toolkit/gles/texture.cc | 3 + impeller/toolkit/gles/texture.h | 39 ++++ shell/platform/android/BUILD.gn | 6 + .../android/android_context_gl_skia.cc | 8 + .../android/android_context_gl_skia.h | 14 ++ .../android/android_external_texture_gl.cc | 14 +- .../android/android_external_texture_gl.h | 4 +- .../android/android_shell_holder_unittests.cc | 6 + .../hardware_buffer_external_texture_gl.cc | 164 +++++++++++++++++ .../hardware_buffer_external_texture_gl.h | 71 +++++++ .../flutter/embedding/engine/FlutterJNI.java | 22 +++ .../engine/renderer/FlutterRenderer.java | 89 ++++++++- .../ImageReaderPlatformViewRenderTarget.java | 140 ++++++++++++++ .../plugin/platform/PlatformViewWrapper.java | 7 - .../platform/PlatformViewsController.java | 24 ++- ...urfaceTexturePlatformViewRenderTarget.java | 2 +- .../platform/VirtualDisplayController.java | 12 +- .../android/io/flutter/view/FlutterView.java | 6 + .../io/flutter/view/TextureRegistry.java | 59 +++++- shell/platform/android/jni/jni_mock.h | 17 ++ .../android/jni/platform_view_android_jni.h | 21 +++ shell/platform/android/library_loader.cc | 1 + shell/platform/android/ndk_helpers.cc | 101 ++++++++++ shell/platform/android/ndk_helpers.h | 37 ++++ .../platform/android/platform_view_android.cc | 14 ++ .../platform/android/platform_view_android.h | 5 + .../android/platform_view_android_jni_impl.cc | 174 +++++++++++++++++- .../android/platform_view_android_jni_impl.h | 9 + .../platform/PlatformViewsControllerTest.java | 22 +++ 36 files changed, 1167 insertions(+), 51 deletions(-) create mode 100644 impeller/toolkit/egl/image.cc create mode 100644 impeller/toolkit/egl/image.h create mode 100644 impeller/toolkit/gles/BUILD.gn create mode 100644 impeller/toolkit/gles/gles.h create mode 100644 impeller/toolkit/gles/texture.cc create mode 100644 impeller/toolkit/gles/texture.h create mode 100644 shell/platform/android/hardware_buffer_external_texture_gl.cc create mode 100644 shell/platform/android/hardware_buffer_external_texture_gl.h create mode 100644 shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java create mode 100644 shell/platform/android/ndk_helpers.cc create mode 100644 shell/platform/android/ndk_helpers.h diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 9207b90c4caae..787aec593cd39 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1688,8 +1688,11 @@ ORIGIN: ../../../flutter/impeller/toolkit/egl/display.cc + ../../../flutter/LICE ORIGIN: ../../../flutter/impeller/toolkit/egl/display.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/toolkit/egl/egl.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/toolkit/egl/egl.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/toolkit/egl/image.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/toolkit/egl/surface.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/toolkit/egl/surface.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/toolkit/gles/gles.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/toolkit/gles/texture.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_render_context_skia.cc + ../../../flutter/LICENSE @@ -2327,6 +2330,8 @@ ORIGIN: ../../../flutter/shell/platform/android/external_view_embedder/surface_p ORIGIN: ../../../flutter/shell/platform/android/external_view_embedder/surface_pool.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/flutter_main.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/flutter_main.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/Log.java + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/app/FlutterActivity.java + ../../../flutter/LICENSE @@ -2468,6 +2473,8 @@ ORIGIN: ../../../flutter/shell/platform/android/jni/jni_mock.h + ../../../flutte ORIGIN: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/library_loader.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/ndk_helpers.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/ndk_helpers.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/platform_message_handler_android.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/platform_message_response_android.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/platform_message_response_android.h + ../../../flutter/LICENSE @@ -4389,8 +4396,13 @@ FILE: ../../../flutter/impeller/toolkit/egl/display.cc FILE: ../../../flutter/impeller/toolkit/egl/display.h FILE: ../../../flutter/impeller/toolkit/egl/egl.cc FILE: ../../../flutter/impeller/toolkit/egl/egl.h +FILE: ../../../flutter/impeller/toolkit/egl/image.cc +FILE: ../../../flutter/impeller/toolkit/egl/image.h FILE: ../../../flutter/impeller/toolkit/egl/surface.cc FILE: ../../../flutter/impeller/toolkit/egl/surface.h +FILE: ../../../flutter/impeller/toolkit/gles/gles.h +FILE: ../../../flutter/impeller/toolkit/gles/texture.cc +FILE: ../../../flutter/impeller/toolkit/gles/texture.h FILE: ../../../flutter/impeller/tools/malioc.json FILE: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.cc FILE: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.h @@ -5030,6 +5042,8 @@ FILE: ../../../flutter/shell/platform/android/external_view_embedder/surface_poo FILE: ../../../flutter/shell/platform/android/external_view_embedder/surface_pool.h FILE: ../../../flutter/shell/platform/android/flutter_main.cc FILE: ../../../flutter/shell/platform/android/flutter_main.h +FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.cc +FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.h FILE: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java FILE: ../../../flutter/shell/platform/android/io/flutter/Log.java FILE: ../../../flutter/shell/platform/android/io/flutter/app/FlutterActivity.java @@ -5147,6 +5161,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/editing/TextInpu FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/localization/LocalizationPlugin.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/mouse/MouseCursorPlugin.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/AccessibilityEventsDelegate.java +FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformOverlayView.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformPlugin.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformView.java @@ -5179,6 +5194,8 @@ FILE: ../../../flutter/shell/platform/android/jni/jni_mock.h FILE: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.cc FILE: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.h FILE: ../../../flutter/shell/platform/android/library_loader.cc +FILE: ../../../flutter/shell/platform/android/ndk_helpers.cc +FILE: ../../../flutter/shell/platform/android/ndk_helpers.h FILE: ../../../flutter/shell/platform/android/platform_message_handler_android.cc FILE: ../../../flutter/shell/platform/android/platform_message_handler_android.h FILE: ../../../flutter/shell/platform/android/platform_message_response_android.cc diff --git a/impeller/toolkit/egl/BUILD.gn b/impeller/toolkit/egl/BUILD.gn index ec34bfead9829..82d10a14d195e 100644 --- a/impeller/toolkit/egl/BUILD.gn +++ b/impeller/toolkit/egl/BUILD.gn @@ -14,6 +14,8 @@ impeller_component("egl") { "display.h", "egl.cc", "egl.h", + "image.cc", + "image.h", "surface.cc", "surface.h", ] diff --git a/impeller/toolkit/egl/egl.h b/impeller/toolkit/egl/egl.h index 63c57075d63a9..af1bcca52ffb3 100644 --- a/impeller/toolkit/egl/egl.h +++ b/impeller/toolkit/egl/egl.h @@ -5,6 +5,8 @@ #pragma once #include +#define EGL_EGLEXT_PROTOTYPES +#include #include diff --git a/impeller/toolkit/egl/image.cc b/impeller/toolkit/egl/image.cc new file mode 100644 index 0000000000000..c1f17ebddc960 --- /dev/null +++ b/impeller/toolkit/egl/image.cc @@ -0,0 +1,3 @@ +#include "flutter/impeller/toolkit/egl/image.h" + +namespace impeller {} diff --git a/impeller/toolkit/egl/image.h b/impeller/toolkit/egl/image.h new file mode 100644 index 0000000000000..2fafe0f994b82 --- /dev/null +++ b/impeller/toolkit/egl/image.h @@ -0,0 +1,74 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "flutter/fml/unique_object.h" +#include "flutter/impeller/toolkit/egl/egl.h" + +namespace impeller { + +// Simple holder of an EGLImage and the owning EGLDisplay. +struct EGLImageWithDisplay { + EGLImage image = EGL_NO_IMAGE; + EGLDisplay display = EGL_NO_DISPLAY; + + constexpr bool operator==(const EGLImageWithDisplay& other) const { + return image == other.image && display == other.display; + } + + constexpr bool operator!=(const EGLImageWithDisplay& other) const { + return !(*this == other); + } +}; + +struct EGLImageWithDisplayTraits { + static EGLImageWithDisplay InvalidValue() { + return {EGL_NO_IMAGE, EGL_NO_DISPLAY}; + } + + static bool IsValid(const EGLImageWithDisplay& value) { + return value != InvalidValue(); + } + + static void Free(EGLImageWithDisplay image) { + eglDestroyImage(image.display, image.image); + } +}; + +using UniqueEGLImage = + fml::UniqueObject; + +// Simple holder of an EGLImageKHR and the owning EGLDisplay. +struct EGLImageKHRWithDisplay { + EGLImageKHR image = EGL_NO_IMAGE_KHR; + EGLDisplay display = EGL_NO_DISPLAY; + + constexpr bool operator==(const EGLImageKHRWithDisplay& other) const { + return image == other.image && display == other.display; + } + + constexpr bool operator!=(const EGLImageKHRWithDisplay& other) const { + return !(*this == other); + } +}; + +struct EGLImageKHRWithDisplayTraits { + static EGLImageKHRWithDisplay InvalidValue() { + return {EGL_NO_IMAGE_KHR, EGL_NO_DISPLAY}; + } + + static bool IsValid(const EGLImageKHRWithDisplay& value) { + return value != InvalidValue(); + } + + static void Free(EGLImageKHRWithDisplay image) { + eglDestroyImageKHR(image.display, image.image); + } +}; + +using UniqueEGLImageKHR = + fml::UniqueObject; + +} // namespace impeller diff --git a/impeller/toolkit/gles/BUILD.gn b/impeller/toolkit/gles/BUILD.gn new file mode 100644 index 0000000000000..06a16f95bf354 --- /dev/null +++ b/impeller/toolkit/gles/BUILD.gn @@ -0,0 +1,20 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("../../tools/impeller.gni") + +impeller_component("gles") { + sources = [ + "gles.h", + "texture.cc", + "texture.h", + ] + + deps = [ + "../../base", + "//flutter/fml", + ] + + libs = [] +} diff --git a/impeller/toolkit/gles/gles.h b/impeller/toolkit/gles/gles.h new file mode 100644 index 0000000000000..599e3c9433c97 --- /dev/null +++ b/impeller/toolkit/gles/gles.h @@ -0,0 +1,9 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "GLES3/gl3.h" +#define GL_GLEXT_PROTOTYPES +#include "GLES2/gl2ext.h" diff --git a/impeller/toolkit/gles/texture.cc b/impeller/toolkit/gles/texture.cc new file mode 100644 index 0000000000000..56f3d5b7b746f --- /dev/null +++ b/impeller/toolkit/gles/texture.cc @@ -0,0 +1,3 @@ +#include "flutter/impeller/toolkit/gles/texture.h" + +namespace impeller {} diff --git a/impeller/toolkit/gles/texture.h b/impeller/toolkit/gles/texture.h new file mode 100644 index 0000000000000..b90007b414237 --- /dev/null +++ b/impeller/toolkit/gles/texture.h @@ -0,0 +1,39 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "flutter/fml/unique_object.h" +#include "flutter/impeller/toolkit/gles/gles.h" + +namespace impeller { + +// Simple holder of an GLTexture and the owning EGLDisplay. +struct GLTexture { + GLuint texture_name; + + constexpr bool operator==(const GLTexture& other) const { + return texture_name == other.texture_name; + } + + constexpr bool operator!=(const GLTexture& other) const { + return !(*this == other); + } +}; + +struct GLTextureTraits { + static GLTexture InvalidValue() { return {0}; } + + static bool IsValid(const GLTexture& value) { + return value != InvalidValue(); + } + + static void Free(GLTexture image) { + glDeleteTextures(1, &image.texture_name); + } +}; + +using UniqueGLTexture = fml::UniqueObject; + +} // namespace impeller diff --git a/shell/platform/android/BUILD.gn b/shell/platform/android/BUILD.gn index 80fd97c3991f6..d2e201ea9f184 100644 --- a/shell/platform/android/BUILD.gn +++ b/shell/platform/android/BUILD.gn @@ -98,7 +98,11 @@ source_set("flutter_shell_native_src") { "apk_asset_provider.h", "flutter_main.cc", "flutter_main.h", + "hardware_buffer_external_texture_gl.cc", + "hardware_buffer_external_texture_gl.h", "library_loader.cc", + "ndk_helpers.cc", + "ndk_helpers.h", "platform_message_handler_android.cc", "platform_message_handler_android.h", "platform_message_response_android.cc", @@ -122,6 +126,7 @@ source_set("flutter_shell_native_src") { "//flutter/fml", "//flutter/impeller", "//flutter/impeller/toolkit/egl", + "//flutter/impeller/toolkit/gles", "//flutter/lib/ui", "//flutter/runtime", "//flutter/runtime:libdart", @@ -287,6 +292,7 @@ android_java_sources = [ "io/flutter/plugin/localization/LocalizationPlugin.java", "io/flutter/plugin/mouse/MouseCursorPlugin.java", "io/flutter/plugin/platform/AccessibilityEventsDelegate.java", + "io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java", "io/flutter/plugin/platform/PlatformOverlayView.java", "io/flutter/plugin/platform/PlatformPlugin.java", "io/flutter/plugin/platform/PlatformView.java", diff --git a/shell/platform/android/android_context_gl_skia.cc b/shell/platform/android/android_context_gl_skia.cc index adc29695d7347..8dcdeb5b62fab 100644 --- a/shell/platform/android/android_context_gl_skia.cc +++ b/shell/platform/android/android_context_gl_skia.cc @@ -205,6 +205,14 @@ bool AndroidContextGLSkia::ClearCurrent() const { return true; } +EGLContext AndroidContextGLSkia::GetEGLContext() const { + return context_; +} + +EGLDisplay AndroidContextGLSkia::GetEGLDisplay() const { + return environment_->Display(); +} + EGLContext AndroidContextGLSkia::CreateNewContext() const { bool success; EGLContext context; diff --git a/shell/platform/android/android_context_gl_skia.h b/shell/platform/android/android_context_gl_skia.h index 145be6144b218..1c9e092ff3a33 100644 --- a/shell/platform/android/android_context_gl_skia.h +++ b/shell/platform/android/android_context_gl_skia.h @@ -76,6 +76,20 @@ class AndroidContextGLSkia : public AndroidContext { /// bool ClearCurrent() const; + //---------------------------------------------------------------------------- + /// @brief Returns the EGLContext. + /// + /// @return EGLContext. + /// + EGLContext GetEGLContext() const; + + //---------------------------------------------------------------------------- + /// @brief Returns the EGLDisplay. + /// + /// @return EGLDisplay. + /// + EGLDisplay GetEGLDisplay() const; + //---------------------------------------------------------------------------- /// @brief Create a new EGLContext using the same EGLConfig. /// diff --git a/shell/platform/android/android_external_texture_gl.cc b/shell/platform/android/android_external_texture_gl.cc index 3e40b99c3d582..472c426747299 100644 --- a/shell/platform/android/android_external_texture_gl.cc +++ b/shell/platform/android/android_external_texture_gl.cc @@ -29,13 +29,13 @@ AndroidExternalTextureGL::AndroidExternalTextureGL( transform(SkMatrix::I()) {} AndroidExternalTextureGL::~AndroidExternalTextureGL() { - if (state_ == AttachmentState::attached) { + if (state_ == AttachmentState::kAttached) { glDeleteTextures(1, &texture_name_); } } void AndroidExternalTextureGL::OnGrContextCreated() { - state_ = AttachmentState::uninitialized; + state_ = AttachmentState::kUninitialized; } void AndroidExternalTextureGL::MarkNewFrameAvailable() { @@ -46,13 +46,13 @@ void AndroidExternalTextureGL::Paint(PaintContext& context, const SkRect& bounds, bool freeze, const DlImageSampling sampling) { - if (state_ == AttachmentState::detached) { + if (state_ == AttachmentState::kDetached) { return; } - if (state_ == AttachmentState::uninitialized) { + if (state_ == AttachmentState::kUninitialized) { glGenTextures(1, &texture_name_); Attach(static_cast(texture_name_)); - state_ = AttachmentState::attached; + state_ = AttachmentState::kAttached; } if (!freeze && new_frame_ready_) { Update(); @@ -108,11 +108,11 @@ void AndroidExternalTextureGL::UpdateTransform() { } void AndroidExternalTextureGL::OnGrContextDestroyed() { - if (state_ == AttachmentState::attached) { + if (state_ == AttachmentState::kAttached) { Detach(); glDeleteTextures(1, &texture_name_); } - state_ = AttachmentState::detached; + state_ = AttachmentState::kDetached; } void AndroidExternalTextureGL::Attach(jint textureName) { diff --git a/shell/platform/android/android_external_texture_gl.h b/shell/platform/android/android_external_texture_gl.h index d1c8f08be3d2b..ee2a88358ff5d 100644 --- a/shell/platform/android/android_external_texture_gl.h +++ b/shell/platform/android/android_external_texture_gl.h @@ -43,13 +43,13 @@ class AndroidExternalTextureGL : public flutter::Texture { void UpdateTransform(); - enum class AttachmentState { uninitialized, attached, detached }; + enum class AttachmentState { kUninitialized, kAttached, kDetached }; std::shared_ptr jni_facade_; fml::jni::ScopedJavaGlobalRef surface_texture_; - AttachmentState state_ = AttachmentState::uninitialized; + AttachmentState state_ = AttachmentState::kUninitialized; bool new_frame_ready_ = false; diff --git a/shell/platform/android/android_shell_holder_unittests.cc b/shell/platform/android/android_shell_holder_unittests.cc index fa138c920371b..831abc2cd6269 100644 --- a/shell/platform/android/android_shell_holder_unittests.cc +++ b/shell/platform/android/android_shell_holder_unittests.cc @@ -2,6 +2,7 @@ #include "flutter/shell/platform/android/android_shell_holder.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "shell/platform/android/jni/platform_view_android_jni.h" namespace flutter { namespace testing { @@ -30,6 +31,11 @@ class MockPlatformViewAndroidJNI : public PlatformViewAndroidJNI { void(JavaLocalRef surface_texture, SkMatrix& transform)); MOCK_METHOD1(SurfaceTextureDetachFromGLContext, void(JavaLocalRef surface_texture)); + MOCK_METHOD1(ImageTextureEntryAcquireLatestImage, + JavaLocalRef(JavaLocalRef image_texture_entry)); + MOCK_METHOD1(ImageGetHardwareBuffer, JavaLocalRef(JavaLocalRef image)); + MOCK_METHOD1(ImageClose, void(JavaLocalRef image)); + MOCK_METHOD1(HardwareBufferClose, void(JavaLocalRef hardware_buffer)); MOCK_METHOD8(FlutterViewOnDisplayPlatformView, void(int view_id, int x, diff --git a/shell/platform/android/hardware_buffer_external_texture_gl.cc b/shell/platform/android/hardware_buffer_external_texture_gl.cc new file mode 100644 index 0000000000000..1b513917c1643 --- /dev/null +++ b/shell/platform/android/hardware_buffer_external_texture_gl.cc @@ -0,0 +1,164 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/android/hardware_buffer_external_texture_gl.h" + +#include +#include +#include "impeller/toolkit/egl/image.h" +#include "impeller/toolkit/gles/texture.h" +#include "shell/platform/android/ndk_helpers.h" + +#include "flutter/display_list/effects/dl_color_source.h" +#include "third_party/skia/include/core/SkAlphaType.h" +#include "third_party/skia/include/core/SkColorSpace.h" +#include "third_party/skia/include/core/SkColorType.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/gpu/GrBackendSurface.h" +#include "third_party/skia/include/gpu/GrDirectContext.h" +#include "third_party/skia/include/gpu/ganesh/SkImageGanesh.h" + +namespace flutter { + +HardwareBufferExternalTextureGL::HardwareBufferExternalTextureGL( + const std::shared_ptr& context, + int64_t id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry, + const std::shared_ptr& jni_facade) + : Texture(id), + context_(context), + image_texture_entry_(image_texture_entry), + jni_facade_(jni_facade) {} + +HardwareBufferExternalTextureGL::~HardwareBufferExternalTextureGL() {} + +// Implementing flutter::Texture. +void HardwareBufferExternalTextureGL::Paint(PaintContext& context, + const SkRect& bounds, + bool freeze, + const DlImageSampling sampling) { + if (state_ == AttachmentState::kDetached) { + return; + } + if (state_ == AttachmentState::kUninitialized) { + GLuint texture_name; + glGenTextures(1, &texture_name); + texture_.reset(impeller::GLTexture{texture_name}); + state_ = AttachmentState::kAttached; + } + glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture_.get().texture_name); + if (!freeze && new_frame_ready_) { + new_frame_ready_ = false; + Update(); + } + GrGLTextureInfo textureInfo = {GL_TEXTURE_EXTERNAL_OES, + texture_.get().texture_name, GL_RGBA8_OES}; + GrBackendTexture backendTexture(1, 1, GrMipMapped::kNo, textureInfo); + sk_sp image = SkImages::BorrowTextureFrom( + context.gr_context, backendTexture, kTopLeft_GrSurfaceOrigin, + kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr); + if (image) { + DlAutoCanvasRestore autoRestore(context.canvas, true); + context.canvas->Translate(bounds.x(), bounds.y()); + context.canvas->Scale(bounds.width(), bounds.height()); + auto dl_image = DlImage::Make(image); + context.canvas->DrawImage(dl_image, {0, 0}, sampling, context.paint); + } else { + FML_LOG(ERROR) << "Skia could not borrow texture"; + } +} + +// Implementing flutter::Texture. +void HardwareBufferExternalTextureGL::MarkNewFrameAvailable() { + new_frame_ready_ = true; +} + +// Implementing flutter::Texture. +void HardwareBufferExternalTextureGL::OnTextureUnregistered() {} + +// Implementing flutter::ContextListener. +void HardwareBufferExternalTextureGL::OnGrContextCreated() { + state_ = AttachmentState::kUninitialized; +} + +AHardwareBuffer* HardwareBufferExternalTextureGL::GetLatestHardwareBuffer() { + JNIEnv* env = fml::jni::AttachCurrentThread(); + FML_CHECK(env != nullptr); + + // ImageTextureEntry.acquireLatestImage. + JavaLocalRef image_java = jni_facade_->ImageTextureEntryAcquireLatestImage( + JavaLocalRef(image_texture_entry_)); + if (image_java.obj() == nullptr) { + return nullptr; + } + + // Image.getHardwareBuffer. + JavaLocalRef hardware_buffer_java = + jni_facade_->ImageGetHardwareBuffer(image_java); + if (hardware_buffer_java.obj() == nullptr) { + jni_facade_->ImageClose(image_java); + return nullptr; + } + + // Convert into NDK HardwareBuffer. + AHardwareBuffer* latest_hardware_buffer = + NDKHelpers::AHardwareBuffer_fromHardwareBuffer( + env, hardware_buffer_java.obj()); + if (latest_hardware_buffer == nullptr) { + return nullptr; + } + + // Keep hardware buffer alive. + NDKHelpers::AHardwareBuffer_acquire(latest_hardware_buffer); + + // Now that we have referenced the native hardware buffer, close the Java + // Image and HardwareBuffer objects. + jni_facade_->HardwareBufferClose(hardware_buffer_java); + jni_facade_->ImageClose(image_java); + + return latest_hardware_buffer; +} + +void HardwareBufferExternalTextureGL::Update() { + EGLDisplay display = eglGetCurrentDisplay(); + FML_CHECK(display != EGL_NO_DISPLAY); + + image_.reset(); + + AHardwareBuffer* latest_hardware_buffer = GetLatestHardwareBuffer(); + if (latest_hardware_buffer == nullptr) { + FML_LOG(WARNING) << "GetLatestHardwareBuffer returned null."; + return; + } + + EGLClientBuffer client_buffer = + NDKHelpers::eglGetNativeClientBufferANDROID(latest_hardware_buffer); + if (client_buffer == nullptr) { + FML_LOG(WARNING) << "eglGetNativeClientBufferAndroid returned null."; + return; + } + FML_CHECK(client_buffer != nullptr); + image_.reset(impeller::EGLImageKHRWithDisplay{ + eglCreateImageKHR(display, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, + client_buffer, 0), + display}); + FML_CHECK(image_.get().image != EGL_NO_IMAGE_KHR); + glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, + (GLeglImageOES)image_.get().image); + + // Drop our temporary reference to the hardware buffer as the call to + // eglCreateImageKHR now has the reference. + NDKHelpers::AHardwareBuffer_release(latest_hardware_buffer); +} + +// Implementing flutter::ContextListener. +void HardwareBufferExternalTextureGL::OnGrContextDestroyed() { + if (state_ == AttachmentState::kAttached) { + image_.reset(); + texture_.reset(); + } + state_ = AttachmentState::kDetached; +} + +} // namespace flutter diff --git a/shell/platform/android/hardware_buffer_external_texture_gl.h b/shell/platform/android/hardware_buffer_external_texture_gl.h new file mode 100644 index 0000000000000..6890f5bcb3007 --- /dev/null +++ b/shell/platform/android/hardware_buffer_external_texture_gl.h @@ -0,0 +1,71 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ +#define FLUTTER_SHELL_PLATFORM_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ + +#include + +#include "flutter/impeller/toolkit/egl/egl.h" +#include "flutter/impeller/toolkit/egl/image.h" +#include "flutter/impeller/toolkit/gles/texture.h" + +#include +#include + +#include "flutter/common/graphics/texture.h" +#include "flutter/fml/logging.h" +#include "flutter/shell/platform/android/android_context_gl_skia.h" +#include "flutter/shell/platform/android/platform_view_android_jni_impl.h" + +namespace flutter { + +class HardwareBufferExternalTextureGL : public flutter::Texture { + public: + explicit HardwareBufferExternalTextureGL( + const std::shared_ptr& context, + int64_t id, + const fml::jni::ScopedJavaGlobalRef& + hardware_buffer_texture_entry, + const std::shared_ptr& jni_facade); + ~HardwareBufferExternalTextureGL() override; + + // |flutter::Texture|. + void Paint(PaintContext& context, + const SkRect& bounds, + bool freeze, + const DlImageSampling sampling) override; + + // |flutter::Texture|. + void MarkNewFrameAvailable() override; + + // |flutter::Texture| + void OnTextureUnregistered() override; + + // |flutter::ContextListener| + void OnGrContextCreated() override; + + // |flutter::ContextListener| + void OnGrContextDestroyed() override; + + private: + AHardwareBuffer* GetLatestHardwareBuffer(); + void Update(); + + const std::shared_ptr context_; + fml::jni::ScopedJavaGlobalRef image_texture_entry_; + std::shared_ptr jni_facade_; + + enum class AttachmentState { kUninitialized, kAttached, kDetached }; + AttachmentState state_ = AttachmentState::kUninitialized; + bool new_frame_ready_ = false; + impeller::UniqueEGLImageKHR image_; + impeller::UniqueGLTexture texture_; + + FML_DISALLOW_COPY_AND_ASSIGN(HardwareBufferExternalTextureGL); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_ANDROID_EXTERNAL_TEXTURE_GL_H_ diff --git a/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java b/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java index 4471d73f0f098..d579f1fceb108 100644 --- a/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java +++ b/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java @@ -33,6 +33,7 @@ import io.flutter.util.Preconditions; import io.flutter.view.AccessibilityBridge; import io.flutter.view.FlutterCallbackInformation; +import io.flutter.view.TextureRegistry; import java.io.IOException; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; @@ -901,6 +902,27 @@ private native void nativeRegisterTexture( long textureId, @NonNull WeakReference textureWrapper); + /** + * Registers a ImageTexture with the given id. + * + *

REQUIRED: Callers should eventually unregisterTexture with the same id. + */ + @UiThread + public void registerImageTexture( + long textureId, @NonNull TextureRegistry.ImageTextureEntry imageTextureEntry) { + ensureRunningOnMainThread(); + ensureAttachedToNative(); + nativeRegisterImageTexture( + nativeShellHolderId, + textureId, + new WeakReference(imageTextureEntry)); + } + + private native void nativeRegisterImageTexture( + long nativeShellHolderId, + long textureId, + @NonNull WeakReference imageTextureEntry); + /** * Call this method to inform Flutter that a texture previously registered with {@link * #registerTexture(long, SurfaceTextureWrapper)} has a new frame available. diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java index 1eac92546a659..423a355038c4a 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java @@ -4,9 +4,11 @@ package io.flutter.embedding.engine.renderer; +import android.annotation.TargetApi; import android.graphics.Bitmap; import android.graphics.Rect; import android.graphics.SurfaceTexture; +import android.media.Image; import android.os.Build; import android.os.Handler; import android.view.Surface; @@ -15,7 +17,9 @@ import androidx.annotation.VisibleForTesting; import io.flutter.Log; import io.flutter.embedding.engine.FlutterJNI; +import io.flutter.embedding.engine.renderer.FlutterRenderer.ImageTextureRegistryEntry; import io.flutter.view.TextureRegistry; +import io.flutter.view.TextureRegistry.ImageTextureEntry; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -158,6 +162,15 @@ public SurfaceTextureEntry registerSurfaceTexture(@NonNull SurfaceTexture surfac return entry; } + @Override + public ImageTextureEntry createImageTexture() { + final ImageTextureRegistryEntry entry = + new ImageTextureRegistryEntry(nextTextureId.getAndIncrement()); + Log.v(TAG, "New ImageTextureEntry ID: " + entry.id()); + registerImageTexture(entry.id(), entry); + return entry; + } + @Override public void onTrimMemory(int level) { final Iterator> iterator = onTrimMemoryListeners.iterator(); @@ -270,7 +283,7 @@ protected void finalize() throws Throwable { return; } - handler.post(new SurfaceTextureFinalizerRunnable(id, flutterJNI)); + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); } finally { super.finalize(); } @@ -287,11 +300,11 @@ public void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) { } } - static final class SurfaceTextureFinalizerRunnable implements Runnable { + static final class TextureFinalizerRunnable implements Runnable { private final long id; private final FlutterJNI flutterJNI; - SurfaceTextureFinalizerRunnable(long id, @NonNull FlutterJNI flutterJNI) { + TextureFinalizerRunnable(long id, @NonNull FlutterJNI flutterJNI) { this.id = id; this.flutterJNI = flutterJNI; } @@ -301,10 +314,73 @@ public void run() { if (!flutterJNI.isAttached()) { return; } - Log.v(TAG, "Releasing a SurfaceTexture (" + id + ")."); + Log.v(TAG, "Releasing a Texture (" + id + ")."); flutterJNI.unregisterTexture(id); } } + + final class ImageTextureRegistryEntry implements TextureRegistry.ImageTextureEntry { + private final long id; + private boolean released; + private Image image; + + ImageTextureRegistryEntry(long id) { + this.id = id; + } + + @Override + public long id() { + return id; + } + + @Override + public void release() { + if (released) { + return; + } + released = true; + unregisterTexture(id); + } + + @Override + @TargetApi(19) + public void pushImage(Image image) { + Image toClose; + synchronized (this) { + toClose = this.image; + this.image = image; + } + // Close the previously pushed buffer. + if (toClose != null) { + toClose.close(); + } + // Mark that we have a new frame available. + markTextureFrameAvailable(id); + } + + @Override + public Image acquireLatestImage() { + Image r; + synchronized (this) { + r = this.image; + this.image = null; + } + return r; + } + + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; + } + + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } + } + } // ------ END TextureRegistry IMPLEMENTATION ---- /** @@ -489,6 +565,11 @@ private void registerTexture(long textureId, @NonNull SurfaceTextureWrapper text flutterJNI.registerTexture(textureId, textureWrapper); } + private void registerImageTexture( + long textureId, @NonNull TextureRegistry.ImageTextureEntry textureEntry) { + flutterJNI.registerImageTexture(textureId, textureEntry); + } + // TODO(mattcarroll): describe the native behavior that this invokes private void markTextureFrameAvailable(long textureId) { flutterJNI.markTextureFrameAvailable(textureId); diff --git a/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java b/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java new file mode 100644 index 0000000000000..8a7c41b8b4c63 --- /dev/null +++ b/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java @@ -0,0 +1,140 @@ +package io.flutter.plugin.platform; + +import android.annotation.TargetApi; +import android.graphics.Canvas; +import android.graphics.ImageFormat; +import android.hardware.HardwareBuffer; +import android.media.Image; +import android.media.ImageReader; +import android.os.Build; +import android.os.Handler; +import android.view.Surface; +import io.flutter.view.TextureRegistry.ImageTextureEntry; + +@TargetApi(26) +public class ImageReaderPlatformViewRenderTarget implements PlatformViewRenderTarget { + private ImageTextureEntry textureEntry; + private boolean mustRecreateImageReader = false; + private ImageReader reader; + private int bufferWidth = 0; + private int bufferHeight = 0; + private static final String TAG = "ImageReaderPlatformViewRenderTarget"; + + private void closeReader() { + if (this.reader != null) { + this.reader.close(); + this.reader = null; + } + } + + private void recreateImageReaderIfNeeded() { + if (!mustRecreateImageReader) { + return; + } + mustRecreateImageReader = false; + closeReader(); + this.reader = createImageReader(); + } + + private final Handler onImageAvailableHandler = new Handler(); + private final ImageReader.OnImageAvailableListener onImageAvailableListener = + new ImageReader.OnImageAvailableListener() { + @Override + public void onImageAvailable(ImageReader reader) { + final Image image = reader.acquireLatestImage(); + if (image == null) { + return; + } + textureEntry.pushImage(image); + } + }; + + @TargetApi(33) + protected ImageReader createImageReader33() { + final ImageReader.Builder builder = new ImageReader.Builder(bufferWidth, bufferHeight); + // Allow for double buffering. + builder.setMaxImages(2); + // Assume that we will be producing 32-bit RGBA values. + builder.setDefaultHardwareBufferFormat(HardwareBuffer.RGBA_8888); + // Hint that consumed images will only be read by GPU. + builder.setUsage(HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + final ImageReader reader = builder.build(); + reader.setOnImageAvailableListener(this.onImageAvailableListener, onImageAvailableHandler); + return reader; + } + + @TargetApi(29) + protected ImageReader createImageReader29() { + return ImageReader.newInstance( + bufferWidth, bufferHeight, ImageFormat.UNKNOWN, 2, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + } + + @TargetApi(26) + protected ImageReader createImageReader26() { + return ImageReader.newInstance(bufferWidth, bufferHeight, ImageFormat.UNKNOWN, 2); + } + + protected ImageReader createImageReader() { + if (Build.VERSION.SDK_INT >= 33) { + return createImageReader33(); + } else if (Build.VERSION.SDK_INT >= 29) { + return createImageReader29(); + } else { + return createImageReader26(); + } + } + + public ImageReaderPlatformViewRenderTarget(ImageTextureEntry textureEntry) { + if (Build.VERSION.SDK_INT < 26) { + throw new UnsupportedOperationException( + "ImageReaderPlatformViewRenderTarget requires API version 26+"); + } + this.textureEntry = textureEntry; + } + + public void resize(int width, int height) { + if (this.reader != null && bufferWidth == width && bufferHeight == height) { + // No size change. + return; + } + closeReader(); + bufferWidth = width; + bufferHeight = height; + this.reader = createImageReader(); + } + + public int getWidth() { + return this.bufferWidth; + } + + public int getHeight() { + return this.bufferHeight; + } + + public Canvas lockHardwareCanvas() { + return getSurface().lockHardwareCanvas(); + } + + public void unlockCanvasAndPost(Canvas canvas) { + getSurface().unlockCanvasAndPost(canvas); + } + + public long getId() { + return this.textureEntry.id(); + } + + public void release() { + // textureEntry has a finalizer attached. + textureEntry = null; + closeReader(); + } + + public boolean isReleased() { + return this.textureEntry == null; + } + + public Surface getSurface() { + recreateImageReaderIfNeeded(); + return this.reader.getSurface(); + } +} diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java index c5d2282b9dbb2..aca60f4ca02e2 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java @@ -24,7 +24,6 @@ import io.flutter.Log; import io.flutter.embedding.android.AndroidTouchProcessor; import io.flutter.util.ViewUtils; -import io.flutter.view.TextureRegistry; /** * Wraps a platform view to intercept gestures and project this view onto a {@link @@ -55,12 +54,6 @@ public PlatformViewWrapper(@NonNull Context context) { setWillNotDraw(false); } - public PlatformViewWrapper( - @NonNull Context context, @NonNull TextureRegistry.SurfaceTextureEntry textureEntry) { - this(context); - this.renderTarget = new SurfaceTexturePlatformViewRenderTarget(textureEntry); - } - public PlatformViewWrapper( @NonNull Context context, @NonNull PlatformViewRenderTarget renderTarget) { this(context); diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java index b348c1d0b53c9..b0600b65bd22a 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java @@ -550,7 +550,7 @@ private long configureForVirtualDisplay( Log.i(TAG, "Hosting view in a virtual display for platform view: " + request.viewId); - final TextureRegistry.SurfaceTextureEntry textureEntry = textureRegistry.createSurfaceTexture(); + final PlatformViewRenderTarget renderTarget = makePlatformViewRenderTarget(textureRegistry); final int physicalWidth = toPhysicalPixels(request.logicalWidth); final int physicalHeight = toPhysicalPixels(request.logicalHeight); final VirtualDisplayController vdController = @@ -558,7 +558,7 @@ private long configureForVirtualDisplay( context, accessibilityEventsDelegate, platformView, - textureEntry, + renderTarget, physicalWidth, physicalHeight, request.viewId, @@ -584,7 +584,7 @@ private long configureForVirtualDisplay( final View embeddedView = platformView.getView(); contextToEmbeddedView.put(embeddedView.getContext(), embeddedView); - return textureEntry.id(); + return renderTarget.getId(); } // Configures the view for Texture Layer Hybrid Composition mode, returning the associated @@ -610,10 +610,9 @@ public long configureForTextureLayerComposition( viewWrapper = new PlatformViewWrapper(context); textureId = -1; } else { - final TextureRegistry.SurfaceTextureEntry textureEntry = - textureRegistry.createSurfaceTexture(); - viewWrapper = new PlatformViewWrapper(context, textureEntry); - textureId = textureEntry.id(); + final PlatformViewRenderTarget renderTarget = makePlatformViewRenderTarget(textureRegistry); + viewWrapper = new PlatformViewWrapper(context, renderTarget); + textureId = renderTarget.getId(); } viewWrapper.setTouchProcessor(androidTouchProcessor); viewWrapper.resizeRenderTarget(physicalWidth, physicalHeight); @@ -968,6 +967,17 @@ private void unlockInputConnection(@NonNull VirtualDisplayController controller) controller.onInputConnectionUnlocked(); } + private static PlatformViewRenderTarget makePlatformViewRenderTarget( + TextureRegistry textureRegistry) { + // TODO(johnmccutchan): Enable ImageReaderPlatformViewRenderTarget for public use. + if (false) { + final TextureRegistry.ImageTextureEntry textureEntry = textureRegistry.createImageTexture(); + return new ImageReaderPlatformViewRenderTarget(textureEntry); + } + final TextureRegistry.SurfaceTextureEntry textureEntry = textureRegistry.createSurfaceTexture(); + return new SurfaceTexturePlatformViewRenderTarget(textureEntry); + } + private static boolean validateDirection(int direction) { return direction == View.LAYOUT_DIRECTION_LTR || direction == View.LAYOUT_DIRECTION_RTL; } diff --git a/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java b/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java index 4f2fb12beac22..deff5834aadfe 100644 --- a/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java +++ b/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java @@ -164,7 +164,7 @@ public boolean isReleased() { } public void release() { - // Don't release the texture. + // Don't release the texture, let the GC finalize it. surfaceTexture = null; if (surface != null) { surface.release(); diff --git a/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java b/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java index 20886358086be..1b90ca5c3a368 100644 --- a/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java +++ b/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java @@ -17,7 +17,6 @@ import android.view.ViewTreeObserver; import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; -import io.flutter.view.TextureRegistry; @TargetApi(20) class VirtualDisplayController { @@ -27,7 +26,7 @@ public static VirtualDisplayController create( Context context, AccessibilityEventsDelegate accessibilityEventsDelegate, PlatformView view, - TextureRegistry.SurfaceTextureEntry textureEntry, + PlatformViewRenderTarget renderTarget, int width, int height, int viewId, @@ -40,9 +39,6 @@ public static VirtualDisplayController create( DisplayManager displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); final DisplayMetrics metrics = context.getResources().getDisplayMetrics(); - final PlatformViewRenderTarget renderTarget = - new SurfaceTexturePlatformViewRenderTarget(textureEntry); - // Virtual Display crashes for some PlatformViews if the width or height is bigger // than the physical screen size. We have tried to clamp or scale down the size to prevent // the crash, but both solutions lead to unwanted behavior because the @@ -73,7 +69,6 @@ public static VirtualDisplayController create( virtualDisplay, view, renderTarget, - textureEntry, focusChangeListener, viewId, createParams); @@ -86,7 +81,6 @@ public static VirtualDisplayController create( private final AccessibilityEventsDelegate accessibilityEventsDelegate; private final int densityDpi; private final int viewId; - private final TextureRegistry.SurfaceTextureEntry textureEntry; private final PlatformViewRenderTarget renderTarget; private final OnFocusChangeListener focusChangeListener; @@ -98,14 +92,12 @@ private VirtualDisplayController( VirtualDisplay virtualDisplay, PlatformView view, PlatformViewRenderTarget renderTarget, - TextureRegistry.SurfaceTextureEntry textureEntry, OnFocusChangeListener focusChangeListener, int viewId, Object createParams) { this.context = context; this.accessibilityEventsDelegate = accessibilityEventsDelegate; this.renderTarget = renderTarget; - this.textureEntry = textureEntry; this.focusChangeListener = focusChangeListener; this.viewId = viewId; this.virtualDisplay = virtualDisplay; @@ -204,7 +196,7 @@ public void dispose() { presentation.cancel(); presentation.detachState(); virtualDisplay.release(); - textureEntry.release(); + renderTarget.release(); } /** See {@link PlatformView#onFlutterViewAttached(View)} */ diff --git a/shell/platform/android/io/flutter/view/FlutterView.java b/shell/platform/android/io/flutter/view/FlutterView.java index eb498a2335e96..7d15aab7ad89d 100644 --- a/shell/platform/android/io/flutter/view/FlutterView.java +++ b/shell/platform/android/io/flutter/view/FlutterView.java @@ -898,6 +898,12 @@ public TextureRegistry.SurfaceTextureEntry createSurfaceTexture() { return registerSurfaceTexture(surfaceTexture); } + @Override + @NonNull + public ImageTextureEntry createImageTexture() { + throw new UnsupportedOperationException("Image textures are not supported in this mode."); + } + @Override @NonNull public TextureRegistry.SurfaceTextureEntry registerSurfaceTexture( diff --git a/shell/platform/android/io/flutter/view/TextureRegistry.java b/shell/platform/android/io/flutter/view/TextureRegistry.java index 18dcbe100774e..98ec56476668d 100644 --- a/shell/platform/android/io/flutter/view/TextureRegistry.java +++ b/shell/platform/android/io/flutter/view/TextureRegistry.java @@ -5,6 +5,7 @@ package io.flutter.view; import android.graphics.SurfaceTexture; +import android.media.Image; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -31,6 +32,14 @@ public interface TextureRegistry { @NonNull SurfaceTextureEntry registerSurfaceTexture(@NonNull SurfaceTexture surfaceTexture); + /** + * Creates and registers a texture managed by the Flutter engine. + * + * @return a ImageTextureEntry. + */ + @NonNull + ImageTextureEntry createImageTexture(); + /** * Callback invoked when memory is low. * @@ -38,18 +47,27 @@ public interface TextureRegistry { */ default void onTrimMemory(int level) {} - /** A registry entry for a managed SurfaceTexture. */ - interface SurfaceTextureEntry { - /** @return The managed SurfaceTexture. */ - @NonNull - SurfaceTexture surfaceTexture(); + /** An entry in the texture registry. */ + interface TextureEntry { + /** @return The identity of this texture. */ + long id(); - /** @return The identity of this SurfaceTexture. */ + /** Deregisters and releases all resources . */ + void release(); + } + + /** A registry entry for a managed SurfaceTexture. */ + interface SurfaceTextureEntry extends TextureEntry { + /** @return The identity of this texture. */ long id(); - /** Deregisters and releases this SurfaceTexture. */ + /** Deregisters and releases all resources . */ void release(); + /** @return The managed SurfaceTexture. */ + @NonNull + SurfaceTexture surfaceTexture(); + /** Set a listener that will be notified when the most recent image has been consumed. */ default void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) {} @@ -57,6 +75,33 @@ default void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listen default void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) {} } + interface ImageTextureEntry extends TextureEntry { + /** @return the identity of this ImageTextureEntry */ + long id(); + + /** Deregisters and releases all resources. */ + void release(); + + /** + * Next paint will update texture to use the contents of image. + * + *

NOTE: Caller should not call Image.close() on the pushed image. + * + *

NOTE: In the case that multiple calls to PushFrame occur before the next paint only the + * last frame pushed will be used (dropping the missed frames). + */ + void pushImage(Image image); + + /** + * Retrieve the last Image pushed. + * + *

NOTE: Caller must call Image.close() on returned image. + * + * @return Image or null. + */ + Image acquireLatestImage(); + } + /** Listener invoked when the most recent image has been consumed. */ interface OnFrameConsumedListener { /** diff --git a/shell/platform/android/jni/jni_mock.h b/shell/platform/android/jni/jni_mock.h index b2966e6e67156..8b84793844857 100644 --- a/shell/platform/android/jni/jni_mock.h +++ b/shell/platform/android/jni/jni_mock.h @@ -59,6 +59,23 @@ class JNIMock final : public PlatformViewAndroidJNI { (JavaLocalRef surface_texture, SkMatrix& transform), (override)); + MOCK_METHOD(JavaLocalRef, + ImageTextureEntryAcquireLatestImage, + (JavaLocalRef image_texture_entry), + (override)); + + MOCK_METHOD(JavaLocalRef, + ImageGetHardwareBuffer, + (JavaLocalRef image), + (override)); + + MOCK_METHOD(void, ImageClose, (JavaLocalRef image), (override)); + + MOCK_METHOD(void, + HardwareBufferClose, + (JavaLocalRef hardware_buffer), + (override)); + MOCK_METHOD(void, SurfaceTextureDetachFromGLContext, (JavaLocalRef surface_texture), diff --git a/shell/platform/android/jni/platform_view_android_jni.h b/shell/platform/android/jni/platform_view_android_jni.h index 35e949f0bc9dd..c876b093dc08e 100644 --- a/shell/platform/android/jni/platform_view_android_jni.h +++ b/shell/platform/android/jni/platform_view_android_jni.h @@ -109,6 +109,27 @@ class PlatformViewAndroidJNI { virtual void SurfaceTextureDetachFromGLContext( JavaLocalRef surface_texture) = 0; + //---------------------------------------------------------------------------- + /// @brief Acquire the latest image available. + /// + virtual JavaLocalRef ImageTextureEntryAcquireLatestImage( + JavaLocalRef image_texture_entry) = 0; + + //---------------------------------------------------------------------------- + /// @brief Grab the HardwareBuffer from image. + /// + virtual JavaLocalRef ImageGetHardwareBuffer(JavaLocalRef image) = 0; + + //---------------------------------------------------------------------------- + /// @brief Call close on image. + /// + virtual void ImageClose(JavaLocalRef image) = 0; + + //---------------------------------------------------------------------------- + /// @brief Call close on hardware_buffer. + /// + virtual void HardwareBufferClose(JavaLocalRef hardware_buffer) = 0; + //---------------------------------------------------------------------------- /// @brief Positions and sizes a platform view if using hybrid /// composition. diff --git a/shell/platform/android/library_loader.cc b/shell/platform/android/library_loader.cc index 644bd5de3b8ec..646f232e8bc4b 100644 --- a/shell/platform/android/library_loader.cc +++ b/shell/platform/android/library_loader.cc @@ -5,6 +5,7 @@ #include "flutter/fml/platform/android/jni_util.h" #include "flutter/shell/platform/android/android_image_generator.h" #include "flutter/shell/platform/android/flutter_main.h" +#include "flutter/shell/platform/android/ndk_helpers.h" #include "flutter/shell/platform/android/platform_view_android.h" #include "flutter/shell/platform/android/vsync_waiter_android.h" diff --git a/shell/platform/android/ndk_helpers.cc b/shell/platform/android/ndk_helpers.cc new file mode 100644 index 0000000000000..568be5469ab74 --- /dev/null +++ b/shell/platform/android/ndk_helpers.cc @@ -0,0 +1,101 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/android/ndk_helpers.h" + +#include "fml/native_library.h" +#include "shell/platform/android/ndk_helpers.h" + +#include "flutter/fml/logging.h" + +#include + +namespace flutter { + +namespace { + +typedef AHardwareBuffer* (*fp_AHardwareBuffer_fromHardwareBuffer)( + JNIEnv* env, + jobject hardwareBufferObj); +typedef void (*fp_AHardwareBuffer_acquire)(AHardwareBuffer* buffer); +typedef void (*fp_AHardwareBuffer_release)(AHardwareBuffer* buffer); +typedef EGLClientBuffer (*fp_eglGetNativeClientBufferANDROID)( + AHardwareBuffer* buffer); + +AHardwareBuffer* (*_AHardwareBuffer_fromHardwareBuffer)( + JNIEnv* env, + jobject hardwareBufferObj) = nullptr; +void (*_AHardwareBuffer_acquire)(AHardwareBuffer* buffer) = nullptr; +void (*_AHardwareBuffer_release)(AHardwareBuffer* buffer) = nullptr; +EGLClientBuffer (*_eglGetNativeClientBufferANDROID)(AHardwareBuffer* buffer) = + nullptr; + +std::once_flag init_once; + +void InitOnceCallback() { + static fml::RefPtr android = + fml::NativeLibrary::Create("libandroid.so"); + FML_CHECK(android.get() != nullptr); + static fml::RefPtr egl = + fml::NativeLibrary::Create("libEGL.so"); + FML_CHECK(egl.get() != nullptr); + _eglGetNativeClientBufferANDROID = + egl->ResolveFunction( + "eglGetNativeClientBufferANDROID") + .value_or(nullptr); + _AHardwareBuffer_fromHardwareBuffer = + android + ->ResolveFunction( + "AHardwareBuffer_fromHardwareBuffer") + .value_or(nullptr); + _AHardwareBuffer_acquire = android + ->ResolveFunction( + "AHardwareBuffer_acquire") + .value_or(nullptr); + _AHardwareBuffer_release = android + ->ResolveFunction( + "AHardwareBuffer_release") + .value_or(nullptr); +} + +} // namespace + +void NDKHelpers::Init() { + std::call_once(init_once, InitOnceCallback); +} + +bool NDKHelpers::HardwareBufferSupported() { + NDKHelpers::Init(); + const bool r = _AHardwareBuffer_fromHardwareBuffer != nullptr; + return r; +} + +AHardwareBuffer* NDKHelpers::AHardwareBuffer_fromHardwareBuffer( + JNIEnv* env, + jobject hardwareBufferObj) { + NDKHelpers::Init(); + FML_CHECK(_AHardwareBuffer_fromHardwareBuffer != nullptr); + return _AHardwareBuffer_fromHardwareBuffer(env, hardwareBufferObj); +} + +void NDKHelpers::AHardwareBuffer_acquire(AHardwareBuffer* buffer) { + NDKHelpers::Init(); + FML_CHECK(_AHardwareBuffer_acquire != nullptr); + _AHardwareBuffer_acquire(buffer); +} + +void NDKHelpers::AHardwareBuffer_release(AHardwareBuffer* buffer) { + NDKHelpers::Init(); + FML_CHECK(_AHardwareBuffer_release != nullptr); + _AHardwareBuffer_release(buffer); +} + +EGLClientBuffer NDKHelpers::eglGetNativeClientBufferANDROID( + AHardwareBuffer* buffer) { + NDKHelpers::Init(); + FML_CHECK(_eglGetNativeClientBufferANDROID != nullptr); + return _eglGetNativeClientBufferANDROID(buffer); +} + +} // namespace flutter diff --git a/shell/platform/android/ndk_helpers.h b/shell/platform/android/ndk_helpers.h new file mode 100644 index 0000000000000..3ec0500d99e9a --- /dev/null +++ b/shell/platform/android/ndk_helpers.h @@ -0,0 +1,37 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_ANDROID_NDK_HELPERS_H_ +#define FLUTTER_SHELL_PLATFORM_ANDROID_NDK_HELPERS_H_ + +#include "flutter/fml/native_library.h" +#include "flutter/fml/platform/android/jni_util.h" + +#include "flutter/impeller/toolkit/egl/egl.h" + +#include + +namespace flutter { + +// A collection of NDK functions that are available depending on the version of +// the Android SDK we are linked with at runtime. +class NDKHelpers { + public: + // API Version 26 + static bool HardwareBufferSupported(); + static AHardwareBuffer* AHardwareBuffer_fromHardwareBuffer( + JNIEnv* env, + jobject hardwareBufferObj); + static void AHardwareBuffer_acquire(AHardwareBuffer* buffer); + static void AHardwareBuffer_release(AHardwareBuffer* buffer); + static EGLClientBuffer eglGetNativeClientBufferANDROID( + AHardwareBuffer* buffer); + + private: + static void Init(); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_ANDROID_NDK_HELPERS_H_ diff --git a/shell/platform/android/platform_view_android.cc b/shell/platform/android/platform_view_android.cc index 2993c3d8804a3..ff941ef81dc3c 100644 --- a/shell/platform/android/platform_view_android.cc +++ b/shell/platform/android/platform_view_android.cc @@ -7,6 +7,7 @@ #include #include +#include "common/graphics/texture.h" #include "flutter/fml/synchronization/waitable_event.h" #include "flutter/shell/common/shell_io_manager.h" #include "flutter/shell/gpu/gpu_surface_gl_delegate.h" @@ -16,6 +17,7 @@ #include "flutter/shell/platform/android/android_surface_gl_impeller.h" #include "flutter/shell/platform/android/android_surface_gl_skia.h" #include "flutter/shell/platform/android/android_surface_software.h" +#include "flutter/shell/platform/android/hardware_buffer_external_texture_gl.h" #if IMPELLER_ENABLE_VULKAN // b/258506856 for why this is behind an if #include "flutter/shell/platform/android/android_surface_vulkan_impeller.h" #endif @@ -304,6 +306,18 @@ void PlatformViewAndroid::RegisterExternalTexture( } } +void PlatformViewAndroid::RegisterImageTexture( + int64_t texture_id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry) { + if (android_context_->RenderingApi() == AndroidRenderingAPI::kOpenGLES) { + RegisterTexture(std::make_shared( + std::static_pointer_cast(android_context_), + texture_id, image_texture_entry, jni_facade_)); + } else { + FML_LOG(INFO) << "Attempted to use a GL texture in a non GL context."; + } +} + // |PlatformView| std::unique_ptr PlatformViewAndroid::CreateVSyncWaiter() { return std::make_unique(task_runners_); diff --git a/shell/platform/android/platform_view_android.h b/shell/platform/android/platform_view_android.h index 069b2f920b3af..014e2c8c6fa2d 100644 --- a/shell/platform/android/platform_view_android.h +++ b/shell/platform/android/platform_view_android.h @@ -10,6 +10,7 @@ #include #include +#include #include "flutter/fml/memory/weak_ptr.h" #include "flutter/fml/platform/android/scoped_java_ref.h" #include "flutter/lib/ui/window/platform_message.h" @@ -91,6 +92,10 @@ class PlatformViewAndroid final : public PlatformView { int64_t texture_id, const fml::jni::ScopedJavaGlobalRef& surface_texture); + void RegisterImageTexture( + int64_t texture_id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry); + // |PlatformView| void LoadDartDeferredLibrary( intptr_t loading_unit_id, diff --git a/shell/platform/android/platform_view_android_jni_impl.cc b/shell/platform/android/platform_view_android_jni_impl.cc index 72abd446061b0..9a37889861afc 100644 --- a/shell/platform/android/platform_view_android_jni_impl.cc +++ b/shell/platform/android/platform_view_android_jni_impl.cc @@ -4,6 +4,7 @@ #include "flutter/shell/platform/android/platform_view_android_jni_impl.h" +#include #include #include #include @@ -11,6 +12,8 @@ #include #include +#include "include/android/SkImageAndroid.h" +#include "shell/platform/android/ndk_helpers.h" #include "unicode/uchar.h" #include "flutter/assets/directory_asset_bundle.h" @@ -29,6 +32,7 @@ #include "flutter/shell/platform/android/android_shell_holder.h" #include "flutter/shell/platform/android/apk_asset_provider.h" #include "flutter/shell/platform/android/flutter_main.h" +#include "flutter/shell/platform/android/hardware_buffer_external_texture_gl.h" #include "flutter/shell/platform/android/jni/platform_view_android_jni.h" #include "flutter/shell/platform/android/platform_view_android.h" @@ -49,6 +53,13 @@ static fml::jni::ScopedJavaGlobalRef* g_java_weak_reference_class = static fml::jni::ScopedJavaGlobalRef* g_texture_wrapper_class = nullptr; +static fml::jni::ScopedJavaGlobalRef* g_image_texture_entry_class = + nullptr; + +static fml::jni::ScopedJavaGlobalRef* g_image_class = nullptr; + +static fml::jni::ScopedJavaGlobalRef* g_hardware_buffer_class = nullptr; + static fml::jni::ScopedJavaGlobalRef* g_java_long_class = nullptr; static fml::jni::ScopedJavaGlobalRef* g_bitmap_class = nullptr; @@ -106,6 +117,14 @@ static jmethodID g_get_transform_matrix_method = nullptr; static jmethodID g_detach_from_gl_context_method = nullptr; +static jmethodID g_acquire_latest_image_method = nullptr; + +static jmethodID g_image_get_hardware_buffer_method = nullptr; + +static jmethodID g_image_close_method = nullptr; + +static jmethodID g_hardware_buffer_close_method = nullptr; + static jmethodID g_compute_platform_resolved_locale_method = nullptr; static jmethodID g_request_dart_deferred_library_method = nullptr; @@ -475,12 +494,15 @@ static void RegisterTexture(JNIEnv* env, ); } -static void MarkTextureFrameAvailable(JNIEnv* env, - jobject jcaller, - jlong shell_holder, - jlong texture_id) { - ANDROID_SHELL_HOLDER->GetPlatformView()->MarkTextureFrameAvailable( - static_cast(texture_id)); +static void RegisterImageTexture(JNIEnv* env, + jobject jcaller, + jlong shell_holder, + jlong texture_id, + jobject image_texture_entry) { + ANDROID_SHELL_HOLDER->GetPlatformView()->RegisterImageTexture( + static_cast(texture_id), // + fml::jni::ScopedJavaGlobalRef(env, image_texture_entry) // + ); } static void UnregisterTexture(JNIEnv* env, @@ -491,6 +513,14 @@ static void UnregisterTexture(JNIEnv* env, static_cast(texture_id)); } +static void MarkTextureFrameAvailable(JNIEnv* env, + jobject jcaller, + jlong shell_holder, + jlong texture_id) { + ANDROID_SHELL_HOLDER->GetPlatformView()->MarkTextureFrameAvailable( + static_cast(texture_id)); +} + static void InvokePlatformMessageResponseCallback(JNIEnv* env, jobject jcaller, jlong shell_holder, @@ -742,6 +772,12 @@ bool RegisterApi(JNIEnv* env) { "WeakReference;)V", .fnPtr = reinterpret_cast(&RegisterTexture), }, + { + .name = "nativeRegisterImageTexture", + .signature = "(JJLjava/lang/ref/" + "WeakReference;)V", + .fnPtr = reinterpret_cast(&RegisterImageTexture), + }, { .name = "nativeMarkTextureFrameAvailable", .signature = "(JJ)V", @@ -1126,6 +1162,70 @@ bool PlatformViewAndroid::Register(JNIEnv* env) { FML_LOG(ERROR) << "Could not locate detachFromGlContext method"; return false; } + g_image_texture_entry_class = new fml::jni::ScopedJavaGlobalRef( + env, env->FindClass("io/flutter/view/TextureRegistry$ImageTextureEntry")); + if (g_image_texture_entry_class->is_null()) { + FML_LOG(ERROR) << "Could not locate ImageTextureEntry class"; + return false; + } + + g_acquire_latest_image_method = + env->GetMethodID(g_image_texture_entry_class->obj(), "acquireLatestImage", + "()Landroid/media/Image;"); + if (g_acquire_latest_image_method == nullptr) { + FML_LOG(ERROR) << "Could not locate acquireLatestImage on " + "ImageTextureEntry class"; + return false; + } + + g_image_class = new fml::jni::ScopedJavaGlobalRef( + env, env->FindClass("android/media/Image")); + if (g_image_texture_entry_class->is_null()) { + FML_LOG(ERROR) << "Could not locate Image class"; + return false; + } + + // Ensure we don't have any pending exceptions. + FML_CHECK(fml::jni::CheckException(env)); + + g_image_get_hardware_buffer_method = + env->GetMethodID(g_image_class->obj(), "getHardwareBuffer", + "()Landroid/hardware/HardwareBuffer;"); + + if (g_image_get_hardware_buffer_method == nullptr) { + FML_LOG(WARNING) << "Could not locate getHardwareBuffer on " + "android.media.Image"; + // Continue on as this method may not exist at API <= 29. + fml::jni::ClearException(env); + } + + g_image_close_method = env->GetMethodID(g_image_class->obj(), "close", "()V"); + + if (g_image_close_method == nullptr) { + FML_LOG(ERROR) << "Could not locate close on Image class"; + return false; + } + + // Ensure we don't have any pending exceptions. + FML_CHECK(fml::jni::CheckException(env)); + g_hardware_buffer_class = new fml::jni::ScopedJavaGlobalRef( + env, env->FindClass("android/hardware/HardwareBuffer")); + + if (!g_hardware_buffer_class->is_null()) { + g_hardware_buffer_close_method = + env->GetMethodID(g_hardware_buffer_class->obj(), "close", "()V"); + if (g_hardware_buffer_close_method == nullptr) { + FML_LOG(WARNING) + << "Could not locate close on android.hardware.HardwareBuffer"; + // Continue on as this class may not exist at API <= 26. + fml::jni::ClearException(env); + } + } else { + FML_LOG(WARNING) + << "Could not locate android.hardware.HardwareBuffer class"; + // Continue on as this class may not exist at API <= 26. + fml::jni::ClearException(env); + } g_compute_platform_resolved_locale_method = env->GetMethodID( g_flutter_jni_class->obj(), "computePlatformResolvedLocale", @@ -1420,6 +1520,68 @@ void PlatformViewAndroidJNIImpl::SurfaceTextureDetachFromGLContext( FML_CHECK(fml::jni::CheckException(env)); } +JavaLocalRef PlatformViewAndroidJNIImpl::ImageTextureEntryAcquireLatestImage( + JavaLocalRef image_texture_entry) { + JNIEnv* env = fml::jni::AttachCurrentThread(); + + if (image_texture_entry.is_null()) { + // Return null. + return JavaLocalRef(); + } + + // Convert the weak reference to ImageTextureEntry into a strong local + // reference. + fml::jni::ScopedJavaLocalRef image_texture_entry_local_ref( + env, env->CallObjectMethod(image_texture_entry.obj(), + g_java_weak_reference_get_method)); + + if (image_texture_entry_local_ref.is_null()) { + // Return null. + return JavaLocalRef(); + } + + JavaLocalRef r = JavaLocalRef( + env, env->CallObjectMethod(image_texture_entry_local_ref.obj(), + g_acquire_latest_image_method)); + FML_CHECK(fml::jni::CheckException(env)); + return r; +} + +JavaLocalRef PlatformViewAndroidJNIImpl::ImageGetHardwareBuffer( + JavaLocalRef image) { + FML_CHECK(g_image_get_hardware_buffer_method != nullptr); + JNIEnv* env = fml::jni::AttachCurrentThread(); + if (image.is_null()) { + // Return null. + return JavaLocalRef(); + } + JavaLocalRef r = JavaLocalRef( + env, + env->CallObjectMethod(image.obj(), g_image_get_hardware_buffer_method)); + FML_CHECK(fml::jni::CheckException(env)); + return r; +} + +void PlatformViewAndroidJNIImpl::ImageClose(JavaLocalRef image) { + JNIEnv* env = fml::jni::AttachCurrentThread(); + if (image.is_null()) { + return; + } + env->CallVoidMethod(image.obj(), g_image_close_method); + FML_CHECK(fml::jni::CheckException(env)); +} + +void PlatformViewAndroidJNIImpl::HardwareBufferClose( + JavaLocalRef hardware_buffer) { + FML_CHECK(g_hardware_buffer_close_method != nullptr); + JNIEnv* env = fml::jni::AttachCurrentThread(); + if (hardware_buffer.is_null()) { + return; + } + env->CallVoidMethod(hardware_buffer.obj(), g_hardware_buffer_close_method); + FML_CHECK(fml::jni::CheckException(env)); +} + void PlatformViewAndroidJNIImpl::FlutterViewOnDisplayPlatformView( int view_id, int x, diff --git a/shell/platform/android/platform_view_android_jni_impl.h b/shell/platform/android/platform_view_android_jni_impl.h index 6b1e283e812d0..33de0ae255bd0 100644 --- a/shell/platform/android/platform_view_android_jni_impl.h +++ b/shell/platform/android/platform_view_android_jni_impl.h @@ -52,6 +52,15 @@ class PlatformViewAndroidJNIImpl final : public PlatformViewAndroidJNI { void SurfaceTextureDetachFromGLContext(JavaLocalRef surface_texture) override; + JavaLocalRef ImageTextureEntryAcquireLatestImage( + JavaLocalRef image_texture_entry) override; + + JavaLocalRef ImageGetHardwareBuffer(JavaLocalRef image) override; + + void ImageClose(JavaLocalRef image) override; + + void HardwareBufferClose(JavaLocalRef hardware_buffer) override; + void FlutterViewOnDisplayPlatformView(int view_id, int x, int y, diff --git a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java index 0d2b573fe6522..9dfd58ddeaa4f 100644 --- a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java +++ b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java @@ -14,6 +14,7 @@ import android.content.MutableContextWrapper; import android.content.res.AssetManager; import android.graphics.SurfaceTexture; +import android.media.Image; import android.util.SparseArray; import android.view.MotionEvent; import android.view.Surface; @@ -1566,6 +1567,27 @@ public long id() { public void release() {} }; } + + @Override + public ImageTextureEntry createImageTexture() { + return new ImageTextureEntry() { + @Override + public long id() { + return 0; + } + + @Override + public void release() {} + + @Override + public void pushImage(Image image) {} + + @Override + public Image acquireLatestImage() { + return null; + } + }; + } }; platformViewsController.attach(context, registry, executor); From 9522ce9b4efdeded6472d094ac41916b76f5ee39 Mon Sep 17 00:00:00 2001 From: skia-flutter-autoroll Date: Thu, 3 Aug 2023 13:12:59 -0400 Subject: [PATCH 4/9] Roll Skia from 768050436c0a to bd3ee535e246 (2 revisions) (#44327) https://skia.googlesource.com/skia.git/+log/768050436c0a..bd3ee535e246 2023-08-03 johnstiles@google.com Eliminate DSLExpression. 2023-08-03 brianosman@google.com Disable implicit function declaration warning in microhttpd If this roll has caused a breakage, revert this CL and stop the roller using the controls here: https://autoroll.skia.org/r/skia-flutter-autoroll Please CC brianosman@google.com,jacksongardner@google.com,jvanverth@google.com,rmistry@google.com on the revert to ensure that a human is aware of the problem. To file a bug in Skia: https://bugs.chromium.org/p/skia/issues/entry To file a bug in Flutter: https://github.com/flutter/flutter/issues/new/choose To report a problem with the AutoRoller itself, please file a bug: https://bugs.chromium.org/p/skia/issues/entry?template=Autoroller+Bug Documentation for the AutoRoller is here: https://skia.googlesource.com/buildbot/+doc/main/autoroll/README.md --- DEPS | 2 +- ci/licenses_golden/licenses_skia | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/DEPS b/DEPS index 8469aab4ce005..7cc9c1f331c2c 100644 --- a/DEPS +++ b/DEPS @@ -18,7 +18,7 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', # OCMock is for testing only so there is no google clone 'ocmock_git': 'https://github.com/erikdoe/ocmock.git', - 'skia_revision': '768050436c0abe18284c9d0e9371ab102854fba3', + 'skia_revision': 'bd3ee535e24687c1268e336a58be5a98748056c3', # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. diff --git a/ci/licenses_golden/licenses_skia b/ci/licenses_golden/licenses_skia index 43b703e7bdd4d..1e7b3547b7a6e 100644 --- a/ci/licenses_golden/licenses_skia +++ b/ci/licenses_golden/licenses_skia @@ -1,4 +1,4 @@ -Signature: a6e2c98e2d8de2ed07543a855c3d85dc +Signature: e49fa1ed220cc564b13990cad09a0d5d ==================================================================================================== LIBRARY: etc1 @@ -6923,8 +6923,6 @@ ORIGIN: ../../../third_party/skia/src/sksl/SkSLSampleUsage.cpp + ../../../third_ ORIGIN: ../../../third_party/skia/src/sksl/SkSLThreadContext.cpp + ../../../third_party/skia/LICENSE ORIGIN: ../../../third_party/skia/src/sksl/codegen/SkSLSPIRVtoHLSL.cpp + ../../../third_party/skia/LICENSE ORIGIN: ../../../third_party/skia/src/sksl/codegen/SkSLSPIRVtoHLSL.h + ../../../third_party/skia/LICENSE -ORIGIN: ../../../third_party/skia/src/sksl/dsl/DSLExpression.cpp + ../../../third_party/skia/LICENSE -ORIGIN: ../../../third_party/skia/src/sksl/dsl/DSLExpression.h + ../../../third_party/skia/LICENSE ORIGIN: ../../../third_party/skia/src/sksl/ir/SkSLFunctionPrototype.h + ../../../third_party/skia/LICENSE ORIGIN: ../../../third_party/skia/src/sksl/ir/SkSLModifiers.h + ../../../third_party/skia/LICENSE ORIGIN: ../../../third_party/skia/src/sksl/ir/SkSLPrefixExpression.cpp + ../../../third_party/skia/LICENSE @@ -7059,8 +7057,6 @@ FILE: ../../../third_party/skia/src/sksl/SkSLSampleUsage.cpp FILE: ../../../third_party/skia/src/sksl/SkSLThreadContext.cpp FILE: ../../../third_party/skia/src/sksl/codegen/SkSLSPIRVtoHLSL.cpp FILE: ../../../third_party/skia/src/sksl/codegen/SkSLSPIRVtoHLSL.h -FILE: ../../../third_party/skia/src/sksl/dsl/DSLExpression.cpp -FILE: ../../../third_party/skia/src/sksl/dsl/DSLExpression.h FILE: ../../../third_party/skia/src/sksl/ir/SkSLFunctionPrototype.h FILE: ../../../third_party/skia/src/sksl/ir/SkSLModifiers.h FILE: ../../../third_party/skia/src/sksl/ir/SkSLPrefixExpression.cpp From 26b9d3bcd9c51c59f9c1824580c61f8b88b67a33 Mon Sep 17 00:00:00 2001 From: hellohuanlin <41930132+hellohuanlin@users.noreply.github.com> Date: Thu, 3 Aug 2023 10:19:16 -0700 Subject: [PATCH 5/9] [ios]make the screenIfViewLoaded and windowSceneIfLoaded helpers reusable (#44303) The existing `screenIfViewLoaded` and `windowSceneIfLoaded` functions are private helpers of `FlutterViewController` class. This PR moves the logic to a category of the `UIViewController`, so it can be reused for any `UIViewController`s. *List which issues are fixed by this PR. You must list at least one issue.* https://github.com/flutter/engine/pull/43972#discussion_r1282214557 *If you had to change anything in the [flutter/tests] repo, include a link to the migration guide as per the [breaking change policy].* [C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style --- ci/licenses_golden/licenses_flutter | 6 ++ shell/platform/darwin/ios/BUILD.gn | 3 + .../ios/framework/Source/FlutterEngine.mm | 3 +- .../framework/Source/FlutterPlatformPlugin.mm | 3 +- .../framework/Source/FlutterViewController.mm | 59 +++++--------- .../Source/FlutterViewControllerTest.mm | 16 ++-- .../Source/FlutterViewController_Internal.h | 1 - ...Controller+FlutterScreenAndSceneIfLoaded.h | 21 +++++ ...ontroller+FlutterScreenAndSceneIfLoaded.mm | 30 +++++++ ...oller_FlutterScreenAndSceneIfLoadedTest.mm | 81 +++++++++++++++++++ 10 files changed, 174 insertions(+), 49 deletions(-) create mode 100644 shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h create mode 100644 shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.mm create mode 100644 shell/platform/darwin/ios/framework/Source/UIViewController_FlutterScreenAndSceneIfLoadedTest.mm diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 787aec593cd39..5c55d6239f579 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -2672,6 +2672,9 @@ ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap_I ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.mm + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/UIViewController_FlutterScreenAndSceneIfLoadedTest.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/VsyncWaiterIosTest.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm + ../../../flutter/LICENSE @@ -5395,6 +5398,9 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap_Int FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm +FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h +FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.mm +FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/UIViewController_FlutterScreenAndSceneIfLoadedTest.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/VsyncWaiterIosTest.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn index 762d3ca94a228..6a62586420148 100644 --- a/shell/platform/darwin/ios/BUILD.gn +++ b/shell/platform/darwin/ios/BUILD.gn @@ -123,6 +123,8 @@ source_set("flutter_framework_source") { "framework/Source/KeyCodeMap_Internal.h", "framework/Source/SemanticsObject.h", "framework/Source/SemanticsObject.mm", + "framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h", + "framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.mm", "framework/Source/accessibility_bridge.h", "framework/Source/accessibility_bridge.mm", "framework/Source/accessibility_text_entry.h", @@ -298,6 +300,7 @@ shared_library("ios_test_flutter") { "framework/Source/FlutterUndoManagerPluginTest.mm", "framework/Source/FlutterViewControllerTest.mm", "framework/Source/SemanticsObjectTest.mm", + "framework/Source/UIViewController_FlutterScreenAndSceneIfLoadedTest.mm", "framework/Source/connection_collection_test.mm", ] deps = [ diff --git a/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm b/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm index 00637023f3e6e..2680567f79be0 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm @@ -30,6 +30,7 @@ #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterUndoManagerDelegate.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterUndoManagerPlugin.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h" +#import "flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h" #import "flutter/shell/platform/darwin/ios/framework/Source/connection_collection.h" #import "flutter/shell/platform/darwin/ios/framework/Source/platform_message_response_darwin.h" #import "flutter/shell/platform/darwin/ios/framework/Source/profiler_metrics_ios.h" @@ -880,7 +881,7 @@ - (BOOL)createShell:(NSString*)entrypoint #if APPLICATION_EXTENSION_API_ONLY if (@available(iOS 13.0, *)) { - _isGpuDisabled = self.viewController.windowSceneIfViewLoaded.activationState == + _isGpuDisabled = self.viewController.flutterWindowSceneIfViewLoaded.activationState == UISceneActivationStateBackground; } else { // [UIApplication sharedApplication API is not available for app extension. diff --git a/shell/platform/darwin/ios/framework/Source/FlutterPlatformPlugin.mm b/shell/platform/darwin/ios/framework/Source/FlutterPlatformPlugin.mm index ab270725675be..8a3688cb11ba6 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterPlatformPlugin.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterPlatformPlugin.mm @@ -11,6 +11,7 @@ #include "flutter/fml/logging.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h" +#import "flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h" namespace { @@ -293,7 +294,7 @@ - (void)popSystemNavigator:(BOOL)isAnimated { #if APPLICATION_EXTENSION_API_ONLY if (@available(iOS 15.0, *)) { rootViewController = - [engineViewController windowSceneIfViewLoaded].keyWindow.rootViewController; + [engineViewController flutterWindowSceneIfViewLoaded].keyWindow.rootViewController; } else { FML_LOG(WARNING) << "rootViewController is not available in application extension prior to iOS 15.0."; diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 7d9c226d1d567..47f6481fe98fb 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -26,6 +26,7 @@ #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputDelegate.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h" +#import "flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h" #import "flutter/shell/platform/darwin/ios/framework/Source/platform_message_response_darwin.h" #import "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h" #import "flutter/shell/platform/darwin/ios/platform_view_ios.h" @@ -516,25 +517,6 @@ - (void)loadView { return pointer_data; } -- (UIWindowScene*)windowSceneIfViewLoaded { - if (self.viewIfLoaded == nil) { - FML_LOG(WARNING) << "Trying to access the window scene before the view is loaded."; - return nil; - } - return self.viewIfLoaded.window.windowScene; -} - -- (UIScreen*)screenIfViewLoaded { - if (@available(iOS 13.0, *)) { - if (self.viewIfLoaded == nil) { - FML_LOG(WARNING) << "Trying to access the screen before the view is loaded."; - return nil; - } - return [self windowSceneIfViewLoaded].screen; - } - return UIScreen.mainScreen; -} - static void SendFakeTouchEvent(UIScreen* screen, FlutterEngine* engine, CGPoint location, @@ -554,7 +536,7 @@ - (BOOL)scrollViewShouldScrollToTop:(UIScrollView*)scrollView { return NO; } CGPoint statusBarPoint = CGPointZero; - UIScreen* screen = [self screenIfViewLoaded]; + UIScreen* screen = [self flutterScreenIfViewLoaded]; if (screen) { SendFakeTouchEvent(screen, _engine.get(), statusBarPoint, flutter::PointerData::Change::kDown); SendFakeTouchEvent(screen, _engine.get(), statusBarPoint, flutter::PointerData::Change::kUp); @@ -869,8 +851,8 @@ - (void)viewDidAppear:(BOOL)animated { BOOL stateIsActive = YES; #if APPLICATION_EXTENSION_API_ONLY if (@available(iOS 13.0, *)) { - stateIsActive = - self.windowSceneIfViewLoaded.activationState == UISceneActivationStateForegroundActive; + stateIsActive = self.flutterWindowSceneIfViewLoaded.activationState == + UISceneActivationStateForegroundActive; } #else stateIsActive = UIApplication.sharedApplication.applicationState == UIApplicationStateActive; @@ -1166,7 +1148,7 @@ - (void)dispatchTouches:(NSSet*)touches // Activate or pause the correction of delivery frame rate of touch events. [self triggerTouchRateCorrectionIfNeeded:touches]; - const CGFloat scale = [self screenIfViewLoaded].scale; + const CGFloat scale = [self flutterScreenIfViewLoaded].scale; auto packet = std::make_unique(touches.count + touches_to_remove_count); @@ -1381,7 +1363,7 @@ - (void)updateViewportMetricsIfNeeded { - (void)viewDidLayoutSubviews { CGRect viewBounds = self.view.bounds; - CGFloat scale = [self screenIfViewLoaded].scale; + CGFloat scale = [self flutterScreenIfViewLoaded].scale; // Purposefully place this not visible. _scrollView.get().frame = CGRectMake(0.0, 0.0, viewBounds.size.width, 0.0); @@ -1400,8 +1382,8 @@ - (void)viewDidLayoutSubviews { bool applicationOrSceneIsActive = YES; #if APPLICATION_EXTENSION_API_ONLY if (@available(iOS 13.0, *)) { - applicationOrSceneIsActive = - self.windowSceneIfViewLoaded.activationState == UISceneActivationStateForegroundActive; + applicationOrSceneIsActive = self.flutterWindowSceneIfViewLoaded.activationState == + UISceneActivationStateForegroundActive; } #else applicationOrSceneIsActive = @@ -1436,7 +1418,7 @@ - (void)viewSafeAreaInsetsDidChange { // Set _viewportMetrics physical size. - (void)setViewportMetricsSize { - UIScreen* screen = [self screenIfViewLoaded]; + UIScreen* screen = [self flutterScreenIfViewLoaded]; if (!screen) { return; } @@ -1450,7 +1432,7 @@ - (void)setViewportMetricsSize { // // Viewport paddings represent the iOS safe area insets. - (void)setViewportMetricsPaddings { - UIScreen* screen = [self screenIfViewLoaded]; + UIScreen* screen = [self flutterScreenIfViewLoaded]; if (!screen) { return; } @@ -1616,7 +1598,7 @@ - (FlutterKeyboardMode)calculateKeyboardAttachMode:(NSNotification*)notification return FlutterKeyboardModeHidden; } - CGRect screenRect = [self screenIfViewLoaded].bounds; + CGRect screenRect = [self flutterScreenIfViewLoaded].bounds; CGRect adjustedKeyboardFrame = keyboardFrame; adjustedKeyboardFrame.origin.y += [self calculateMultitaskingAdjustment:screenRect keyboardFrame:keyboardFrame]; @@ -1656,7 +1638,7 @@ - (CGFloat)calculateMultitaskingAdjustment:(CGRect)screenRect keyboardFrame:(CGR } CGRect viewRectRelativeToScreen = [self.viewIfLoaded convertRect:self.viewIfLoaded.frame - toCoordinateSpace:[self screenIfViewLoaded].coordinateSpace]; + toCoordinateSpace:[self flutterScreenIfViewLoaded].coordinateSpace]; CGFloat viewBottom = CGRectGetMaxY(viewRectRelativeToScreen); CGFloat offset = screenHeight - viewBottom; if (offset > 0) { @@ -1672,14 +1654,14 @@ - (CGFloat)calculateKeyboardInset:(CGRect)keyboardFrame keyboardMode:(NSInteger) // Calculate how much of the keyboard intersects with the view. CGRect viewRectRelativeToScreen = [self.viewIfLoaded convertRect:self.viewIfLoaded.frame - toCoordinateSpace:[self screenIfViewLoaded].coordinateSpace]; + toCoordinateSpace:[self flutterScreenIfViewLoaded].coordinateSpace]; CGRect intersection = CGRectIntersection(keyboardFrame, viewRectRelativeToScreen); CGFloat portionOfKeyboardInView = CGRectGetHeight(intersection); // The keyboard is treated as an inset since we want to effectively reduce the window size by // the keyboard height. The Dart side will compute a value accounting for the keyboard-consuming // bottom padding. - CGFloat scale = [self screenIfViewLoaded].scale; + CGFloat scale = [self flutterScreenIfViewLoaded].scale; return portionOfKeyboardInView * scale; } return 0; @@ -1974,8 +1956,9 @@ - (void)performOrientationUpdate:(UIInterfaceOrientationMask)new_preferences { if (@available(iOS 16.0, *)) { NSSet* scenes = #if APPLICATION_EXTENSION_API_ONLY - self.windowSceneIfViewLoaded ? [NSSet setWithObject:self.windowSceneIfViewLoaded] - : [NSSet set]; + self.flutterWindowSceneIfViewLoaded + ? [NSSet setWithObject:self.flutterWindowSceneIfViewLoaded] + : [NSSet set]; #else [UIApplication.sharedApplication.connectedScenes filteredSetUsingPredicate:[NSPredicate predicateWithBlock:^BOOL( @@ -1987,7 +1970,7 @@ - (void)performOrientationUpdate:(UIInterfaceOrientationMask)new_preferences { } else { UIInterfaceOrientationMask currentInterfaceOrientation; if (@available(iOS 13.0, *)) { - UIWindowScene* windowScene = [self windowSceneIfViewLoaded]; + UIWindowScene* windowScene = [self flutterWindowSceneIfViewLoaded]; if (!windowScene) { FML_LOG(WARNING) << "Accessing the interface orientation when the window scene is unavailable."; @@ -2410,7 +2393,7 @@ - (BOOL)gestureRecognizer:(UIGestureRecognizer*)gestureRecognizer - (void)hoverEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)) { CGPoint location = [recognizer locationInView:self.view]; - CGFloat scale = [self screenIfViewLoaded].scale; + CGFloat scale = [self flutterScreenIfViewLoaded].scale; CGPoint oldLocation = _mouseState.location; _mouseState.location = {location.x * scale, location.y * scale}; @@ -2467,7 +2450,7 @@ - (void)hoverEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)) - (void)discreteScrollEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)) { CGPoint translation = [recognizer translationInView:self.view]; - const CGFloat scale = [self screenIfViewLoaded].scale; + const CGFloat scale = [self flutterScreenIfViewLoaded].scale; translation.x *= scale; translation.y *= scale; @@ -2496,7 +2479,7 @@ - (void)discreteScrollEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(io - (void)continuousScrollEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)) { CGPoint translation = [recognizer translationInView:self.view]; - const CGFloat scale = [self screenIfViewLoaded].scale; + const CGFloat scale = [self flutterScreenIfViewLoaded].scale; flutter::PointerData pointer_data = [self generatePointerDataAtLastMouseLocation]; pointer_data.device = reinterpret_cast(recognizer); diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm index 8723a418d50ea..d642b2c4cefeb 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm @@ -16,6 +16,7 @@ #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterFakeKeyEvents.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h" +#import "flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h" #import "flutter/shell/platform/embedder/embedder.h" #import "flutter/third_party/spring_animation/spring_animation.h" @@ -129,7 +130,6 @@ - (void)updateViewportMetricsIfNeeded; - (void)onUserSettingsChanged:(NSNotification*)notification; - (void)applicationWillTerminate:(NSNotification*)notification; - (void)goToApplicationLifecycle:(nonnull NSString*)state; -- (UIScreen*)screenIfViewLoaded; - (void)handleKeyboardNotification:(NSNotification*)notification; - (CGFloat)calculateKeyboardInset:(CGRect)keyboardFrame keyboardMode:(int)keyboardMode; - (BOOL)shouldIgnoreKeyboardNotification:(NSNotification*)notification; @@ -199,7 +199,7 @@ - (id)setUpMockView:(FlutterViewController*)viewControllerMock screen:(UIScreen*)screen viewFrame:(CGRect)viewFrame convertedFrame:(CGRect)convertedFrame { - OCMStub([viewControllerMock screenIfViewLoaded]).andReturn(screen); + OCMStub([viewControllerMock flutterScreenIfViewLoaded]).andReturn(screen); id mockView = OCMClassMock([UIView class]); OCMStub([mockView frame]).andReturn(viewFrame); OCMStub([mockView convertRect:viewFrame toCoordinateSpace:[OCMArg any]]) @@ -680,7 +680,7 @@ - (void)testCalculateKeyboardInset { bundle:nil]; FlutterViewController* viewControllerMock = OCMPartialMock(viewController); UIScreen* screen = [self setUpMockScreen]; - OCMStub([viewControllerMock screenIfViewLoaded]).andReturn(screen); + OCMStub([viewControllerMock flutterScreenIfViewLoaded]).andReturn(screen); CGFloat screenWidth = screen.bounds.size.width; CGFloat screenHeight = screen.bounds.size.height; @@ -943,7 +943,7 @@ - (void)testUpdateViewportMetricsIfNeeded_DoesNotInvokeEngineWhenShouldBeIgnored bundle:nil]; FlutterViewController* viewControllerMock = OCMPartialMock(viewController); UIScreen* screen = [self setUpMockScreen]; - OCMStub([viewControllerMock screenIfViewLoaded]).andReturn(screen); + OCMStub([viewControllerMock flutterScreenIfViewLoaded]).andReturn(screen); mockEngine.viewController = viewController; id mockCoordinator = OCMProtocolMock(@protocol(UIViewControllerTransitionCoordinator)); @@ -965,7 +965,7 @@ - (void)testViewWillTransitionToSize_DoesDelayEngineCallIfNonZeroDuration { bundle:nil]; FlutterViewController* viewControllerMock = OCMPartialMock(viewController); UIScreen* screen = [self setUpMockScreen]; - OCMStub([viewControllerMock screenIfViewLoaded]).andReturn(screen); + OCMStub([viewControllerMock flutterScreenIfViewLoaded]).andReturn(screen); mockEngine.viewController = viewController; // Mimic the device rotation with non-zero transition duration. @@ -999,7 +999,7 @@ - (void)testViewWillTransitionToSize_DoesNotDelayEngineCallIfZeroDuration { bundle:nil]; FlutterViewController* viewControllerMock = OCMPartialMock(viewController); UIScreen* screen = [self setUpMockScreen]; - OCMStub([viewControllerMock screenIfViewLoaded]).andReturn(screen); + OCMStub([viewControllerMock flutterScreenIfViewLoaded]).andReturn(screen); mockEngine.viewController = viewController; // Mimic the device rotation with zero transition duration. @@ -1512,7 +1512,7 @@ - (void)orientationTestWithOrientationUpdate:(UIInterfaceOrientationMask)mask if (@available(iOS 16.0, *)) { mockWindowScene = OCMClassMock([UIWindowScene class]); mockVC = OCMPartialMock(realVC); - OCMStub([mockVC windowSceneIfViewLoaded]).andReturn(mockWindowScene); + OCMStub([mockVC flutterWindowSceneIfViewLoaded]).andReturn(mockWindowScene); if (realVC.supportedInterfaceOrientations == mask) { OCMReject([mockWindowScene requestGeometryUpdateWithPreferences:[OCMArg any] errorHandler:[OCMArg any]]); @@ -1540,7 +1540,7 @@ - (void)orientationTestWithOrientationUpdate:(UIInterfaceOrientationMask)mask if (@available(iOS 13.0, *)) { mockWindowScene = OCMClassMock([UIWindowScene class]); mockVC = OCMPartialMock(realVC); - OCMStub([mockVC windowSceneIfViewLoaded]).andReturn(mockWindowScene); + OCMStub([mockVC flutterWindowSceneIfViewLoaded]).andReturn(mockWindowScene); OCMStub(((UIWindowScene*)mockWindowScene).interfaceOrientation) .andReturn(currentOrientation); } else { diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h b/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h index 09cb7c455446d..657c189dc1628 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h @@ -65,7 +65,6 @@ typedef void (^FlutterKeyboardAnimationCallback)(fml::TimePoint); - (void)addInternalPlugins; - (void)deregisterNotifications; - (int32_t)accessibilityFlags; -- (UIWindowScene*)windowSceneIfViewLoaded API_AVAILABLE(ios(13.0)); @end diff --git a/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h b/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h new file mode 100644 index 0000000000000..6ea449287093b --- /dev/null +++ b/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h @@ -0,0 +1,21 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_UIVIEWCONTROLLER_FLUTTERSCREENANDSCENEIFLOADED_H_ +#define FLUTTER_UIVIEWCONTROLLER_FLUTTERSCREENANDSCENEIFLOADED_H_ + +#import + +@interface UIViewController (FlutterScreenAndSceneIfLoaded) + +/// Returns a UIWindowScene if the UIViewController's view is loaded, and nil otherwise. +- (UIWindowScene*)flutterWindowSceneIfViewLoaded API_AVAILABLE(ios(13.0)); + +/// Before iOS 13, returns the main screen; After iOS 13, returns the screen the UIViewController is +/// attached to if its view is loaded, and nil otherwise. +- (UIScreen*)flutterScreenIfViewLoaded; + +@end + +#endif // FLUTTER_UIVIEWCONTROLLER_FLUTTERSCREENANDSCENEIFLOADED_H_ diff --git a/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.mm b/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.mm new file mode 100644 index 0000000000000..5eb4f71103056 --- /dev/null +++ b/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.mm @@ -0,0 +1,30 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import "flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h" + +#include "flutter/fml/logging.h" + +@implementation UIViewController (FlutterScreenAndSceneIfLoaded) + +- (UIWindowScene*)flutterWindowSceneIfViewLoaded { + if (self.viewIfLoaded == nil) { + FML_LOG(WARNING) << "Trying to access the window scene before the view is loaded."; + return nil; + } + return self.viewIfLoaded.window.windowScene; +} + +- (UIScreen*)flutterScreenIfViewLoaded { + if (@available(iOS 13.0, *)) { + if (self.viewIfLoaded == nil) { + FML_LOG(WARNING) << "Trying to access the screen before the view is loaded."; + return nil; + } + return [self flutterWindowSceneIfViewLoaded].screen; + } + return UIScreen.mainScreen; +} + +@end diff --git a/shell/platform/darwin/ios/framework/Source/UIViewController_FlutterScreenAndSceneIfLoadedTest.mm b/shell/platform/darwin/ios/framework/Source/UIViewController_FlutterScreenAndSceneIfLoadedTest.mm new file mode 100644 index 0000000000000..c68e22da6bcac --- /dev/null +++ b/shell/platform/darwin/ios/framework/Source/UIViewController_FlutterScreenAndSceneIfLoadedTest.mm @@ -0,0 +1,81 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import + +#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" +#import "flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h" + +FLUTTER_ASSERT_ARC + +@interface UIViewController_FlutterViewAndSceneIfLoadedTest : XCTestCase +@end + +@implementation UIViewController_FlutterViewAndSceneIfLoadedTest + +- (void)testWindowSceneIfViewLoadedReturnsWindowSceneIfViewLoaded { + if (@available(iOS 13.0, *)) { + UIViewController* viewController = [[UIViewController alloc] initWithNibName:nil bundle:nil]; + + NSSet* scenes = UIApplication.sharedApplication.connectedScenes; + XCTAssertEqual(scenes.count, 1UL, @"There must only be 1 scene for test"); + UIScene* scene = scenes.anyObject; + XCTAssert([scene isKindOfClass:[UIWindowScene class]], @"Must be a window scene for test"); + UIWindowScene* windowScene = (UIWindowScene*)scene; + XCTAssert(windowScene.windows.count > 0, @"There must be at least 1 window for test"); + UIWindow* window = windowScene.windows[0]; + [window addSubview:viewController.view]; + + [viewController loadView]; + XCTAssertEqual(viewController.flutterWindowSceneIfViewLoaded, windowScene, + @"Must return the correct window scene when view loaded"); + } +} + +- (void)testWindowSceneIfViewLoadedReturnsNilIfViewNotLoaded { + if (@available(iOS 13.0, *)) { + UIViewController* viewController = [[UIViewController alloc] initWithNibName:nil bundle:nil]; + XCTAssertNil(viewController.flutterWindowSceneIfViewLoaded, + @"Must return nil window scene when view not loaded"); + } +} + +- (void)testScreenIfViewLoadedReturnsMainScreenBeforeIOS13 { + if (@available(iOS 13.0, *)) { + return; + } + + UIViewController* viewController = [[UIViewController alloc] initWithNibName:nil bundle:nil]; + XCTAssertEqual(viewController.flutterScreenIfViewLoaded, UIScreen.mainScreen, + @"Must return UIScreen.mainScreen before iOS 13"); +} + +- (void)testScreenIfViewLoadedReturnsScreenIfViewLoadedAfterIOS13 { + if (@available(iOS 13.0, *)) { + UIViewController* viewController = [[UIViewController alloc] initWithNibName:nil bundle:nil]; + + NSSet* scenes = UIApplication.sharedApplication.connectedScenes; + XCTAssertEqual(scenes.count, 1UL, @"There must only be 1 scene for test"); + UIScene* scene = scenes.anyObject; + XCTAssert([scene isKindOfClass:[UIWindowScene class]], @"Must be a window scene for test"); + UIWindowScene* windowScene = (UIWindowScene*)scene; + XCTAssert(windowScene.windows.count > 0, @"There must be at least 1 window for test"); + UIWindow* window = windowScene.windows[0]; + [window addSubview:viewController.view]; + + [viewController loadView]; + XCTAssertEqual(viewController.flutterScreenIfViewLoaded, windowScene.screen, + @"Must return the correct screen when view loaded"); + } +} + +- (void)testScreenIfViewLoadedReturnsNilIfViewNotLoadedAfterIOS13 { + if (@available(iOS 13.0, *)) { + UIViewController* viewController = [[UIViewController alloc] initWithNibName:nil bundle:nil]; + XCTAssertNil(viewController.flutterScreenIfViewLoaded, + @"Must return nil screen when view not loaded"); + } +} + +@end From 9025bfb13ea2fccaabafad05d10cae5c21d098a5 Mon Sep 17 00:00:00 2001 From: gaaclarke <30870216+gaaclarke@users.noreply.github.com> Date: Thu, 3 Aug 2023 10:50:41 -0700 Subject: [PATCH 6/9] [Impeller] updated validation layers documentation (#44328) [C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style --- impeller/docs/android_validation_layers.md | 36 ++++++++++++---------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/impeller/docs/android_validation_layers.md b/impeller/docs/android_validation_layers.md index df0125c4224ac..55ada1d469552 100644 --- a/impeller/docs/android_validation_layers.md +++ b/impeller/docs/android_validation_layers.md @@ -1,23 +1,27 @@ # Android Vulkan Validation Layers -This is a quick guide to get Vulkan validation layers support for a Flutter application. This guide assumes that you've created the application with `flutter create`, otherwise the locations might vary. +If you want to run Vulkan Validation Layers with a custom engine build you need +to add the `--enable-vulkan-validation-layers` to the `gn` invocation to make +sure the layers are built and injected into the Flutter jar. -1. Download the validation layers from this [GitHub](https://github.com/KhronosGroup/Vulkan-ValidationLayers/releases) releases. Typically named `android-binaries-1.3.231.1.zip`. -2. When you unzip the file, you will see: `arm64-v8a armeabi-v7a x86 x86_64` -3. Copy these directories to `${FLUTTER_APP}/android/app/src/main/vklibs`. The layout should look similar to: +Example: -``` -src/main/vklibs/ - arm64-v8a/ - libVkLayer_khronos_validation.so - armeabi-v7a/ - libVkLayer_khronos_validation.so - x86/ - libVkLayer_khronos_validation.so - x86-64/ - libVkLayer_khronos_validation.so +```sh +flutter/tools/gn \ + --runtime-mode=debug \ + --enable-impeller-vulkan \ + --enable-vulkan-validation-layers \ + --no-lto \ + --unoptimized \ + --android \ + --android-cpu=arm64 ``` -4. Add the following line to `${FLUTTER_APP}/android/app/build.gradle`, `android > sourceSets` section: `main.jniLibs.srcDirs += 'src/main/vklibs'`. +Then adding the following field to the +`android/app/src/main/AndroidManifest.xml` will turn them on: -5. This should enable Vulkan validation layers on your Android application. +```xml + +``` From b5a70db003eb1b1b635891f6bac846f46ec0cafb Mon Sep 17 00:00:00 2001 From: skia-flutter-autoroll Date: Thu, 3 Aug 2023 13:59:03 -0400 Subject: [PATCH 7/9] Roll Dart SDK from dcd09f5726b7 to e3d2b4a190aa (1 revision) (#44330) https://dart.googlesource.com/sdk.git/+log/dcd09f5726b7..e3d2b4a190aa 2023-08-03 dart-internal-merge@dart-ci-internal.iam.gserviceaccount.com Version 3.2.0-34.0.dev If this roll has caused a breakage, revert this CL and stop the roller using the controls here: https://autoroll.skia.org/r/dart-sdk-flutter-engine Please CC dart-vm-team@google.com,jacksongardner@google.com on the revert to ensure that a human is aware of the problem. To file a bug in Flutter Engine: https://github.com/flutter/flutter/issues/new/choose To report a problem with the AutoRoller itself, please file a bug: https://bugs.chromium.org/p/skia/issues/entry?template=Autoroller+Bug Documentation for the AutoRoller is here: https://skia.googlesource.com/buildbot/+doc/main/autoroll/README.md --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 7cc9c1f331c2c..7c4f3b5dc24ce 100644 --- a/DEPS +++ b/DEPS @@ -53,7 +53,7 @@ vars = { # Dart is: https://github.com/dart-lang/sdk/blob/main/DEPS # You can use //tools/dart/create_updated_flutter_deps.py to produce # updated revision list of existing dependencies. - 'dart_revision': 'dcd09f5726b770df00ae0a596eff0817238be232', + 'dart_revision': 'e3d2b4a190aab3ce22bb0f419f1461bfea05c656', # WARNING: DO NOT EDIT MANUALLY # The lines between blank lines above and below are generated by a script. See create_updated_flutter_deps.py From 0a57c5686c487ccb20c0ab2d6a80a37b26b9b1b3 Mon Sep 17 00:00:00 2001 From: skia-flutter-autoroll Date: Thu, 3 Aug 2023 14:02:12 -0400 Subject: [PATCH 8/9] Roll Skia from bd3ee535e246 to 36072a994f11 (2 revisions) (#44329) https://skia.googlesource.com/skia.git/+log/bd3ee535e246..36072a994f11 2023-08-03 skia-autoroll@skia-public.iam.gserviceaccount.com Roll vulkan-deps from e057bba499d3 to d8fdb68e5922 (1 revision) 2023-08-03 lovisolo@google.com [bazel] //gm/BazelGMRunner.cpp: Print successful/skipped/failed GM counts. If this roll has caused a breakage, revert this CL and stop the roller using the controls here: https://autoroll.skia.org/r/skia-flutter-autoroll Please CC brianosman@google.com,jacksongardner@google.com,jvanverth@google.com,rmistry@google.com on the revert to ensure that a human is aware of the problem. To file a bug in Skia: https://bugs.chromium.org/p/skia/issues/entry To file a bug in Flutter: https://github.com/flutter/flutter/issues/new/choose To report a problem with the AutoRoller itself, please file a bug: https://bugs.chromium.org/p/skia/issues/entry?template=Autoroller+Bug Documentation for the AutoRoller is here: https://skia.googlesource.com/buildbot/+doc/main/autoroll/README.md --- DEPS | 2 +- ci/licenses_golden/licenses_skia | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DEPS b/DEPS index 7c4f3b5dc24ce..3bcc0fb0c0710 100644 --- a/DEPS +++ b/DEPS @@ -18,7 +18,7 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', # OCMock is for testing only so there is no google clone 'ocmock_git': 'https://github.com/erikdoe/ocmock.git', - 'skia_revision': 'bd3ee535e24687c1268e336a58be5a98748056c3', + 'skia_revision': '36072a994f119159798b56cb53a77c932c15076b', # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. diff --git a/ci/licenses_golden/licenses_skia b/ci/licenses_golden/licenses_skia index 1e7b3547b7a6e..81a7f0ba547f4 100644 --- a/ci/licenses_golden/licenses_skia +++ b/ci/licenses_golden/licenses_skia @@ -1,4 +1,4 @@ -Signature: e49fa1ed220cc564b13990cad09a0d5d +Signature: d11bb0dd7e11d3f0bce9fc0ce268bf0e ==================================================================================================== LIBRARY: etc1 From b08e1410a06a521087a1d178e16bf76e6a0e678e Mon Sep 17 00:00:00 2001 From: John McCutchan Date: Thu, 3 Aug 2023 11:50:52 -0700 Subject: [PATCH 9/9] Add @Keep annotations to avoid dead code elimination of classes only referenced by JNI (#44337) Missing piece --- .../io/flutter/embedding/engine/renderer/FlutterRenderer.java | 2 ++ shell/platform/android/io/flutter/view/TextureRegistry.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java index 423a355038c4a..270e51a28a1ad 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java @@ -12,6 +12,7 @@ import android.os.Build; import android.os.Handler; import android.view.Surface; +import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; @@ -319,6 +320,7 @@ public void run() { } } + @Keep final class ImageTextureRegistryEntry implements TextureRegistry.ImageTextureEntry { private final long id; private boolean released; diff --git a/shell/platform/android/io/flutter/view/TextureRegistry.java b/shell/platform/android/io/flutter/view/TextureRegistry.java index 98ec56476668d..86b5e71aadffb 100644 --- a/shell/platform/android/io/flutter/view/TextureRegistry.java +++ b/shell/platform/android/io/flutter/view/TextureRegistry.java @@ -6,6 +6,7 @@ import android.graphics.SurfaceTexture; import android.media.Image; +import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -75,6 +76,7 @@ default void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listen default void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) {} } + @Keep interface ImageTextureEntry extends TextureEntry { /** @return the identity of this ImageTextureEntry */ long id();