Skip to content

Commit

Permalink
Simplify ArImmersiveOverlay to only use SurfaceView
Browse files Browse the repository at this point in the history 
ArImmersiveOverlay previously chose between using a SurfaceView or using
a Dialog to show AR content based on whether or not the DOMOverlay
feature was enabled. This changes it to use only a SurfaceView; however,
this now requires the page to always enter fullscreen for Android AR
sessions so that the SurfaceView does not appear behind any system UI.

Not only does this simplify the rendering path and choices for AR, but
it is required to actually show InfoBars or Prompts in front of AR
Content (though there is still some outstanding work to be done before
that can actually happen).

Though the events weren't being forwarded to the page, this adds two
further checks that the DOM Overlay feature is enabled before both
sending or processing data that is used to generate the beforexrselect
event.

Bug: 1203490
Change-Id: I28c5bb6a8c9f2d84ecc03bc0a2084bf84c6e2f7c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2857524
Commit-Queue: Alexander Cooper <alcooper@chromium.org>
Commit-Queue: Daniel Cheng <dcheng@chromium.org>
Auto-Submit: Alexander Cooper <alcooper@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Klaus Weidner <klausw@chromium.org>
Cr-Commit-Position: refs/heads/master@{#879170}
  • Loading branch information
@alcooper91
alcooper91 authored and Chromium LUCI CQ committed May 5, 2021
1 parent 183830a commit af70b30
Show file tree
Hide file tree
Showing 10 changed files with 126 additions and 174 deletions.
137 changes: 21 additions & 116 deletions components/webxr/android/java/src/org/chromium/components/webxr/ArImmersiveOverlay.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,15 @@

import android.annotation.SuppressLint;
import android.app.Activity;
import android.app.Dialog;
import android.content.Context;
import android.content.pm.ActivityInfo;
import android.graphics.PixelFormat;
import android.graphics.Point;
import android.os.Build;
import android.view.Display;
import android.view.Gravity;
import android.view.MotionEvent;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.View;
import android.view.ViewGroup;
import android.view.WindowManager;

import androidx.annotation.NonNull;

Expand All @@ -29,7 +24,6 @@
import org.chromium.content_public.browser.WebContents;
import org.chromium.content_public.browser.WebContentsObserver;
import org.chromium.ui.display.DisplayAndroidManager;
import org.chromium.ui.widget.Toast;

import java.util.HashMap;
import java.util.Map;
Expand All @@ -48,8 +42,9 @@ public class ArImmersiveOverlay
private boolean mSurfaceReportedReady;
private Integer mRestoreOrientation;
private boolean mCleanupInProgress;
private SurfaceUiWrapper mSurfaceUi;
private ArSurfaceView mArSurfaceView;
private WebContents mWebContents;
private boolean mUseOverlay;

// Set containing all currently touching pointers.
private HashMap<Integer, PointerData> mPointerIdToData;
Expand All @@ -70,95 +65,11 @@ public void show(@NonNull ArCompositorDelegate compositorDelegate,
mPointerIdToData = new HashMap<Integer, PointerData>();
mPrimaryPointerId = null;

mUseOverlay = useOverlay;

// Choose a concrete implementation to create a drawable Surface and make it fullscreen.
// It forwards SurfaceHolder callbacks and touch events to this ArImmersiveOverlay object.
if (useOverlay) {
mSurfaceUi = new SurfaceUiCompositor(canRenderDomContent);
} else {
mSurfaceUi = new SurfaceUiDialog();
}
}

private interface SurfaceUiWrapper {
public void onSurfaceVisible();
public void forwardMotionEvent(MotionEvent ev);
public void destroy();
}

// The default Dialog cancellation behavior destroys the Surface before we get notified via the
// Cancelation callback. This is unfortunate, because we need to ensure that the compositor is
// stopped before the surface is destroyed. This class allows us to override the default
// cancellation behavior to properly shutdown the compositor before the surface is destroyed. It
// is unclear why the SurfaceHolder callbacks are not triggered.
private class ArDialog extends Dialog {
public ArDialog(Context context, int themeResId) {
super(context, themeResId);
}

@Override
public void cancel() {
ArCoreJavaUtils.onBackPressed();
super.cancel();
}
}

private class SurfaceUiDialog implements SurfaceUiWrapper {
private Toast mNotificationToast;
private ArDialog mDialog;
// Android supports multiple variants of fullscreen applications. Use fully-immersive
// "sticky" mode without navigation or status bars, and show a toast with a "pull from top
// and press back button to exit" prompt.
private static final int VISIBILITY_FLAGS_IMMERSIVE = View.SYSTEM_UI_FLAG_LAYOUT_STABLE
| View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION
| View.SYSTEM_UI_FLAG_FULLSCREEN | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION
| View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY;

public SurfaceUiDialog() {
// Create a fullscreen dialog and use its backing Surface for drawing.
mDialog = new ArDialog(mActivity, android.R.style.Theme_NoTitleBar_Fullscreen);
mDialog.getWindow().setBackgroundDrawable(null);
mDialog.getWindow().takeSurface(ArImmersiveOverlay.this);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
// Use maximum fullscreen, ignoring a notch if present. This code path is used
// for non-DOM-Overlay mode where the browser compositor view isn't visible.
// In DOM Overlay mode (SurfaceUiCompositor), Blink configures this separately
// via ViewportData::SetExpandIntoDisplayCutout.
mDialog.getWindow().setFlags(WindowManager.LayoutParams.FLAG_LAYOUT_NO_LIMITS,
WindowManager.LayoutParams.FLAG_LAYOUT_NO_LIMITS);
mDialog.getWindow().getAttributes().layoutInDisplayCutoutMode =
WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES;
}
View view = mDialog.getWindow().getDecorView();
view.setSystemUiVisibility(VISIBILITY_FLAGS_IMMERSIVE);
view.setOnTouchListener(ArImmersiveOverlay.this);
view.setKeepScreenOn(true);
mDialog.getWindow().setLayout(
ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT);
mDialog.show();
}

@Override // SurfaceUiWrapper
public void onSurfaceVisible() {
if (mNotificationToast != null) {
mNotificationToast.cancel();
}
int resId = R.string.immersive_fullscreen_api_notification;
mNotificationToast = Toast.makeText(mActivity, resId, Toast.LENGTH_LONG);
mNotificationToast.setGravity(Gravity.TOP | Gravity.CENTER, 0, 0);
mNotificationToast.show();
}

@Override // SurfaceUiWrapper
public void forwardMotionEvent(MotionEvent ev) {}

@Override // SurfaceUiWrapper
public void destroy() {
if (mNotificationToast != null) {
mNotificationToast.cancel();
mNotificationToast = null;
}
mDialog.dismiss();
}
mArSurfaceView = new ArSurfaceView(canRenderDomContent);
}

private class PointerData {
Expand All @@ -173,24 +84,29 @@ public PointerData(float x, float y, boolean touching) {
}
}

private class SurfaceUiCompositor implements SurfaceUiWrapper {
private class ArSurfaceView {
private SurfaceView mSurfaceView;
private WebContentsObserver mWebContentsObserver;
private boolean mDomSurfaceNeedsConfiguring;

@SuppressLint("ClickableViewAccessibility")
public SurfaceUiCompositor(boolean canRenderDomContent) {
// If we can't render the dom content on top of the camera/gl layers manually, then
// we need to configure the DOM content's surface view to overlay ours. We need to
// track this so that we ensure we teardown everything we need to teardown as well.
mDomSurfaceNeedsConfiguring = !canRenderDomContent;
public ArSurfaceView(boolean canRenderDomContent) {
// If we need to show the dom content, but can't render it on top of the camera/gl
// layers manually, then we need to configure the DOM content's surface view to
// overlay ours. We need to track this so that we ensure we teardown everything
// we need to teardown as well.
mDomSurfaceNeedsConfiguring = mUseOverlay && !canRenderDomContent;

// Enable alpha channel for the compositor and make the background transparent.
// Note that this needs to happen before we create and parent our SurfaceView, so that
// it ends up on top if the Dom Surface did not need configuring.
if (DEBUG_LOGS) {
Log.i(TAG, "calling mArCompositorDelegate.setOverlayImmersiveArMode(true)");
}

// While it's fine to omit if the page does not use DOMOverlay, once the page does
// use DOMOverlay, something appears to have changed such that it becomes required,
// otherwies the DOM SurfaceView will be in front of the XR content.
mArCompositorDelegate.setOverlayImmersiveArMode(true, mDomSurfaceNeedsConfiguring);

mSurfaceView = new SurfaceView(mActivity);
Expand Down Expand Up @@ -226,15 +142,6 @@ public void didToggleFullscreenModeForTab(
mWebContents.addObserver(mWebContentsObserver);
}

@Override // SurfaceUiWrapper
public void onSurfaceVisible() {}

@Override // SurfaceUiWrapper
public void forwardMotionEvent(MotionEvent ev) {
mArCompositorDelegate.dispatchTouchEvent(ev);
}

@Override // SurfaceUiWrapper
public void destroy() {
mWebContents.removeObserver(mWebContentsObserver);
View content = mActivity.getWindow().findViewById(android.R.id.content);
Expand Down Expand Up @@ -400,7 +307,9 @@ public boolean onTouch(View v, MotionEvent ev) {
// We need to consume the touch (returning true) to ensure that we get
// followup events such as MOVE and UP. DOM Overlay mode needs to forward
// the touch to the content view so that its UI elements keep working.
mSurfaceUi.forwardMotionEvent(ev);
if (mUseOverlay) {
mArCompositorDelegate.dispatchTouchEvent(ev);
}
return true;
}

Expand Down Expand Up @@ -480,7 +389,7 @@ public void surfaceChanged(SurfaceHolder holder, int format, int width, int heig
//
// While it would be preferable to wait until the surface is at the desired fullscreen
// resolution, i.e. via mActivity.getFullscreenManager().getPersistentFullscreenMode(), that
// causes a chicken-and-egg problem for SurfaceUiCompositor mode as used for DOM overlay.
// causes a chicken-and-egg problem for ArSurfaceView mode as used for DOM overlay.
// Chrome's fullscreen mode is triggered by the Blink side setting an element fullscreen
// after the session starts, but the session doesn't start until we report the drawing
// surface being ready (including a configured size), so we use this reported size assuming
Expand All @@ -505,10 +414,6 @@ public void surfaceChanged(SurfaceHolder holder, int format, int width, int heig
mArCoreJavaUtils.onDrawingSurfaceReady(holder.getSurface(),
mWebContents.getTopLevelNativeWindow(), rotation, width, height);
mSurfaceReportedReady = true;

// Show the toast with instructions how to exit fullscreen mode now if necessary.
// Not needed in DOM overlay mode which uses FullscreenHtmlApiHandler to do so.
mSurfaceUi.onSurfaceVisible();
}

@Override // SurfaceHolder.Callback2
Expand All @@ -533,7 +438,7 @@ public void cleanupAndExit() {
// the destroy callbacks to ensure consistent state after non-exiting lifecycle events.
mArCoreJavaUtils.onDrawingSurfaceDestroyed();

mSurfaceUi.destroy();
mArSurfaceView.destroy();

// The JS app may have put an element into fullscreen mode during the immersive session,
// even if this wasn't visible to the user. Ensure that we fully exit out of any active
Expand Down
8 changes: 7 additions & 1 deletion device/vr/android/arcore/ar_compositor_frame_sink.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,13 @@ viz::CompositorFrame ArCompositorFrameSink::CreateFrame(WebXrFrame* xr_frame,
// First the DOM, if it's enabled
if (should_composite_dom_overlay_) {
auto dom_surface_id = xr_frame_sink_client_->GetDOMSurface();
if (dom_surface_id && dom_surface_id->is_valid()) {
bool can_composite_dom_overlay =
dom_surface_id && dom_surface_id->is_valid();
DVLOG(3)
<< __func__
<< " Attempting to composite DOMOverlay, can_composite_dom_overlay="
<< can_composite_dom_overlay;
if (can_composite_dom_overlay) {
viz::SharedQuadState* dom_quad_state =
render_pass->CreateAndAppendSharedQuadState();
dom_quad_state->SetAll(
Expand Down
4 changes: 3 additions & 1 deletion device/vr/android/arcore/arcore_gl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1654,7 +1654,9 @@ std::vector<mojom::XRInputSourceStatePtr> ArCoreGl::GetInputSourceStates() {
}

// Save the touch point for use in Blink's XR input event deduplication.
state->overlay_pointer_position = screen_last_touch;
if (IsFeatureEnabled(device::mojom::XRSessionFeature::DOM_OVERLAY)) {
state->overlay_pointer_position = screen_last_touch;
}

state->description = device::mojom::XRInputSourceDescription::New();

Expand Down
23 changes: 16 additions & 7 deletions third_party/blink/renderer/core/fullscreen/fullscreen.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,16 +283,25 @@ bool AllowedToRequestFullscreen(Document& document) {
//
// The current implementation of WebXR's "dom-overlay" mode internally uses
// the Fullscreen API to show a single DOM element based on configuration at
// XR session start. The WebXR API doesn't support changing elements during
// the session, so to avoid inconsistencies between implementations we need
// to block changes via Fullscreen API while the XR session is active, while
// still allowing the XR code to set up fullscreen mode on session start.
// XR session start. In addition, for WebXR sessions without "dom-overlay"
// the renderer may need to force the page to fullscreen to ensure that
// browser UI hides/responds accordingly. In either case, requesting a WebXR
// Session does require a user gesture, but it has likely expired by the time
// the renderer actually gets the XR session from the device and attempts
// to fullscreen the page.
if (ScopedAllowFullscreen::FullscreenAllowedReason() ==
ScopedAllowFullscreen::kXrOverlay) {
DVLOG(1) << __func__
<< ": allowing fullscreen element setup for XR DOM overlay";
ScopedAllowFullscreen::kXrOverlay ||
ScopedAllowFullscreen::FullscreenAllowedReason() ==
ScopedAllowFullscreen::kXrSession) {
DVLOG(1) << __func__ << ": allowing fullscreen element setup for XR";
return true;
}

// The WebXR API doesn't support changing elements during the session if the
// dom-overlay feature is in use (indicated by the IsXrOverlay property). To
// avoid inconsistencies between implementations we need to block changes via
// Fullscreen API while the XR session is active, while still allowing the XR
// code to set up fullscreen mode on session start.
if (document.IsXrOverlay()) {
DVLOG(1) << __func__
<< ": rejecting change of fullscreen element for XR DOM overlay";
Expand Down
2 changes: 1 addition & 1 deletion third_party/blink/renderer/core/fullscreen/scoped_allow_fullscreen.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class CORE_EXPORT ScopedAllowFullscreen {
STACK_ALLOCATED();

public:
enum Reason { kOrientationChange, kXrOverlay };
enum Reason { kOrientationChange, kXrOverlay, kXrSession };

static base::Optional<Reason> FullscreenAllowedReason();
explicit ScopedAllowFullscreen(Reason);
Expand Down
14 changes: 10 additions & 4 deletions third_party/blink/renderer/modules/xr/xr_enter_fullscreen_observer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ void XrEnterFullscreenObserver::Invoke(ExecutionContext* execution_context,

void XrEnterFullscreenObserver::RequestFullscreen(
Element* fullscreen_element,
bool setup_for_dom_overlay,
base::OnceCallback<void(bool)> on_completed) {
DCHECK(!on_completed_);
DCHECK(fullscreen_element);
Expand Down Expand Up @@ -84,11 +85,16 @@ void XrEnterFullscreenObserver::RequestFullscreen(
// immersive session had required a user activation state, but that may have
// expired by now due to the user taking time to respond to the consent
// prompt.
ScopedAllowFullscreen scope(ScopedAllowFullscreen::kXrOverlay);
ScopedAllowFullscreen scope(setup_for_dom_overlay
? ScopedAllowFullscreen::kXrOverlay
: ScopedAllowFullscreen::kXrSession);

Fullscreen::RequestFullscreen(*fullscreen_element_, options,
FullscreenRequestType::kUnprefixed |
FullscreenRequestType::kForXrOverlay);
FullscreenRequestType request_type = FullscreenRequestType::kUnprefixed;
if (setup_for_dom_overlay) {
request_type = request_type | FullscreenRequestType::kForXrOverlay;
}

Fullscreen::RequestFullscreen(*fullscreen_element_, options, request_type);

if (!wait_for_fullscreen_change) {
// Element was already fullscreen, proceed with session creation.
Expand Down
1 change: 1 addition & 0 deletions third_party/blink/renderer/modules/xr/xr_enter_fullscreen_observer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class XrEnterFullscreenObserver : public NativeEventListener {
// Attempt to enter fullscreen with |element| as the root. |on_completed| will
// be notified with whether or not fullscreen was successfully entered.
void RequestFullscreen(Element* element,
bool setup_for_dom_overlay,
base::OnceCallback<void(bool)> on_completed);

void Trace(Visitor*) const override;
Expand Down
3 changes: 2 additions & 1 deletion third_party/blink/renderer/modules/xr/xr_session.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2018,7 +2018,8 @@ void XRSession::OnInputStateChangeInternal(
// cross-origin content. If that's the case, the input source is set as
// invisible, and must not return poses or hit test results.
bool hide_input_source = false;
if (overlay_element_ && input_state->overlay_pointer_position) {
if (IsFeatureEnabled(device::mojom::XRSessionFeature::DOM_OVERLAY) &&
overlay_element_ && input_state->overlay_pointer_position) {
input_source->ProcessOverlayHitTest(overlay_element_, input_state);
if (!stored_input_source && !input_source->IsVisible()) {
DVLOG(2) << __func__ << ": (new) hidden_input_source";
Expand Down
Loading

0 comments on commit af70b30

Please sign in to comment.