Allow drawing outside of platform layers (#1190)

## Proposed Changes

- Use skia's BBHFactory to track real drawing bounds - requires
JetBrains/skiko#889 (skiko `0.7.98`)
- Adding additional click filtering to handle clicks on "shadow" area as
click outside

## Testing

Test: Try to use new platform layers with shadows

<img width="695" alt="image"
src="https://github.com/JetBrains/compose-multiplatform-core/assets/1836384/80dce85f-814d-4d51-ab5e-6535ed976d48">

## Issues Fixed

Fixes: JetBrains/compose-multiplatform#4460

compose/ui/ui/api/desktop/ui.api

@@ -3134,6 +3134,7 @@ public abstract interface class androidx/compose/ui/platform/PlatformContext {
 	public fun calculateLocalPosition-MK-Hz9U (J)J
 	public fun calculatePositionInWindow-MK-Hz9U (J)J
 	public abstract fun getInputModeManager ()Landroidx/compose/ui/input/InputModeManager;
+	public fun getMeasureDrawLayerBounds ()Z
 	public fun getParentFocusManager ()Landroidx/compose/ui/focus/FocusManager;
 	public fun getRootForTestListener ()Landroidx/compose/ui/platform/PlatformContext$RootForTestListener;
 	public fun getSemanticsOwnerListener ()Landroidx/compose/ui/platform/PlatformContext$SemanticsOwnerListener;

compose/ui/ui/src/desktopMain/kotlin/androidx/compose/ui/ComposeFeatureFlags.desktop.kt

@@ -22,6 +22,13 @@ import androidx.compose.ui.window.Popup
 internal enum class LayerType {
     OnSameCanvas,
     OnComponent,
+
+    /**
+     * TODO known issues:
+     *  - [Rendering issues on Linux](https://github.com/JetBrains/compose-multiplatform/issues/4437)
+     *  - [Blinking when showing](https://github.com/JetBrains/compose-multiplatform/issues/4475)
+     *  - [Resizing the parent window clips the dialog](https://github.com/JetBrains/compose-multiplatform/issues/4484)
+     */
     OnWindow;
 
     companion object {

compose/ui/ui/src/desktopMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.desktop.kt

@@ -101,6 +101,11 @@ internal class ComposeSceneMediator(
     private var exceptionHandler: WindowExceptionHandler?,
     private val eventListener: AwtEventListener = OnlyValidPrimaryMouseButtonFilter,
 
+    /**
+     * @see PlatformContext.measureDrawLayerBounds
+     */
+    private val measureDrawLayerBounds: Boolean = false,
+
     val coroutineContext: CoroutineContext,
 
     skiaLayerComponentFactory: (ComposeSceneMediator) -> SkiaLayerComponent,
@@ -625,6 +630,7 @@ internal class ComposeSceneMediator(
         override fun calculateLocalPosition(positionInWindow: Offset): Offset =
             windowContext.calculateLocalPosition(container, positionInWindow)
 
+        override val measureDrawLayerBounds: Boolean = this@ComposeSceneMediator.measureDrawLayerBounds
         override val viewConfiguration: ViewConfiguration = DesktopViewConfiguration()
         override val textInputService: PlatformTextInputService = this@ComposeSceneMediator.textInputService
 

compose/ui/ui/src/desktopMain/kotlin/androidx/compose/ui/scene/DesktopComposeSceneLayer.desktop.kt

@@ -24,14 +24,21 @@ import androidx.compose.ui.awt.AwtEventListeners
 import androidx.compose.ui.awt.OnlyValidPrimaryMouseButtonFilter
 import androidx.compose.ui.awt.toAwtRectangle
 import androidx.compose.ui.input.pointer.PointerEventType
+import androidx.compose.ui.skiko.RecordDrawRectSkikoViewDecorator
 import androidx.compose.ui.unit.Density
 import androidx.compose.ui.unit.IntOffset
+import androidx.compose.ui.unit.IntRect
 import androidx.compose.ui.unit.LayoutDirection
+import androidx.compose.ui.unit.roundToIntRect
 import androidx.compose.ui.util.fastForEachReversed
+import java.awt.Rectangle
 import java.awt.event.KeyEvent
 import java.awt.event.MouseEvent
 import javax.swing.SwingUtilities
+import kotlin.math.max
+import kotlin.math.min
 import org.jetbrains.skia.Canvas
+import org.jetbrains.skiko.SkikoView
 
 /**
  * Represents an abstract class for a desktop Compose scene layer.
@@ -49,11 +56,25 @@ internal abstract class DesktopComposeSceneLayer(
     protected val eventListener get() = AwtEventListeners(
         OnlyValidPrimaryMouseButtonFilter,
         DetectEventOutsideLayer(),
+        boundsEventFilter,
         FocusableLayerEventFilter()
     )
+    private val boundsEventFilter = BoundsEventFilter(
+        bounds = Rectangle(windowContainer.size)
+    )
 
     protected abstract val mediator: ComposeSceneMediator?
 
+    /**
+     * Bounds of real drawings based on previous renders.
+     */
+    protected var drawBounds = IntRect.Zero
+
+    /**
+     * The maximum amount to inflate the [drawBounds] comparing to [boundsInWindow].
+     */
+    private var maxDrawInflate = IntRect.Zero
+
     private var outsidePointerCallback: ((eventType: PointerEventType) -> Unit)? = null
     private var isClosed = false
 
@@ -69,6 +90,13 @@ internal abstract class DesktopComposeSceneLayer(
             mediator?.onChangeLayoutDirection(value)
         }
 
+    // It shouldn't be used for setting canvas size - it will crop drawings outside
+    override var boundsInWindow: IntRect = IntRect.Zero
+        set(value) {
+            field = value
+            boundsEventFilter.bounds = value.toAwtRectangle(density)
+        }
+
     final override var compositionLocalContext: CompositionLocalContext?
         get() = mediator?.compositionLocalContext
         set(value) { mediator?.compositionLocalContext = value }
@@ -101,6 +129,20 @@ internal abstract class DesktopComposeSceneLayer(
     override fun calculateLocalPosition(positionInWindow: IntOffset) =
         positionInWindow // [ComposeScene] is equal to [windowContainer] for the layer.
 
+    protected fun recordDrawBounds(skikoView: SkikoView) =
+        RecordDrawRectSkikoViewDecorator(skikoView) { canvasBoundsInPx ->
+            val currentCanvasOffset = drawBounds.topLeft
+            val drawBoundsInWindow = canvasBoundsInPx.roundToIntRect().translate(currentCanvasOffset)
+            maxDrawInflate = maxInflate(boundsInWindow, drawBoundsInWindow, maxDrawInflate)
+            drawBounds = IntRect(
+                left = boundsInWindow.left - maxDrawInflate.left,
+                top = boundsInWindow.top - maxDrawInflate.top,
+                right = boundsInWindow.right + maxDrawInflate.right,
+                bottom = boundsInWindow.bottom + maxDrawInflate.bottom
+            )
+            onUpdateBounds()
+        }
+
     /**
      * Called when the focus of the window containing main Compose view has changed.
      */
@@ -125,6 +167,12 @@ internal abstract class DesktopComposeSceneLayer(
     open fun onLayersChange() {
     }
 
+    /**
+     * Called when bounds of the layer has been updated.
+     */
+    open fun onUpdateBounds() {
+    }
+
     /**
      * Renders an overlay on the canvas.
      *
@@ -185,7 +233,44 @@ internal abstract class DesktopComposeSceneLayer(
         override fun onMouseEvent(event: MouseEvent): Boolean = !noFocusableLayersAbove
         override fun onKeyEvent(event: KeyEvent): Boolean = !focusable || !noFocusableLayersAbove
     }
+
+    private inner class BoundsEventFilter(
+        var bounds: Rectangle,
+    ) : AwtEventListener {
+        private val MouseEvent.isInBounds: Boolean
+            get() {
+                val localPoint = if (component != windowContainer) {
+                    SwingUtilities.convertPoint(component, point, windowContainer)
+                } else {
+                    point
+                }
+                return bounds.contains(localPoint)
+            }
+
+        override fun onMouseEvent(event: MouseEvent): Boolean {
+            when (event.id) {
+                // Do not filter motion events
+                MouseEvent.MOUSE_MOVED,
+                MouseEvent.MOUSE_ENTERED,
+                MouseEvent.MOUSE_EXITED,
+                MouseEvent.MOUSE_DRAGGED -> return false
+            }
+            return if (event.isInBounds) {
+                false
+            } else {
+                onMouseEventOutside(event)
+                true
+            }
+        }
+    }
 }
 
 private fun MouseEvent.isMainAction() =
     button == MouseEvent.BUTTON1
+
+private fun maxInflate(baseBounds: IntRect, currentBounds: IntRect, maxInflate: IntRect) = IntRect(
+    left = max(baseBounds.left - currentBounds.left, maxInflate.left),
+    top = max(baseBounds.top - currentBounds.top, maxInflate.top),
+    right = max(currentBounds.right - baseBounds.right, maxInflate.right),
+    bottom = max(currentBounds.bottom - baseBounds.bottom, maxInflate.bottom)
+)

compose/ui/ui/src/desktopMain/kotlin/androidx/compose/ui/scene/SwingComposeSceneLayer.desktop.kt

@@ -19,14 +19,16 @@ package androidx.compose.ui.scene
 import androidx.compose.runtime.CompositionContext
 import androidx.compose.ui.awt.toAwtColor
 import androidx.compose.ui.awt.toAwtRectangle
+import androidx.compose.ui.geometry.toRect
 import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.scene.skia.SkiaLayerComponent
 import androidx.compose.ui.scene.skia.SwingSkiaLayerComponent
 import androidx.compose.ui.unit.Density
-import androidx.compose.ui.unit.IntRect
 import androidx.compose.ui.unit.IntSize
 import androidx.compose.ui.unit.LayoutDirection
+import androidx.compose.ui.unit.roundToIntRect
 import androidx.compose.ui.window.density
+import androidx.compose.ui.window.sizeInPx
 import java.awt.Dimension
 import java.awt.Graphics
 import java.awt.event.MouseAdapter
@@ -52,9 +54,7 @@ internal class SwingComposeSceneLayer(
         override fun addNotify() {
             super.addNotify()
             mediator?.onComponentAttached()
-            _boundsInWindow?.let {
-                mediator?.contentComponent?.bounds = it.toAwtRectangle(density)
-            }
+            onUpdateBounds()
         }
 
         override fun paint(g: Graphics) {
@@ -71,20 +71,14 @@ internal class SwingComposeSceneLayer(
         it.background = Color.Transparent.toAwtColor()
         it.size = Dimension(windowContainer.width, windowContainer.height)
         it.addMouseListener(backgroundMouseListener)
-
-        // TODO: Currently it works only with offscreen rendering
-        // TODO: Do not clip this from main scene if layersContainer == main container
-        windowContainer.add(it, JLayeredPane.POPUP_LAYER, 0)
     }
 
     private var containerSize = IntSize.Zero
         set(value) {
             if (field.width != value.width || field.height != value.height) {
                 field = value
                 container.setBounds(0, 0, value.width, value.height)
-                if (_boundsInWindow == null) {
-                    mediator?.contentComponent?.size = container.size
-                }
+                mediator?.contentComponent?.size = container.size
                 mediator?.onChangeComponentSize()
             }
         }
@@ -97,18 +91,6 @@ internal class SwingComposeSceneLayer(
             container.isFocusable = value
         }
 
-    private var _boundsInWindow: IntRect? = null
-    override var boundsInWindow: IntRect
-        get() = _boundsInWindow ?: IntRect.Zero
-        set(value) {
-            _boundsInWindow = value
-            val localBounds = SwingUtilities.convertRectangle(
-                /* source = */ windowContainer,
-                /* aRectangle = */ value.toAwtRectangle(container.density),
-                /* destination = */ container)
-            mediator?.contentComponent?.bounds = localBounds
-        }
-
     override var scrimColor: Color? = null
         set(value) {
             field = value
@@ -117,20 +99,28 @@ internal class SwingComposeSceneLayer(
         }
 
     init {
+        val boundsInPx = windowContainer.sizeInPx.toRect()
+        drawBounds = boundsInPx.roundToIntRect()
         mediator = ComposeSceneMediator(
             container = container,
             windowContext = composeContainer.windowContext,
             exceptionHandler = {
                 composeContainer.exceptionHandler?.onException(it) ?: throw it
             },
             eventListener = eventListener,
+            measureDrawLayerBounds = true,
             coroutineContext = compositionContext.effectCoroutineContext,
             skiaLayerComponentFactory = ::createSkiaLayerComponent,
             composeSceneFactory = ::createComposeScene,
         ).also {
             it.onChangeWindowTransparency(true)
             it.contentComponent.size = container.size
         }
+
+        // TODO: Currently it works only with offscreen rendering
+        // TODO: Do not clip this from main scene if layersContainer == main container
+        windowContainer.add(container, JLayeredPane.POPUP_LAYER, 0)
+
         composeContainer.attachLayer(this)
     }
 
@@ -149,10 +139,20 @@ internal class SwingComposeSceneLayer(
         containerSize = IntSize(windowContainer.width, windowContainer.height)
     }
 
+    override fun onUpdateBounds() {
+        val scaledRectangle = drawBounds.toAwtRectangle(density)
+        val localBounds = SwingUtilities.convertRectangle(
+            /* source = */ windowContainer,
+            /* aRectangle = */ scaledRectangle,
+            /* destination = */ container)
+        mediator?.contentComponent?.bounds = localBounds
+    }
+
     private fun createSkiaLayerComponent(mediator: ComposeSceneMediator): SkiaLayerComponent {
+        val skikoView = recordDrawBounds(mediator)
         return SwingSkiaLayerComponent(
             mediator = mediator,
-            skikoView = mediator,
+            skikoView = skikoView,
             skiaLayerAnalytics = skiaLayerAnalytics
         )
     }