From 2cd716abd7cf8816bd4f507438e1d8c6d115c05a Mon Sep 17 00:00:00 2001 From: Saagar Jha Date: Fri, 26 Jan 2024 11:21:19 -0800 Subject: [PATCH] Improve event handling code * Fix scrolling moving the cursor to (0, 0) on end * Forward multi-click events * Support drag events * Watch for events on child windows as well --- Shared/Messages.swift | 7 +++- Shared/macOSInterface.swift | 68 +++++++++++++++++++++++++++++-- macOS/Events.swift | 43 ++++++++++++++++--- macOS/Local.swift | 53 +++++++++++++++++++++--- visionOS/EventView.swift | 56 ++++++++++++++++++++----- visionOS/Remote.swift | 24 ++++++++++- visionOS/RootWindowView.swift | 1 - visionOS/WindowView.swift | 77 ++++++++++++++++++++++++++++++++++- 8 files changed, 298 insertions(+), 31 deletions(-) diff --git a/Shared/Messages.swift b/Shared/Messages.swift index dd762db..c8868cb 100644 --- a/Shared/Messages.swift +++ b/Shared/Messages.swift @@ -22,7 +22,12 @@ enum Messages: UInt8, CaseIterable { case childWindows case mouseMoved case clicked - case scrolled + case scrollBegan + case scrollChanged + case scrollEnded + case dragBegan + case dragChanged + case dragEnded case typed } diff --git a/Shared/macOSInterface.swift b/Shared/macOSInterface.swift index 14f010d..6129b6a 100644 --- a/Shared/macOSInterface.swift +++ b/Shared/macOSInterface.swift @@ -22,7 +22,12 @@ protocol macOSInterface { func _stopWatchingForChildWindows(parameters: M.StopWatchingForChildWindows.Request) async throws -> M.StopWatchingForChildWindows.Reply func _mouseMoved(parameters: M.MouseMoved.Request) async throws -> M.MouseMoved.Reply func _clicked(parameters: M.Clicked.Request) async throws -> M.Clicked.Reply - func _scrolled(parameters: M.Scrolled.Request) async throws -> M.Scrolled.Reply + func _scrollBegan(parameters: M.ScrollBegan.Request) async throws -> M.ScrollBegan.Reply + func _scrollChanged(parameters: M.ScrollChanged.Request) async throws -> M.ScrollChanged.Reply + func _scrollEnded(parameters: M.ScrollEnded.Request) async throws -> M.ScrollEnded.Reply + func _dragBegan(parameters: M.DragBegan.Request) async throws -> M.DragBegan.Reply + func _dragChanged(parameters: M.DragChanged.Request) async throws -> M.DragChanged.Reply + func _dragEnded(parameters: M.DragEnded.Request) async throws -> M.DragEnded.Reply func _typed(parameters: M.Typed.Request) async throws -> M.Typed.Reply } @@ -143,13 +148,70 @@ enum macOSInterfaceMessages { let windowID: Window.ID let x: CGFloat let y: CGFloat + let count: Int } typealias Reply = SerializableVoid } - struct Scrolled: Message { - static let id = Messages.scrolled + struct ScrollBegan: Message { + static let id = Messages.scrollBegan + + struct Request: Serializable, Codable { + let windowID: Window.ID + } + + typealias Reply = SerializableVoid + } + + struct ScrollChanged: Message { + static let id = Messages.scrollChanged + + struct Request: Serializable, Codable { + let windowID: Window.ID + let x: CGFloat + let y: CGFloat + } + + typealias Reply = SerializableVoid + } + + struct ScrollEnded: Message { + static let id = Messages.scrollEnded + + struct Request: Serializable, Codable { + let windowID: Window.ID + } + + typealias Reply = SerializableVoid + } + + struct DragBegan: Message { + static let id = Messages.dragBegan + + struct Request: Serializable, Codable { + let windowID: Window.ID + let x: CGFloat + let y: CGFloat + } + + typealias Reply = SerializableVoid + } + + struct DragChanged: Message { + static let id = Messages.dragChanged + + struct Request: Serializable, Codable { + let windowID: Window.ID + let x: CGFloat + let y: CGFloat + } + + typealias Reply = SerializableVoid + } + + struct DragEnded: Message { + static let id = Messages.dragEnded struct Request: Serializable, Codable { let windowID: Window.ID diff --git a/macOS/Events.swift b/macOS/Events.swift index 3ccfebd..f363db6 100644 --- a/macOS/Events.swift +++ b/macOS/Events.swift @@ -13,15 +13,46 @@ actor EventDispatcher { event.post(tap: .cghidEventTap) } - func injectClick(at location: NSPoint) { - let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: location, mouseButton: .left)! - let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: location, mouseButton: .left)! - down.post(tap: .cghidEventTap) - up.post(tap: .cghidEventTap) + func injectClick(at location: NSPoint, count: Int) { + for direction in [.leftMouseDown, .leftMouseUp] as [CGEventType] { + let event = CGEvent(mouseEventSource: nil, mouseType: direction, mouseCursorPosition: location, mouseButton: .left)! + event.setIntegerValueField(.mouseEventClickState, value: Int64(count)) + event.post(tap: .cghidEventTap) + } } - func injectScroll(translationX: CGFloat, translationY: CGFloat) { + func injectScrollBegan() { + let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: 0, wheel2: 0, wheel3: 0)! + event.setIntegerValueField(.scrollWheelEventScrollPhase, value: Int64(CGGesturePhase.began.rawValue)) + event.post(tap: .cghidEventTap) + + } + + func injectScrollChanged(translationX: CGFloat, translationY: CGFloat) { let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: Int32(translationY), wheel2: Int32(translationX), wheel3: 0)! + event.setIntegerValueField(.scrollWheelEventScrollCount, value: 1) + event.setIntegerValueField(.scrollWheelEventScrollPhase, value: Int64(CGGesturePhase.changed.rawValue)) + event.post(tap: .cghidEventTap) + } + + func injectScrollEnded() { + let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: 0, wheel2: 0, wheel3: 0)! + event.setIntegerValueField(.scrollWheelEventScrollPhase, value: Int64(CGGesturePhase.ended.rawValue)) + event.post(tap: .cghidEventTap) + } + + func injectDragBegan(at location: NSPoint) { + let event = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: location, mouseButton: .left)! + event.post(tap: .cghidEventTap) + } + + func injectDragChanged(to location: NSPoint) { + let event = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDragged, mouseCursorPosition: location, mouseButton: .left)! + event.post(tap: .cghidEventTap) + } + + func injectDragEnded(at location: NSPoint) { + let event = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: location, mouseButton: .left)! event.post(tap: .cghidEventTap) } diff --git a/macOS/Local.swift b/macOS/Local.swift index 11be427..5111d38 100644 --- a/macOS/Local.swift +++ b/macOS/Local.swift @@ -83,8 +83,18 @@ class Local: LocalInterface, macOSInterface { return try await _mouseMoved(parameters: .decode(data)).encode() case .clicked: return try await _clicked(parameters: .decode(data)).encode() - case .scrolled: - return try await _scrolled(parameters: .decode(data)).encode() + case .scrollBegan: + return try await _scrollBegan(parameters: .decode(data)).encode() + case .scrollChanged: + return try await _scrollChanged(parameters: .decode(data)).encode() + case .scrollEnded: + return try await _scrollEnded(parameters: .decode(data)).encode() + case .dragBegan: + return try await _dragBegan(parameters: .decode(data)).encode() + case .dragChanged: + return try await _dragChanged(parameters: .decode(data)).encode() + case .dragEnded: + return try await _dragEnded(parameters: .decode(data)).encode() case .typed: return try await _typed(parameters: .decode(data)).encode() default: @@ -170,12 +180,45 @@ class Local: LocalInterface, macOSInterface { func _clicked(parameters: M.Clicked.Request) async throws -> M.Clicked.Reply { let window = try await screenRecorder.lookup(windowID: parameters.windowID)! - await eventDispatcher.injectClick(at: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y)) + await eventDispatcher.injectClick(at: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y), count: parameters.count) return .init() } - func _scrolled(parameters: M.Scrolled.Request) async throws -> M.Scrolled.Reply { - await eventDispatcher.injectScroll(translationX: parameters.x, translationY: parameters.y) + func _scrollBegan(parameters: M.ScrollBegan.Request) async throws -> M.ScrollBegan.Reply { + await eventDispatcher.injectScrollBegan() + + return .init() + } + + func _scrollChanged(parameters: M.ScrollChanged.Request) async throws -> M.ScrollChanged.Reply { + await eventDispatcher.injectScrollChanged(translationX: parameters.x, translationY: parameters.y) + + return .init() + } + + func _scrollEnded(parameters: M.ScrollEnded.Request) async throws -> M.ScrollEnded.Reply { + await eventDispatcher.injectScrollEnded() + + return .init() + } + + func _dragBegan(parameters: M.DragBegan.Request) async throws -> M.DragBegan.Reply { + let window = try await screenRecorder.lookup(windowID: parameters.windowID)! + await eventDispatcher.injectDragBegan(at: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y)) + + return .init() + } + + func _dragChanged(parameters: M.DragChanged.Request) async throws -> M.DragChanged.Reply { + let window = try await screenRecorder.lookup(windowID: parameters.windowID)! + await eventDispatcher.injectDragChanged(to: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y)) + + return .init() + } + + func _dragEnded(parameters: M.DragEnded.Request) async throws -> M.DragEnded.Reply { + let window = try await screenRecorder.lookup(windowID: parameters.windowID)! + await eventDispatcher.injectDragEnded(at: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y)) return .init() } diff --git a/visionOS/EventView.swift b/visionOS/EventView.swift index d74068d..0530ad0 100644 --- a/visionOS/EventView.swift +++ b/visionOS/EventView.swift @@ -39,28 +39,61 @@ struct EventView: UIViewRepresentable { let view = KeyView() let coordinator: Coordinator + enum ScrollEvent { + case began + case changed(CGPoint) + case ended + } + + enum DragEvent { + case began(CGPoint) + case changed(CGPoint) + case ended(CGPoint) + } + init() { coordinator = .init(view: view) } class Coordinator { let view: UIView - let (hoverStream, hoverContinuation) = AsyncStream.makeStream(of: CGPoint.self) - let (panStream, panContinuation) = AsyncStream.makeStream(of: CGPoint.self) + let (scrollStream, scrollContinuation) = AsyncStream.makeStream(of: ScrollEvent.self) + let (dragStream, dragContinuation) = AsyncStream.makeStream(of: DragEvent.self) init(view: UIView) { self.view = view } @objc - func hover(_ sender: UIHoverGestureRecognizer) { - hoverContinuation.yield(sender.location(in: view)) + func scroll(_ sender: UIPanGestureRecognizer) { + switch sender.state { + case .began: + scrollContinuation.yield(.began) + case .changed: + scrollContinuation.yield(.changed(sender.translation(in: view))) + sender.setTranslation(.zero, in: view) + case .ended: + scrollContinuation.yield(.ended) + default: + return + } } @objc func pan(_ sender: UIPanGestureRecognizer) { - panContinuation.yield(sender.translation(in: view)) - sender.setTranslation(.zero, in: view) + var position = sender.location(in: view) + position.x /= view.frame.width + position.y /= view.frame.height + switch sender.state { + case .began: + dragContinuation.yield(.began(position)) + case .changed: + dragContinuation.yield(.changed(position)) + case .ended: + dragContinuation.yield(.ended(position)) + default: + return + } } } @@ -72,12 +105,13 @@ struct EventView: UIViewRepresentable { } func makeCoordinator() -> Coordinator { - let hoverGestureRecognizer = UIHoverGestureRecognizer(target: coordinator, action: #selector(Coordinator.hover(_:))) - view.addGestureRecognizer(hoverGestureRecognizer) + let scrollGestureRecognizer = UIPanGestureRecognizer(target: coordinator, action: #selector(Coordinator.scroll(_:))) + scrollGestureRecognizer.allowedScrollTypesMask = .all + scrollGestureRecognizer.allowedTouchTypes = [] + view.addGestureRecognizer(scrollGestureRecognizer) - let panGestureRecognizer = UIPanGestureRecognizer(target: coordinator, action: #selector(Coordinator.pan(_:))) - panGestureRecognizer.allowedScrollTypesMask = .all - view.addGestureRecognizer(panGestureRecognizer) + let dragGestureRecognizer = UIPanGestureRecognizer(target: coordinator, action: #selector(Coordinator.pan(_:))) + view.addGestureRecognizer(dragGestureRecognizer) return coordinator } } diff --git a/visionOS/Remote.swift b/visionOS/Remote.swift index 02a3166..10aca48 100644 --- a/visionOS/Remote.swift +++ b/visionOS/Remote.swift @@ -102,8 +102,28 @@ struct Remote: macOSInterface { try await M.Clicked.send(parameters, through: connection) } - func _scrolled(parameters: M.Scrolled.Request) async throws -> M.Scrolled.Reply { - try await M.Scrolled.send(parameters, through: connection) + func _scrollBegan(parameters: M.ScrollBegan.Request) async throws -> M.ScrollBegan.Reply { + try await M.ScrollBegan.send(parameters, through: connection) + } + + func _scrollChanged(parameters: M.ScrollChanged.Request) async throws -> M.ScrollChanged.Reply { + try await M.ScrollChanged.send(parameters, through: connection) + } + + func _scrollEnded(parameters: M.ScrollEnded.Request) async throws -> M.ScrollEnded.Reply { + try await M.ScrollEnded.send(parameters, through: connection) + } + + func _dragBegan(parameters: M.DragBegan.Request) async throws -> M.DragBegan.Reply { + try await M.DragBegan.send(parameters, through: connection) + } + + func _dragChanged(parameters: M.DragChanged.Request) async throws -> M.DragChanged.Reply { + try await M.DragChanged.send(parameters, through: connection) + } + + func _dragEnded(parameters: M.DragEnded.Request) async throws -> M.DragEnded.Reply { + try await M.DragEnded.send(parameters, through: connection) } func _typed(parameters: M.Typed.Request) async throws -> M.Typed.Reply { diff --git a/visionOS/RootWindowView.swift b/visionOS/RootWindowView.swift index 9596b61..b133340 100644 --- a/visionOS/RootWindowView.swift +++ b/visionOS/RootWindowView.swift @@ -12,7 +12,6 @@ struct RootWindowView: View { let remote: Remote let window: Window - let eventView = EventView() @State var children = [Window]() diff --git a/visionOS/WindowView.swift b/visionOS/WindowView.swift index 57cfee6..13557c7 100644 --- a/visionOS/WindowView.swift +++ b/visionOS/WindowView.swift @@ -7,19 +7,85 @@ import SwiftUI -struct WindowView: View { +struct WindowView: View, Equatable { let remote: Remote let window: Window @State var frame: Frame? + let eventView = EventView() let decoder = VideoDecoder() var body: some View { Group { if let frame { - FrameView(frame: frame) + GeometryReader { geometry in + FrameView(frame: frame) + .overlay { + eventView + .task { + do { + for await event in eventView.coordinator.scrollStream { + switch event { + case .began: + _ = try await remote._scrollBegan(parameters: .init(windowID: window.windowID)) + case .changed(let translation): + _ = try await remote._scrollChanged(parameters: .init(windowID: window.windowID, x: translation.x, y: translation.y)) + case .ended: + _ = try await remote._scrollEnded(parameters: .init(windowID: window.windowID)) + } + } + } catch {} + } + .task { + do { + for await event in eventView.coordinator.dragStream { + switch event { + case .began(let translation): + _ = try await remote._dragBegan(parameters: .init(windowID: window.windowID, x: translation.x, y: translation.y)) + case .changed(let translation): + _ = try await remote._dragChanged(parameters: .init(windowID: window.windowID, x: translation.x, y: translation.y)) + case .ended(let translation): + _ = try await remote._dragEnded(parameters: .init(windowID: window.windowID, x: translation.x, y: translation.y)) + } + } + } catch {} + } + .task { + do { + for await (key, down) in eventView.view.keyStream { + _ = try await remote._typed(parameters: .init(windowID: window.windowID, key: key, down: down)) + } + } catch {} + } + } + .onTapGesture(count: 3) { + click(at: $0, in: geometry.size, count: 3) + } + // FIXME: This isn't detected + .onTapGesture(count: 2) { + click(at: $0, in: geometry.size, count: 2) + } + .onTapGesture { + click(at: $0, in: geometry.size, count: 1) + } + .onContinuousHover(coordinateSpace: .local) { + switch $0 { + case .active(let location): + Task { + do { + _ = try await remote._mouseMoved(parameters: .init(windowID: window.windowID, x: location.x / geometry.size.width, y: location.y / geometry.size.height)) + } catch {} + } + default: + break + } + } + .onAppear { + eventView.view.becomeFirstResponder() + } + } } else { Text("Loading…") } @@ -32,4 +98,11 @@ struct WindowView: View { } catch {} } } + + func click(at location: CGPoint, in size: CGSize, count: Int) { + eventView.view.becomeFirstResponder() + Task { + _ = try await remote._clicked(parameters: .init(windowID: window.windowID, x: location.x / size.width, y: location.y / size.height, count: count)) + } + } }