Improve event handling code

* Fix scrolling moving the cursor to (0, 0) on end * Forward multi-click events * Support drag events * Watch for events on child windows as well
saagarjha · Jan 26, 2024 · 2cd716a · 2cd716a
1 parent 6fdb2f3
commit 2cd716a
Show file tree

Hide file tree

Showing 8 changed files with 298 additions and 31 deletions.
diff --git a/Shared/Messages.swift b/Shared/Messages.swift
@@ -22,7 +22,12 @@ enum Messages: UInt8, CaseIterable {
 	case childWindows
 	case mouseMoved
 	case clicked
-	case scrolled
+	case scrollBegan
+	case scrollChanged
+	case scrollEnded
+	case dragBegan
+	case dragChanged
+	case dragEnded
 	case typed
 }
 

diff --git a/Shared/macOSInterface.swift b/Shared/macOSInterface.swift
@@ -22,7 +22,12 @@ protocol macOSInterface {
 	func _stopWatchingForChildWindows(parameters: M.StopWatchingForChildWindows.Request) async throws -> M.StopWatchingForChildWindows.Reply
 	func _mouseMoved(parameters: M.MouseMoved.Request) async throws -> M.MouseMoved.Reply
 	func _clicked(parameters: M.Clicked.Request) async throws -> M.Clicked.Reply
-	func _scrolled(parameters: M.Scrolled.Request) async throws -> M.Scrolled.Reply
+	func _scrollBegan(parameters: M.ScrollBegan.Request) async throws -> M.ScrollBegan.Reply
+	func _scrollChanged(parameters: M.ScrollChanged.Request) async throws -> M.ScrollChanged.Reply
+	func _scrollEnded(parameters: M.ScrollEnded.Request) async throws -> M.ScrollEnded.Reply
+	func _dragBegan(parameters: M.DragBegan.Request) async throws -> M.DragBegan.Reply
+	func _dragChanged(parameters: M.DragChanged.Request) async throws -> M.DragChanged.Reply
+	func _dragEnded(parameters: M.DragEnded.Request) async throws -> M.DragEnded.Reply
 	func _typed(parameters: M.Typed.Request) async throws -> M.Typed.Reply
 }
 
@@ -143,13 +148,70 @@ enum macOSInterfaceMessages {
 			let windowID: Window.ID
 			let x: CGFloat
 			let y: CGFloat
+			let count: Int
 		}
 
 		typealias Reply = SerializableVoid
 	}
 
-	struct Scrolled: Message {
-		static let id = Messages.scrolled
+	struct ScrollBegan: Message {
+		static let id = Messages.scrollBegan
+
+		struct Request: Serializable, Codable {
+			let windowID: Window.ID
+		}
+
+		typealias Reply = SerializableVoid
+	}
+
+	struct ScrollChanged: Message {
+		static let id = Messages.scrollChanged
+
+		struct Request: Serializable, Codable {
+			let windowID: Window.ID
+			let x: CGFloat
+			let y: CGFloat
+		}
+
+		typealias Reply = SerializableVoid
+	}
+
+	struct ScrollEnded: Message {
+		static let id = Messages.scrollEnded
+
+		struct Request: Serializable, Codable {
+			let windowID: Window.ID
+		}
+
+		typealias Reply = SerializableVoid
+	}
+
+	struct DragBegan: Message {
+		static let id = Messages.dragBegan
+
+		struct Request: Serializable, Codable {
+			let windowID: Window.ID
+			let x: CGFloat
+			let y: CGFloat
+		}
+
+		typealias Reply = SerializableVoid
+	}
+
+	struct DragChanged: Message {
+		static let id = Messages.dragChanged
+
+		struct Request: Serializable, Codable {
+			let windowID: Window.ID
+			let x: CGFloat
+			let y: CGFloat
+		}
+
+		typealias Reply = SerializableVoid
+	}
+
+	struct DragEnded: Message {
+		static let id = Messages.dragEnded
 
 		struct Request: Serializable, Codable {
 			let windowID: Window.ID

diff --git a/macOS/Events.swift b/macOS/Events.swift
@@ -13,15 +13,46 @@ actor EventDispatcher {
 		event.post(tap: .cghidEventTap)
 	}
 
-	func injectClick(at location: NSPoint) {
-		let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: location, mouseButton: .left)!
-		let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: location, mouseButton: .left)!
-		down.post(tap: .cghidEventTap)
-		up.post(tap: .cghidEventTap)
+	func injectClick(at location: NSPoint, count: Int) {
+		for direction in [.leftMouseDown, .leftMouseUp] as [CGEventType] {
+			let event = CGEvent(mouseEventSource: nil, mouseType: direction, mouseCursorPosition: location, mouseButton: .left)!
+			event.setIntegerValueField(.mouseEventClickState, value: Int64(count))
+			event.post(tap: .cghidEventTap)
+		}
 	}
 
-	func injectScroll(translationX: CGFloat, translationY: CGFloat) {
+	func injectScrollBegan() {
+		let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: 0, wheel2: 0, wheel3: 0)!
+		event.setIntegerValueField(.scrollWheelEventScrollPhase, value: Int64(CGGesturePhase.began.rawValue))
+		event.post(tap: .cghidEventTap)
+
+	}
+
+	func injectScrollChanged(translationX: CGFloat, translationY: CGFloat) {
 		let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: Int32(translationY), wheel2: Int32(translationX), wheel3: 0)!
+		event.setIntegerValueField(.scrollWheelEventScrollCount, value: 1)
+		event.setIntegerValueField(.scrollWheelEventScrollPhase, value: Int64(CGGesturePhase.changed.rawValue))
+		event.post(tap: .cghidEventTap)
+	}
+
+	func injectScrollEnded() {
+		let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: 0, wheel2: 0, wheel3: 0)!
+		event.setIntegerValueField(.scrollWheelEventScrollPhase, value: Int64(CGGesturePhase.ended.rawValue))
+		event.post(tap: .cghidEventTap)
+	}
+
+	func injectDragBegan(at location: NSPoint) {
+		let event = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: location, mouseButton: .left)!
+		event.post(tap: .cghidEventTap)
+	}
+
+	func injectDragChanged(to location: NSPoint) {
+		let event = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDragged, mouseCursorPosition: location, mouseButton: .left)!
+		event.post(tap: .cghidEventTap)
+	}
+
+	func injectDragEnded(at location: NSPoint) {
+		let event = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: location, mouseButton: .left)!
 		event.post(tap: .cghidEventTap)
 	}
 

diff --git a/macOS/Local.swift b/macOS/Local.swift
@@ -83,8 +83,18 @@ class Local: LocalInterface, macOSInterface {
 				return try await _mouseMoved(parameters: .decode(data)).encode()
 			case .clicked:
 				return try await _clicked(parameters: .decode(data)).encode()
-			case .scrolled:
-				return try await _scrolled(parameters: .decode(data)).encode()
+			case .scrollBegan:
+				return try await _scrollBegan(parameters: .decode(data)).encode()
+			case .scrollChanged:
+				return try await _scrollChanged(parameters: .decode(data)).encode()
+			case .scrollEnded:
+				return try await _scrollEnded(parameters: .decode(data)).encode()
+			case .dragBegan:
+				return try await _dragBegan(parameters: .decode(data)).encode()
+			case .dragChanged:
+				return try await _dragChanged(parameters: .decode(data)).encode()
+			case .dragEnded:
+				return try await _dragEnded(parameters: .decode(data)).encode()
 			case .typed:
 				return try await _typed(parameters: .decode(data)).encode()
 			default:
@@ -170,12 +180,45 @@ class Local: LocalInterface, macOSInterface {
 
 	func _clicked(parameters: M.Clicked.Request) async throws -> M.Clicked.Reply {
 		let window = try await screenRecorder.lookup(windowID: parameters.windowID)!
-		await eventDispatcher.injectClick(at: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y))
+		await eventDispatcher.injectClick(at: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y), count: parameters.count)
 		return .init()
 	}
 
-	func _scrolled(parameters: M.Scrolled.Request) async throws -> M.Scrolled.Reply {
-		await eventDispatcher.injectScroll(translationX: parameters.x, translationY: parameters.y)
+	func _scrollBegan(parameters: M.ScrollBegan.Request) async throws -> M.ScrollBegan.Reply {
+		await eventDispatcher.injectScrollBegan()
+
+		return .init()
+	}
+
+	func _scrollChanged(parameters: M.ScrollChanged.Request) async throws -> M.ScrollChanged.Reply {
+		await eventDispatcher.injectScrollChanged(translationX: parameters.x, translationY: parameters.y)
+
+		return .init()
+	}
+
+	func _scrollEnded(parameters: M.ScrollEnded.Request) async throws -> M.ScrollEnded.Reply {
+		await eventDispatcher.injectScrollEnded()
+
+		return .init()
+	}
+
+	func _dragBegan(parameters: M.DragBegan.Request) async throws -> M.DragBegan.Reply {
+		let window = try await screenRecorder.lookup(windowID: parameters.windowID)!
+		await eventDispatcher.injectDragBegan(at: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y))
+
+		return .init()
+	}
+
+	func _dragChanged(parameters: M.DragChanged.Request) async throws -> M.DragChanged.Reply {
+		let window = try await screenRecorder.lookup(windowID: parameters.windowID)!
+		await eventDispatcher.injectDragChanged(to: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y))
+
+		return .init()
+	}
+
+	func _dragEnded(parameters: M.DragEnded.Request) async throws -> M.DragEnded.Reply {
+		let window = try await screenRecorder.lookup(windowID: parameters.windowID)!
+		await eventDispatcher.injectDragEnded(at: .init(x: window.frame.minX + window.frame.width * parameters.x, y: window.frame.minY + window.frame.height * parameters.y))
 
 		return .init()
 	}

diff --git a/visionOS/EventView.swift b/visionOS/EventView.swift
@@ -39,28 +39,61 @@ struct EventView: UIViewRepresentable {
 	let view = KeyView()
 	let coordinator: Coordinator
 
+	enum ScrollEvent {
+		case began
+		case changed(CGPoint)
+		case ended
+	}
+
+	enum DragEvent {
+		case began(CGPoint)
+		case changed(CGPoint)
+		case ended(CGPoint)
+	}
+
 	init() {
 		coordinator = .init(view: view)
 	}
 
 	class Coordinator {
 		let view: UIView
-		let (hoverStream, hoverContinuation) = AsyncStream.makeStream(of: CGPoint.self)
-		let (panStream, panContinuation) = AsyncStream.makeStream(of: CGPoint.self)
+		let (scrollStream, scrollContinuation) = AsyncStream.makeStream(of: ScrollEvent.self)
+		let (dragStream, dragContinuation) = AsyncStream.makeStream(of: DragEvent.self)
 
 		init(view: UIView) {
 			self.view = view
 		}
 
 		@objc
-		func hover(_ sender: UIHoverGestureRecognizer) {
-			hoverContinuation.yield(sender.location(in: view))
+		func scroll(_ sender: UIPanGestureRecognizer) {
+			switch sender.state {
+				case .began:
+					scrollContinuation.yield(.began)
+				case .changed:
+					scrollContinuation.yield(.changed(sender.translation(in: view)))
+					sender.setTranslation(.zero, in: view)
+				case .ended:
+					scrollContinuation.yield(.ended)
+				default:
+					return
+			}
 		}
 
 		@objc
 		func pan(_ sender: UIPanGestureRecognizer) {
-			panContinuation.yield(sender.translation(in: view))
-			sender.setTranslation(.zero, in: view)
+			var position = sender.location(in: view)
+			position.x /= view.frame.width
+			position.y /= view.frame.height
+			switch sender.state {
+				case .began:
+					dragContinuation.yield(.began(position))
+				case .changed:
+					dragContinuation.yield(.changed(position))
+				case .ended:
+					dragContinuation.yield(.ended(position))
+				default:
+					return
+			}
 		}
 	}
 
@@ -72,12 +105,13 @@ struct EventView: UIViewRepresentable {
 	}
 
 	func makeCoordinator() -> Coordinator {
-		let hoverGestureRecognizer = UIHoverGestureRecognizer(target: coordinator, action: #selector(Coordinator.hover(_:)))
-		view.addGestureRecognizer(hoverGestureRecognizer)
+		let scrollGestureRecognizer = UIPanGestureRecognizer(target: coordinator, action: #selector(Coordinator.scroll(_:)))
+		scrollGestureRecognizer.allowedScrollTypesMask = .all
+		scrollGestureRecognizer.allowedTouchTypes = []
+		view.addGestureRecognizer(scrollGestureRecognizer)
 
-		let panGestureRecognizer = UIPanGestureRecognizer(target: coordinator, action: #selector(Coordinator.pan(_:)))
-		panGestureRecognizer.allowedScrollTypesMask = .all
-		view.addGestureRecognizer(panGestureRecognizer)
+		let dragGestureRecognizer = UIPanGestureRecognizer(target: coordinator, action: #selector(Coordinator.pan(_:)))
+		view.addGestureRecognizer(dragGestureRecognizer)
 		return coordinator
 	}
 }

diff --git a/visionOS/Remote.swift b/visionOS/Remote.swift
@@ -102,8 +102,28 @@ struct Remote: macOSInterface {
 		try await M.Clicked.send(parameters, through: connection)
 	}
 
-	func _scrolled(parameters: M.Scrolled.Request) async throws -> M.Scrolled.Reply {
-		try await M.Scrolled.send(parameters, through: connection)
+	func _scrollBegan(parameters: M.ScrollBegan.Request) async throws -> M.ScrollBegan.Reply {
+		try await M.ScrollBegan.send(parameters, through: connection)
+	}
+
+	func _scrollChanged(parameters: M.ScrollChanged.Request) async throws -> M.ScrollChanged.Reply {
+		try await M.ScrollChanged.send(parameters, through: connection)
+	}
+
+	func _scrollEnded(parameters: M.ScrollEnded.Request) async throws -> M.ScrollEnded.Reply {
+		try await M.ScrollEnded.send(parameters, through: connection)
+	}
+
+	func _dragBegan(parameters: M.DragBegan.Request) async throws -> M.DragBegan.Reply {
+		try await M.DragBegan.send(parameters, through: connection)
+	}
+
+	func _dragChanged(parameters: M.DragChanged.Request) async throws -> M.DragChanged.Reply {
+		try await M.DragChanged.send(parameters, through: connection)
+	}
+
+	func _dragEnded(parameters: M.DragEnded.Request) async throws -> M.DragEnded.Reply {
+		try await M.DragEnded.send(parameters, through: connection)
 	}
 
 	func _typed(parameters: M.Typed.Request) async throws -> M.Typed.Reply {

diff --git a/visionOS/RootWindowView.swift b/visionOS/RootWindowView.swift
@@ -12,7 +12,6 @@ struct RootWindowView: View {
 	let remote: Remote
 	let window: Window
 
-	let eventView = EventView()
 
 	@State
 	var children = [Window]()