Improve Frontend -> Backend user input system (#348)

Includes refactor that sends coordinates of the document viewports to the backend so input is sent relative to the application window
Closes #124
Fixes #291

* Improve Frontend -> Backend user input system

* Code review changes

* More code review changes

* Fix TS error

Author: Keavon

editor/src/document/document_file.rs

@@ -284,7 +284,7 @@ impl MessageHandler<DocumentMessage, &InputPreprocessor> for DocumentMessageHand
 			}
 			CommitTransaction => self.document_backup = None,
 			ExportDocument => {
-				let bbox = self.document.visible_layers_bounding_box().unwrap_or([DVec2::ZERO, ipp.viewport_size]);
+				let bbox = self.document.visible_layers_bounding_box().unwrap_or([DVec2::ZERO, ipp.viewport_bounds.size()]);
 				let size = bbox[1] - bbox[0];
 				let name = match self.name.ends_with(FILE_SAVE_SUFFIX) {
 					true => self.name.clone().replace(FILE_SAVE_SUFFIX, FILE_EXPORT_SUFFIX),

editor/src/document/movement_handler.rs

@@ -5,7 +5,7 @@ use super::LayerData;
 use crate::message_prelude::*;
 use crate::{
 	consts::{VIEWPORT_SCROLL_RATE, VIEWPORT_ZOOM_LEVELS, VIEWPORT_ZOOM_MOUSE_RATE, VIEWPORT_ZOOM_SCALE_MAX, VIEWPORT_ZOOM_SCALE_MIN, VIEWPORT_ZOOM_WHEEL_RATE},
-	input::{mouse::ViewportPosition, InputPreprocessor},
+	input::{mouse::ViewportBounds, mouse::ViewportPosition, InputPreprocessor},
 };
 use glam::DVec2;
 use graphene::document::Document;
@@ -42,8 +42,8 @@ pub struct MovementMessageHandler {
 }
 
 impl MovementMessageHandler {
-	fn create_document_transform_from_layerdata(&self, layerdata: &LayerData, viewport_size: &ViewportPosition, responses: &mut VecDeque<Message>) {
-		let half_viewport = *viewport_size / 2.;
+	fn create_document_transform_from_layerdata(&self, layerdata: &LayerData, viewport_bounds: &ViewportBounds, responses: &mut VecDeque<Message>) {
+		let half_viewport = viewport_bounds.size() / 2.;
 		let scaled_half_viewport = half_viewport / layerdata.scale;
 		responses.push_back(
 			DocumentOperation::SetLayerTransform {
@@ -89,10 +89,10 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 					let transformed_delta = document.root.transform.inverse().transform_vector2(delta);
 
 					layerdata.translation += transformed_delta;
-					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				}
 				if self.rotating {
-					let half_viewport = ipp.viewport_size / 2.;
+					let half_viewport = ipp.viewport_bounds.size() / 2.;
 					let rotation = {
 						let start_vec = self.mouse_pos - half_viewport;
 						let end_vec = ipp.mouse.position - half_viewport;
@@ -109,7 +109,7 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 						}
 						.into(),
 					);
-					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				}
 				if self.zooming {
 					let difference = self.mouse_pos.y as f64 - ipp.mouse.position.y as f64;
@@ -118,44 +118,44 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 					let new = (layerdata.scale * amount).clamp(VIEWPORT_ZOOM_SCALE_MIN, VIEWPORT_ZOOM_SCALE_MAX);
 					layerdata.scale = new;
 					responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				}
 				self.mouse_pos = ipp.mouse.position;
 			}
 			SetCanvasZoom(new) => {
 				layerdata.scale = new.clamp(VIEWPORT_ZOOM_SCALE_MIN, VIEWPORT_ZOOM_SCALE_MAX);
 				responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			IncreaseCanvasZoom => {
 				layerdata.scale = *VIEWPORT_ZOOM_LEVELS.iter().find(|scale| **scale > layerdata.scale).unwrap_or(&layerdata.scale);
 				responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			DecreaseCanvasZoom => {
 				layerdata.scale = *VIEWPORT_ZOOM_LEVELS.iter().rev().find(|scale| **scale < layerdata.scale).unwrap_or(&layerdata.scale);
 				responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			WheelCanvasZoom => {
 				let scroll = ipp.mouse.scroll_delta.scroll_delta();
 				let mouse = ipp.mouse.position;
-				let viewport_size = ipp.viewport_size;
+				let viewport_bounds = ipp.viewport_bounds.size();
 				let mut zoom_factor = 1. + scroll.abs() * VIEWPORT_ZOOM_WHEEL_RATE;
 				if ipp.mouse.scroll_delta.y > 0 {
 					zoom_factor = 1. / zoom_factor
 				};
-				let new_viewport_size = viewport_size * (1. / zoom_factor);
-				let delta_size = viewport_size - new_viewport_size;
-				let mouse_percent = mouse / viewport_size;
+				let new_viewport_bounds = viewport_bounds * (1. / zoom_factor);
+				let delta_size = viewport_bounds - new_viewport_bounds;
+				let mouse_percent = mouse / viewport_bounds;
 				let delta = (delta_size * -2.) * (mouse_percent - DVec2::splat(0.5));
 
 				let transformed_delta = document.root.transform.inverse().transform_vector2(delta);
 				let new = (layerdata.scale * zoom_factor).clamp(VIEWPORT_ZOOM_SCALE_MIN, VIEWPORT_ZOOM_SCALE_MAX);
 				layerdata.scale = new;
 				layerdata.translation += transformed_delta;
 				responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			WheelCanvasTranslate { use_y_as_x } => {
 				let delta = match use_y_as_x {
@@ -164,19 +164,19 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 				} * VIEWPORT_SCROLL_RATE;
 				let transformed_delta = document.root.transform.inverse().transform_vector2(delta);
 				layerdata.translation += transformed_delta;
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			SetCanvasRotation(new) => {
 				layerdata.rotation = new;
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				responses.push_back(FrontendMessage::SetCanvasRotation { new_radians: new }.into());
 			}
 			ZoomCanvasToFitAll => {
 				if let Some([pos1, pos2]) = document.visible_layers_bounding_box() {
 					let pos1 = document.root.transform.inverse().transform_point2(pos1);
 					let pos2 = document.root.transform.inverse().transform_point2(pos2);
 					let v1 = document.root.transform.inverse().transform_point2(DVec2::ZERO);
-					let v2 = document.root.transform.inverse().transform_point2(ipp.viewport_size);
+					let v2 = document.root.transform.inverse().transform_point2(ipp.viewport_bounds.size());
 
 					let center = v1.lerp(v2, 0.5) - pos1.lerp(pos2, 0.5);
 					let size = (pos2 - pos1) / (v2 - v1);
@@ -186,7 +186,7 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 					layerdata.translation += center;
 					layerdata.scale *= new_scale;
 					responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				}
 			}
 		}

editor/src/input/input_preprocessor.rs

@@ -1,7 +1,7 @@
 use std::usize;
 
 use super::keyboard::{Key, KeyStates};
-use super::mouse::{MouseKeys, MouseState, ScrollDelta, ViewportPosition};
+use super::mouse::{EditorMouseState, MouseKeys, MouseState, ViewportBounds};
 use crate::message_prelude::*;
 use bitflags::bitflags;
 
@@ -11,13 +11,13 @@ pub use graphene::DocumentResponse;
 #[impl_message(Message, InputPreprocessor)]
 #[derive(PartialEq, Clone, Debug)]
 pub enum InputPreprocessorMessage {
-	MouseDown(MouseState, ModifierKeys),
-	MouseUp(MouseState, ModifierKeys),
-	MouseMove(ViewportPosition, ModifierKeys),
-	MouseScroll(ScrollDelta, ModifierKeys),
+	MouseDown(EditorMouseState, ModifierKeys),
+	MouseUp(EditorMouseState, ModifierKeys),
+	MouseMove(EditorMouseState, ModifierKeys),
+	MouseScroll(EditorMouseState, ModifierKeys),
 	KeyUp(Key, ModifierKeys),
 	KeyDown(Key, ModifierKeys),
-	ViewportResize(ViewportPosition),
+	BoundsOfViewports(Vec<ViewportBounds>),
 }
 
 bitflags! {
@@ -34,7 +34,7 @@ bitflags! {
 pub struct InputPreprocessor {
 	pub keyboard: KeyStates,
 	pub mouse: MouseState,
-	pub viewport_size: ViewportPosition,
+	pub viewport_bounds: ViewportBounds,
 }
 
 enum KeyPosition {
@@ -45,28 +45,43 @@ enum KeyPosition {
 impl MessageHandler<InputPreprocessorMessage, ()> for InputPreprocessor {
 	fn process_action(&mut self, message: InputPreprocessorMessage, _data: (), responses: &mut VecDeque<Message>) {
 		match message {
-			InputPreprocessorMessage::MouseMove(pos, modifier_keys) => {
+			InputPreprocessorMessage::MouseMove(editor_mouse_state, modifier_keys) => {
 				self.handle_modifier_keys(modifier_keys, responses);
-				self.mouse.position = pos;
+
+				let mouse_state = editor_mouse_state.to_mouse_state(&self.viewport_bounds);
+				self.mouse.position = mouse_state.position;
+
 				responses.push_back(InputMapperMessage::PointerMove.into());
 			}
-			InputPreprocessorMessage::MouseScroll(delta, modifier_keys) => {
-				self.handle_modifier_keys(modifier_keys, responses);
-				self.mouse.scroll_delta = delta;
-				responses.push_back(InputMapperMessage::MouseScroll.into());
-			}
-			InputPreprocessorMessage::MouseDown(state, modifier_keys) => {
+			InputPreprocessorMessage::MouseDown(editor_mouse_state, modifier_keys) => {
 				self.handle_modifier_keys(modifier_keys, responses);
-				if let Some(message) = self.translate_mouse_event(state, KeyPosition::Pressed) {
+
+				let mouse_state = editor_mouse_state.to_mouse_state(&self.viewport_bounds);
+				self.mouse.position = mouse_state.position;
+
+				if let Some(message) = self.translate_mouse_event(mouse_state, KeyPosition::Pressed) {
 					responses.push_back(message);
 				}
 			}
-			InputPreprocessorMessage::MouseUp(state, modifier_keys) => {
+			InputPreprocessorMessage::MouseUp(editor_mouse_state, modifier_keys) => {
 				self.handle_modifier_keys(modifier_keys, responses);
-				if let Some(message) = self.translate_mouse_event(state, KeyPosition::Released) {
+
+				let mouse_state = editor_mouse_state.to_mouse_state(&self.viewport_bounds);
+				self.mouse.position = mouse_state.position;
+
+				if let Some(message) = self.translate_mouse_event(mouse_state, KeyPosition::Released) {
 					responses.push_back(message);
 				}
 			}
+			InputPreprocessorMessage::MouseScroll(editor_mouse_state, modifier_keys) => {
+				self.handle_modifier_keys(modifier_keys, responses);
+
+				let mouse_state = editor_mouse_state.to_mouse_state(&self.viewport_bounds);
+				self.mouse.position = mouse_state.position;
+				self.mouse.scroll_delta = mouse_state.scroll_delta;
+
+				responses.push_back(InputMapperMessage::MouseScroll.into());
+			}
 			InputPreprocessorMessage::KeyDown(key, modifier_keys) => {
 				self.handle_modifier_keys(modifier_keys, responses);
 				self.keyboard.set(key as usize);
@@ -77,15 +92,26 @@ impl MessageHandler<InputPreprocessorMessage, ()> for InputPreprocessor {
 				self.keyboard.unset(key as usize);
 				responses.push_back(InputMapperMessage::KeyUp(key).into());
 			}
-			InputPreprocessorMessage::ViewportResize(size) => {
-				responses.push_back(
-					graphene::Operation::TransformLayer {
-						path: vec![],
-						transform: glam::DAffine2::from_translation((size - self.viewport_size) / 2.).to_cols_array(),
-					}
-					.into(),
-				);
-				self.viewport_size = size;
+			InputPreprocessorMessage::BoundsOfViewports(bounds_of_viewports) => {
+				assert_eq!(bounds_of_viewports.len(), 1, "Only one viewport is currently supported");
+
+				for bounds in bounds_of_viewports {
+					let new_size = bounds.size();
+					let existing_size = self.viewport_bounds.size();
+
+					let translation = (new_size - existing_size) / 2.;
+
+					// TODO: Extend this to multiple viewports instead of setting it to the value of this last loop iteration
+					self.viewport_bounds = bounds;
+
+					responses.push_back(
+						graphene::Operation::TransformLayer {
+							path: vec![],
+							transform: glam::DAffine2::from_translation(translation).to_cols_array(),
+						}
+						.into(),
+					);
+				}
 			}
 		};
 	}
@@ -136,12 +162,16 @@ impl InputPreprocessor {
 
 #[cfg(test)]
 mod test {
+	use crate::input::mouse::ViewportPosition;
+
 	use super::*;
 
 	#[test]
 	fn process_action_mouse_move_handle_modifier_keys() {
 		let mut input_preprocessor = InputPreprocessor::default();
-		let message = InputPreprocessorMessage::MouseMove((4., 809.).into(), ModifierKeys::ALT);
+		let mut editor_mouse_state = EditorMouseState::new();
+		editor_mouse_state.editor_position = ViewportPosition::new(4., 809.);
+		let message = InputPreprocessorMessage::MouseMove(editor_mouse_state, ModifierKeys::ALT);
 		let mut responses = VecDeque::new();
 
 		input_preprocessor.process_action(message, (), &mut responses);
@@ -153,7 +183,7 @@ mod test {
 	#[test]
 	fn process_action_mouse_down_handle_modifier_keys() {
 		let mut input_preprocessor = InputPreprocessor::default();
-		let message = InputPreprocessorMessage::MouseDown(MouseState::new(), ModifierKeys::CONTROL);
+		let message = InputPreprocessorMessage::MouseDown(EditorMouseState::new(), ModifierKeys::CONTROL);
 		let mut responses = VecDeque::new();
 
 		input_preprocessor.process_action(message, (), &mut responses);
@@ -165,7 +195,7 @@ mod test {
 	#[test]
 	fn process_action_mouse_up_handle_modifier_keys() {
 		let mut input_preprocessor = InputPreprocessor::default();
-		let message = InputPreprocessorMessage::MouseUp(MouseState::new(), ModifierKeys::SHIFT);
+		let message = InputPreprocessorMessage::MouseUp(EditorMouseState::new(), ModifierKeys::SHIFT);
 		let mut responses = VecDeque::new();
 
 		input_preprocessor.process_action(message, (), &mut responses);

editor/src/input/mouse.rs

@@ -3,6 +3,26 @@ use glam::DVec2;
 
 // Origin is top left
 pub type ViewportPosition = DVec2;
+pub type EditorPosition = DVec2;
+
+#[derive(PartialEq, Clone, Debug, Default)]
+pub struct ViewportBounds {
+	pub top_left: DVec2,
+	pub bottom_right: DVec2,
+}
+
+impl ViewportBounds {
+	pub fn from_slice(slice: &[f64]) -> Self {
+		Self {
+			top_left: DVec2::from_slice(&slice[0..2]),
+			bottom_right: DVec2::from_slice(&slice[2..4]),
+		}
+	}
+
+	pub fn size(&self) -> DVec2 {
+		self.bottom_right - self.top_left
+	}
+}
 
 #[derive(Debug, Copy, Clone, Default, Eq, PartialEq, Hash)]
 pub struct ScrollDelta {
@@ -35,15 +55,15 @@ impl MouseState {
 		Self::default()
 	}
 
-	pub fn from_pos(x: f64, y: f64) -> Self {
+	pub fn from_position(x: f64, y: f64) -> Self {
 		Self {
 			position: (x, y).into(),
 			mouse_keys: MouseKeys::default(),
 			scroll_delta: ScrollDelta::default(),
 		}
 	}
 
-	pub fn from_u8_pos(keys: u8, position: ViewportPosition) -> Self {
+	pub fn from_keys_and_editor_position(keys: u8, position: ViewportPosition) -> Self {
 		let mouse_keys = MouseKeys::from_bits(keys).expect("invalid modifier keys");
 		Self {
 			position,
@@ -52,6 +72,45 @@ impl MouseState {
 		}
 	}
 }
+
+#[derive(Debug, Copy, Clone, Default, PartialEq)]
+pub struct EditorMouseState {
+	pub editor_position: EditorPosition,
+	pub mouse_keys: MouseKeys,
+	pub scroll_delta: ScrollDelta,
+}
+
+impl EditorMouseState {
+	pub fn new() -> Self {
+		Self::default()
+	}
+
+	pub fn from_editor_position(x: f64, y: f64) -> Self {
+		Self {
+			editor_position: (x, y).into(),
+			mouse_keys: MouseKeys::default(),
+			scroll_delta: ScrollDelta::default(),
+		}
+	}
+
+	pub fn from_keys_and_editor_position(keys: u8, editor_position: EditorPosition) -> Self {
+		let mouse_keys = MouseKeys::from_bits(keys).expect("invalid modifier keys");
+		Self {
+			editor_position,
+			mouse_keys,
+			scroll_delta: ScrollDelta::default(),
+		}
+	}
+
+	pub fn to_mouse_state(&self, active_viewport_bounds: &ViewportBounds) -> MouseState {
+		MouseState {
+			position: self.editor_position - active_viewport_bounds.top_left,
+			mouse_keys: self.mouse_keys,
+			scroll_delta: self.scroll_delta,
+		}
+	}
+}
+
 bitflags! {
 	#[derive(Default)]
 	#[repr(transparent)]