micro:bit key presses functionality (accessibility) (#222)

Co-authored-by: Kevin Nguyen <xuannamkevin@gmail.com>
Co-authored-by: Andrea Mah <31675041+andreamah@users.noreply.github.com>

Author: 3 people

README.md

@@ -145,7 +145,7 @@ Device Simulator Express provides several commands in the Command Palette (F1 or
 
 In Device Simulator Express, you can use keyboard to interact with the device:
 
-- Push Button `A & B: A B`
+- Push Button `A for A, B for B, C for A & B`
 - Capacitive Touch Sensor `A1 – A7: SHIFT + 1~7`
 - Slider Switch: `SHIFT + S`
 - Refresh the simulator: `SHIFT + R`
@@ -175,7 +175,7 @@ Using the simulator for the micro:bit is similar to using the one for the CPX. T
 Please review the CPX's ["How to use" guide](#How-to-use) for more info.
 
 ### Keybindings
-- Push Button `A & B: A B`
+- Push Button `A for A, B for B, C for A & B`
 - Refresh the simulator: `SHIFT + R`
 
 ## Contribute

package.json

@@ -171,7 +171,7 @@
                     "type": "boolean",
                     "default": false,
                     "description": "%deviceSimulatorExpressExtension.configuration.properties.previewMode%",
-                    "scope": "resource"    
+                    "scope": "resource"
                 }
             }
         },
@@ -351,4 +351,4 @@
     "extensionDependencies": [
         "ms-python.python"
     ]
-}
+}

src/adafruit_circuitplayground/constants.py

@@ -27,17 +27,6 @@
 
 VALID_PIXEL_ASSIGN_ERROR = "The pixel color value should be a tuple with three values between 0 and 255 or a hexadecimal color between 0x000000 and 0xFFFFFF."
 
-TELEMETRY_EVENT_NAMES = {
-    "TAPPED": "API.TAPPED",
-    "PLAY_FILE": "API.PLAY.FILE",
-    "PLAY_TONE": "API.PLAY.TONE",
-    "START_TONE": "API.START.TONE",
-    "STOP_TONE": "API.STOP.TONE",
-    "DETECT_TAPS": "API.DETECT.TAPS",
-    "ADJUST_THRESHOLD": "API.ADJUST.THRESHOLD",
-    "RED_LED": "API.RED.LED",
-    "PIXELS": "API.PIXELS",
-}
 ERROR_SENDING_EVENT = "Error trying to send event to the process : "
 
 TIME_DELAY = 0.03

src/view/components/cpx/CpxImage.tsx

@@ -27,14 +27,42 @@ export class CpxImage extends React.Component<IProps, any> {
             initSvgStyle(svgElement, this.props.brightness);
             setupButtons(this.props);
             setupPins(this.props);
-            setupKeyPresses(this.props.onKeyEvent);
+            this.setupKeyPresses(this.props.onKeyEvent);
             setupSwitch(this.props);
             this.updateImage();
         }
     }
+    componentWillUnmount() {
+        window.document.removeEventListener("keydown", this.handleKeyDown);
+        window.document.removeEventListener("keyup", this.handleKeyUp);
+    }
     componentDidUpdate() {
         this.updateImage();
     }
+    setupKeyPresses = (
+        onKeyEvent: (event: KeyboardEvent, active: boolean) => void
+    ) => {
+        window.document.addEventListener("keydown", this.handleKeyDown);
+        window.document.addEventListener("keyup", this.handleKeyUp);
+    };
+
+    handleKeyDown = (event: KeyboardEvent) => {
+        const keyEvents = [event.key, event.code];
+        // Don't listen to keydown events for the switch, run button, restart button and enter key
+        if (
+            !(
+                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.S) ||
+                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.CAPITAL_F) ||
+                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.CAPITAL_R) ||
+                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.ENTER)
+            )
+        ) {
+            this.props.onKeyEvent(event, true);
+        }
+    };
+    handleKeyUp = (event: KeyboardEvent) => {
+        this.props.onKeyEvent(event, false);
+    };
     render() {
         return CPX_SVG;
     }
@@ -309,32 +337,18 @@ const setupButton = (button: HTMLElement, className: string, props: IProps) => {
     }
     svgButton.onmousedown = e => props.onMouseDown(button, e);
     svgButton.onmouseup = e => props.onMouseUp(button, e);
-    svgButton.onkeydown = e => props.onKeyEvent(e, true);
+    svgButton.onkeydown = e => {
+        // ensure that the keydown is enter.
+        // Or else, if the key is a shortcut instead,
+        // it may register shortcuts twice
+        if (e.key === CONSTANTS.KEYBOARD_KEYS.ENTER) {
+            props.onKeyEvent(e, true);
+        }
+    };
     svgButton.onkeyup = e => props.onKeyEvent(e, false);
     svgButton.onmouseleave = e => props.onMouseLeave(button, e);
 };
 
-const setupKeyPresses = (
-    onKeyEvent: (event: KeyboardEvent, active: boolean) => void
-) => {
-    window.document.addEventListener("keydown", event => {
-        const keyEvents = [event.key, event.code];
-        // Don't listen to keydown events for the switch, run button and enter key
-        if (
-            !(
-                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.S) ||
-                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.CAPITAL_F) ||
-                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.ENTER)
-            )
-        ) {
-            onKeyEvent(event, true);
-        }
-    });
-    window.document.addEventListener("keyup", event =>
-        onKeyEvent(event, false)
-    );
-};
-
 const setupSwitch = (props: IProps): void => {
     const switchElement = window.document.getElementById("SWITCH");
     const swInnerElement = window.document.getElementById("SWITCH_INNER");

src/view/components/cpx/CpxSimulator.tsx

@@ -132,6 +132,7 @@ class Simulator extends React.Component<{}, IState> {
 
     render() {
         const playStopImage = this.state.play_button ? StopLogo : PlayLogo;
+        const playStopLabel = this.state.play_button ? "stop" : "play";
         return (
             <div className="simulator">
                 <div className="file-selector">
@@ -161,32 +162,33 @@ class Simulator extends React.Component<{}, IState> {
                     onTogglePlay={this.togglePlayClick}
                     onToggleRefresh={this.refreshSimulatorClick}
                     playStopImage={playStopImage}
+                    playStopLabel={playStopLabel}
                 />
             </div>
         );
     }
 
     protected togglePlayClick() {
-        sendMessage(WEBVIEW_MESSAGES.TOGGLE_PLAY_STOP, {
-            selected_file: this.state.selected_file,
-            state: !this.state.play_button,
-        });
         const button =
             window.document.getElementById(CONSTANTS.ID_NAME.PLAY_BUTTON) ||
             window.document.getElementById(CONSTANTS.ID_NAME.STOP_BUTTON);
         if (button) {
             button.focus();
         }
+        sendMessage(WEBVIEW_MESSAGES.TOGGLE_PLAY_STOP, {
+            selected_file: this.state.selected_file,
+            state: !this.state.play_button,
+        });
     }
 
     protected refreshSimulatorClick() {
-        sendMessage(WEBVIEW_MESSAGES.REFRESH_SIMULATOR, true);
         const button = window.document.getElementById(
             CONSTANTS.ID_NAME.REFRESH_BUTTON
         );
         if (button) {
             button.focus();
         }
+        sendMessage(WEBVIEW_MESSAGES.REFRESH_SIMULATOR, true);
     }
 
     protected onSelectBlur(event: React.FocusEvent<HTMLSelectElement>) {
@@ -216,6 +218,12 @@ class Simulator extends React.Component<{}, IState> {
             element = window.document.getElementById(
                 CONSTANTS.ID_NAME.BUTTON_B
             );
+        } else if (
+            [event.code, event.key].includes(CONSTANTS.KEYBOARD_KEYS.C)
+        ) {
+            element = window.document.getElementById(
+                CONSTANTS.ID_NAME.BUTTON_AB
+            );
         } else if (
             [event.code, event.key].includes(CONSTANTS.KEYBOARD_KEYS.S)
         ) {

src/view/components/microbit/MicrobitImage.tsx

@@ -4,13 +4,15 @@
 import * as React from "react";
 import { VIEW_STATE } from "../../constants";
 import { ViewStateContext } from "../../context";
+import CONSTANTS, { MICROBIT_BUTTON_STYLING_CLASSES } from "../../constants";
 import "../../styles/Microbit.css";
 import { IRefObject, MicrobitSvg } from "./Microbit_svg";
 
 interface EventTriggers {
     onMouseUp: (event: Event, buttonKey: string) => void;
     onMouseDown: (event: Event, buttonKey: string) => void;
     onMouseLeave: (event: Event, buttonKey: string) => void;
+    onKeyEvent: (event: KeyboardEvent, active: boolean, key: string) => void;
 }
 interface IProps {
     eventTriggers: EventTriggers;
@@ -22,6 +24,11 @@ const BUTTON_CLASSNAME = {
     DEACTIVATED: "sim-button-deactivated",
 };
 
+export enum BUTTONS_KEYS {
+    BTN_A = "BTN_A",
+    BTN_B = "BTN_B",
+    BTN_AB = "BTN_AB",
+}
 // Displays the SVG and call necessary svg modification.
 export class MicrobitImage extends React.Component<IProps, {}> {
     private svgRef: React.RefObject<MicrobitSvg> = React.createRef();
@@ -33,6 +40,7 @@ export class MicrobitImage extends React.Component<IProps, {}> {
         if (svgElement) {
             updateAllLeds(this.props.leds, svgElement.getLeds());
             setupAllButtons(this.props.eventTriggers, svgElement.getButtons());
+            this.setupKeyPresses(this.props.eventTriggers.onKeyEvent);
         }
     }
     componentDidUpdate() {
@@ -48,9 +56,56 @@ export class MicrobitImage extends React.Component<IProps, {}> {
             }
         }
     }
+    componentWillUnmount() {
+        window.document.removeEventListener("keydown", this.handleKeyDown);
+        window.document.removeEventListener("keyup", this.handleKeyUp);
+    }
+    setupKeyPresses = (
+        onKeyEvent: (event: KeyboardEvent, active: boolean, key: string) => void
+    ) => {
+        window.document.addEventListener("keydown", this.handleKeyDown);
+        window.document.addEventListener("keyup", this.handleKeyUp);
+    };
+    handleKeyDown = (event: KeyboardEvent) => {
+        const keyEvents = [event.key, event.code];
+        // Don't listen to keydown events for the run button, restart button and enter key
+        if (
+            !(
+                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.CAPITAL_F) ||
+                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.CAPITAL_R) ||
+                keyEvents.includes(CONSTANTS.KEYBOARD_KEYS.ENTER)
+            )
+        ) {
+            this.props.eventTriggers.onKeyEvent(event, true, event.key);
+        }
+    };
+    handleKeyUp = (event: KeyboardEvent) => {
+        this.props.eventTriggers.onKeyEvent(event, false, event.key);
+    };
     render() {
         return <MicrobitSvg ref={this.svgRef} />;
     }
+    public updateButtonAttributes(key: BUTTONS_KEYS, isActive: boolean) {
+        if (this.svgRef.current) {
+            const button = this.svgRef.current.getButtons()[key].current;
+            if (button) {
+                button.focus();
+                if (isActive) {
+                    button.children[0].setAttribute(
+                        "class",
+                        MICROBIT_BUTTON_STYLING_CLASSES.KEYPRESSED
+                    );
+                } else {
+                    button.children[0].setAttribute(
+                        "class",
+                        MICROBIT_BUTTON_STYLING_CLASSES.DEFAULT
+                    );
+                }
+                button.setAttribute("pressed", `${isActive}`);
+                button.setAttribute("aria-pressed", `${isActive}`);
+            }
+        }
+    }
 }
 
 MicrobitImage.contextType = ViewStateContext;
@@ -59,8 +114,8 @@ const setupButton = (
     eventTriggers: EventTriggers,
     key: string
 ) => {
-    buttonElement.setAttribute("class", BUTTON_CLASSNAME.ACTIVE);
     buttonElement.onmousedown = e => {
+        buttonElement.focus();
         eventTriggers.onMouseDown(e, key);
     };
     buttonElement.onmouseup = e => {
@@ -69,6 +124,16 @@ const setupButton = (
     buttonElement.onmouseleave = e => {
         eventTriggers.onMouseLeave(e, key);
     };
+    buttonElement.onkeydown = e => {
+        // ensure that the keydown is enter,
+        // or else it may register shortcuts twice
+        if (e.key === CONSTANTS.KEYBOARD_KEYS.ENTER) {
+            eventTriggers.onKeyEvent(e, true, key);
+        }
+    };
+    buttonElement.onkeyup = e => {
+        eventTriggers.onKeyEvent(e, false, key);
+    };
 };
 const setupAllButtons = (
     eventTriggers: EventTriggers,
@@ -87,6 +152,8 @@ const disableAllButtons = (buttonRefs: IRefObject) => {
             ref.current.onmousedown = null;
             ref.current.onmouseup = null;
             ref.current.onmouseleave = null;
+            ref.current.onkeydown = null;
+            ref.current.onkeyup = null;
             ref.current.setAttribute("class", BUTTON_CLASSNAME.DEACTIVATED);
         }
     }