feat(studio): Initial RPC infrastructure and subsystems.

* UART and BLE/GATT transports for a protobuf encoded RPC
  request/response protocol.
* Custom framing protocol is used to frame a give message.
* Requests/responses are divided into major "subsystems" which
  handle requests and create response messages.
* Notification support, including mapping local events to RPC
  notifications by a given subsystem.
* Meta responses for "no response" and "unlock needed".
* Initial basic lock state support in a new core section, and allow specifying
  if a given RPC callback requires unlocked state or not.
* Add behavior subsystem with full metadata support and examples of
  using callback to serialize a repeated field without extra stack space needed.

Co-authored-by: Cem Aksoylar <caksoylar@users.noreply.github.com>

app/CMakeLists.txt

@@ -105,4 +105,27 @@ target_sources(app PRIVATE src/main.c)
 add_subdirectory(src/display/)
 add_subdirectory_ifdef(CONFIG_SETTINGS src/settings/)
 
+if (CONFIG_ZMK_STUDIO_RPC)
+  # For some reason this is failing if run from a different sub-file.
+  list(APPEND CMAKE_MODULE_PATH ${ZEPHYR_BASE}/modules/nanopb)
+
+  include(nanopb)
+
+    # Turn off the default nanopb behavior
+  set(NANOPB_GENERATE_CPP_STANDALONE OFF)
+
+  nanopb_generate_cpp(proto_srcs proto_hdrs RELPATH ${ZEPHYR_ZMK_STUDIO_MESSAGES_MODULE_DIR}
+                        ${ZEPHYR_ZMK_STUDIO_MESSAGES_MODULE_DIR}/proto/zmk/studio.proto
+                        ${ZEPHYR_ZMK_STUDIO_MESSAGES_MODULE_DIR}/proto/zmk/meta.proto
+                        ${ZEPHYR_ZMK_STUDIO_MESSAGES_MODULE_DIR}/proto/zmk/core.proto
+                        ${ZEPHYR_ZMK_STUDIO_MESSAGES_MODULE_DIR}/proto/zmk/behaviors.proto
+                        ${ZEPHYR_ZMK_STUDIO_MESSAGES_MODULE_DIR}/proto/zmk/keymap.proto
+  )
+
+  target_include_directories(app PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+  target_sources(app PRIVATE ${proto_srcs} ${proto_hdrs})
+
+  add_subdirectory(src/studio)
+endif()
+
 zephyr_cc_option(-Wfatal-errors)

app/Kconfig

@@ -258,6 +258,8 @@ rsource "src/split/Kconfig"
 #Basic Keyboard Setup
 endmenu
 
+rsource "src/studio/Kconfig"
+
 menu "Display/LED Options"
 
 rsource "src/display/Kconfig"

app/include/linker/zmk-rpc-event-mappers.ld

@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2024 The ZMK Contributors
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <zephyr/linker/linker-defs.h>
+
+ITERABLE_SECTION_ROM(zmk_rpc_event_mapper, 4)

app/include/linker/zmk-rpc-subsystem-handlers.ld

@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2024 The ZMK Contributors
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <zephyr/linker/linker-defs.h>
+
+ITERABLE_SECTION_ROM(zmk_rpc_subsystem_handler, 4)

app/include/linker/zmk-rpc-subsystems.ld

@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2024 The ZMK Contributors
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <zephyr/linker/linker-defs.h>
+
+ITERABLE_SECTION_RAM(zmk_rpc_subsystem, 4)

app/include/linker/zmk-rpc-transport.ld

@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2024 The ZMK Contributors
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <zephyr/linker/linker-defs.h>
+
+ITERABLE_SECTION_ROM(zmk_rpc_transport, 4)

app/include/zmk/hid.h

@@ -25,6 +25,22 @@
 #define ZMK_HID_KEYBOARD_NKRO_MAX_USAGE HID_USAGE_KEY_KEYPAD_EQUAL
 #endif
 
+#if IS_ENABLED(CONFIG_ZMK_HID_CONSUMER_REPORT_USAGES_BASIC)
+#define ZMK_HID_CONSUMER_MAX_USAGE 0xFF
+#elif IS_ENABLED(CONFIG_ZMK_HID_CONSUMER_REPORT_USAGES_FULL)
+#define ZMK_HID_CONSUMER_MAX_USAGE 0xFFF
+#else
+#error "Unknown consumer report usages configuration"
+#endif
+
+#if IS_ENABLED(CONFIG_ZMK_HID_REPORT_TYPE_NKRO)
+#define ZMK_HID_KEYBOARD_MAX_USAGE ZMK_HID_KEYBOARD_NKRO_MAX_USAGE
+#elif IS_ENABLED(CONFIG_ZMK_HID_REPORT_TYPE_HKRO)
+#define ZMK_HID_KEYBOARD_MAX_USAGE 0xFF
+#else
+#error "Unknown keyboard report usages configuration"
+#endif
+
 #define ZMK_HID_MOUSE_NUM_BUTTONS 0x05
 
 // See https://www.usb.org/sites/default/files/hid1_11.pdf section 6.2.2.4 Main Items

app/include/zmk/studio/core.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2024 The ZMK Contributors
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <zmk/event_manager.h>
+
+enum zmk_studio_core_lock_state {
+    ZMK_STUDIO_CORE_LOCK_STATE_LOCKED = 0,
+    ZMK_STUDIO_CORE_LOCK_STATE_UNLOCKED = 1,
+};
+
+struct zmk_studio_core_lock_state_changed {
+    enum zmk_studio_core_lock_state state;
+};
+
+struct zmk_studio_core_unlock_requested {};
+
+ZMK_EVENT_DECLARE(zmk_studio_core_lock_state_changed);
+
+enum zmk_studio_core_lock_state zmk_studio_core_get_lock_state(void);
+
+void zmk_studio_core_unlock();
+void zmk_studio_core_lock();
+void zmk_studio_core_initiate_unlock();
+void zmk_studio_core_complete_unlock();
+
+void zmk_studio_core_reschedule_lock_timeout();

app/include/zmk/studio/rpc.h

@@ -0,0 +1,215 @@
+/*
+ * Copyright (c) 2024 The ZMK Contributors
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <zephyr/sys/iterable_sections.h>
+#include <zephyr/sys/ring_buffer.h>
+
+#include <proto/zmk/studio.pb.h>
+
+#include <zmk/endpoints_types.h>
+#include <zmk/event_manager.h>
+#include <zmk/studio/core.h>
+
+enum zmk_studio_rpc_handler_security {
+    ZMK_STUDIO_RPC_HANDLER_SECURED,
+    ZMK_STUDIO_RPC_HANDLER_UNSECURED,
+};
+
+struct zmk_studio_rpc_notification {
+    zmk_studio_Notification notification;
+};
+
+ZMK_EVENT_DECLARE(zmk_studio_rpc_notification);
+
+struct zmk_rpc_subsystem;
+
+typedef zmk_studio_Response(subsystem_func)(const struct zmk_rpc_subsystem *subsys,
+                                            const zmk_studio_Request *req);
+
+typedef zmk_studio_Response(rpc_func)(const zmk_studio_Request *neq);
+
+/**
+ * @brief An RPC subsystem is a cohesive collection of related RPCs. A specific RPC is identified by
+ *        the pair or subsystem and request identifiers. This struct is the high level entity to
+ *        aggregate all the possible handler functions for the request in the given subsystem.
+ */
+struct zmk_rpc_subsystem {
+    subsystem_func *func;
+    uint16_t handlers_start_index;
+    uint16_t handlers_end_index;
+    uint8_t subsystem_choice;
+};
+
+/**
+ * @brief An entry for a specific handler function in a given subsystem, including metadata
+ * indicating if the particular handler requires the device be unlock in order to be invoked.
+ */
+struct zmk_rpc_subsystem_handler {
+    rpc_func *func;
+    uint8_t subsystem_choice;
+    uint8_t request_choice;
+    enum zmk_studio_rpc_handler_security security;
+};
+
+/**
+ * @brief Generate a "meta" subsystem response indicating an "empty" response to an RPC request.
+ */
+#define ZMK_RPC_NO_RESPONSE() ZMK_RPC_RESPONSE(meta, no_response, true)
+
+/**
+ * @brief Generate a "meta" subsystem response with one of a few possible simple error responses.
+ * @see   https://github.com/zmkfirmware/zmk-studio-messages/blob/main/proto/zmk/meta.proto#L5
+ */
+#define ZMK_RPC_SIMPLE_ERR(type)                                                                   \
+    ZMK_RPC_RESPONSE(meta, simple_error, zmk_meta_ErrorConditions_##type)
+
+/**
+ * @brief Register an RPC subsystem to aggregate handlers for request to that subsystem.
+ * @param prefix The identifier for the subsystem, e.g. `core`, `keymap`, etc.
+ * @see   https://github.com/zmkfirmware/zmk-studio-messages/blob/main/proto/zmk/studio.proto#L15
+ */
+#define ZMK_RPC_SUBSYSTEM(prefix)                                                                  \
+    zmk_studio_Response subsystem_func_##prefix(const struct zmk_rpc_subsystem *subsys,            \
+                                                const zmk_studio_Request *req) {                   \
+        uint8_t which_req = req->subsystem.prefix.which_request_type;                              \
+        return zmk_rpc_subsystem_delegate_to_subs(subsys, req, which_req);                         \
+    }                                                                                              \
+    STRUCT_SECTION_ITERABLE(zmk_rpc_subsystem, prefix##_subsystem) = {                             \
+        .func = subsystem_func_##prefix,                                                           \
+        .subsystem_choice = zmk_studio_Request_##prefix##_tag,                                     \
+    };
+
+/**
+ * @brief Register an RPC subsystem handler handler a specific request within the subsystem.
+ * @param prefix The identifier for the subsystem, e.g. `core`, `keymap`, etc.
+ * @param request_id The identifier for the request ID, e.g. `save_changes`.
+ * @param _secured Whether the handler requires the device be unlocked to allow invocation.
+ *
+ * @note  A function with a name matching the request_id must be in-scope and will be used as the
+ *        the callback handler. The function must have a signature of
+ *        zmk_studio_Response (*func)(const zmk_studio_Request*)
+ */
+#define ZMK_RPC_SUBSYSTEM_HANDLER(prefix, request_id, _security)                                   \
+    STRUCT_SECTION_ITERABLE(zmk_rpc_subsystem_handler,                                             \
+                            prefix##_subsystem_handler_##request_id) = {                           \
+        .func = request_id,                                                                        \
+        .subsystem_choice = zmk_studio_Request_##prefix##_tag,                                     \
+        .request_choice = zmk_##prefix##_Request_##request_id##_tag,                               \
+        .security = _security,                                                                     \
+    };
+
+/**
+ * @brief Create a zmk_studio_Notification struct for the given subsystem and type, including
+          initialization of the inner fields.
+ * @param subsys The identifier for the subsystem, e.g. `core`, `keymap`, etc.
+ * @param _type The identifier for the notification type in that subsystem, e.g.
+ `unsaved_changes_status_changed`.
+ *
+ * @see   example
+ https://github.com/zmkfirmware/zmk-studio-messages/blob/main/proto/zmk/keymap.proto#L41C14-L41C44
+ */
+#define ZMK_RPC_NOTIFICATION(subsys, _type, ...)                                                   \
+    ((zmk_studio_Notification){                                                                    \
+        .which_subsystem = zmk_studio_Notification_##subsys##_tag,                                 \
+        .subsystem =                                                                               \
+            {                                                                                      \
+                .subsys =                                                                          \
+                    {                                                                              \
+                        .which_notification_type = zmk_##subsys##_Notification_##_type##_tag,      \
+                        .notification_type = {._type = __VA_ARGS__},                               \
+                    },                                                                             \
+            },                                                                                     \
+    })
+
+/**
+ * @brief Create a zmk_studio_Response struct for the given subsystem and type, including
+          initialization of the inner fields.
+ * @param subsys The identifier for the subsystem, e.g. `core`, `keymap`, etc.
+ * @param _type The identifier for the response type in that subsystem, e.g. `get_keymap`.
+ *
+ * @see   example
+ https://github.com/zmkfirmware/zmk-studio-messages/blob/main/proto/zmk/keymap.proto#L24
+ */
+#define ZMK_RPC_RESPONSE(subsys, _type, ...)                                                       \
+    ((zmk_studio_Response){                                                                        \
+        .which_type = zmk_studio_Response_request_response_tag,                                    \
+        .type =                                                                                    \
+            {                                                                                      \
+                .request_response =                                                                \
+                    {                                                                              \
+                        .which_subsystem = zmk_studio_RequestResponse_##subsys##_tag,              \
+                        .subsystem =                                                               \
+                            {                                                                      \
+                                .subsys =                                                          \
+                                    {                                                              \
+                                        .which_response_type =                                     \
+                                            zmk_##subsys##_Response_##_type##_tag,                 \
+                                        .response_type = {._type = __VA_ARGS__},                   \
+                                    },                                                             \
+                            },                                                                     \
+                    },                                                                             \
+            },                                                                                     \
+    })
+
+typedef int(zmk_rpc_event_mapper_cb)(const zmk_event_t *ev, zmk_studio_Notification *n);
+
+struct zmk_rpc_event_mapper {
+    zmk_rpc_event_mapper_cb *func;
+};
+
+/**
+ * @brief A single ZMK event listener is registered that will listen for events and map them to
+ *        RPC notifications to be sent to the connected client. This macro adds additional
+ * subscriptions to that one single registered listener.
+ * @param _t The ZMK event type.
+ */
+#define ZMK_RPC_EVENT_MAPPER_ADD_LISTENER(_t) ZMK_SUBSCRIPTION(studio_rpc, _t)
+
+/**
+ * @brief Register a mapping function that can selectively map a given internal ZMK event type into
+ *        a possible zmk_studio_Notification type.
+ * @param name A unique identifier for the mapper. Often a subsystem identifier like `core` is used.
+ * @param _func The `zmk_rpc_event_mapper_cb` function used to map the internal event type.
+ */
+#define ZMK_RPC_EVENT_MAPPER(name, _func, ...)                                                     \
+    FOR_EACH_NONEMPTY_TERM(ZMK_RPC_EVENT_MAPPER_ADD_LISTENER, (;), __VA_ARGS__)                    \
+    STRUCT_SECTION_ITERABLE(zmk_rpc_event_mapper, name) = {                                        \
+        .func = _func,                                                                             \
+    };
+
+typedef int (*zmk_rpc_rx_start_stop_func)(void);
+
+typedef void (*zmk_rpc_tx_buffer_notify_func)(struct ring_buf *buf, size_t added, bool message_done,
+                                              void *user_data);
+typedef void *(*zmk_rpc_tx_user_data_func)(void);
+
+struct zmk_rpc_transport {
+    enum zmk_transport transport;
+
+    zmk_rpc_tx_user_data_func tx_user_data;
+    zmk_rpc_tx_buffer_notify_func tx_notify;
+    zmk_rpc_rx_start_stop_func rx_start;
+    zmk_rpc_rx_start_stop_func rx_stop;
+};
+
+zmk_studio_Response zmk_rpc_subsystem_delegate_to_subs(const struct zmk_rpc_subsystem *subsys,
+                                                       const zmk_studio_Request *req,
+                                                       uint8_t which_req);
+
+struct ring_buf *zmk_rpc_get_tx_buf(void);
+struct ring_buf *zmk_rpc_get_rx_buf(void);
+void zmk_rpc_rx_notify(void);
+
+#define ZMK_RPC_TRANSPORT(name, _transport, _rx_start, _rx_stop, _tx_user_data, _tx_notify)        \
+    STRUCT_SECTION_ITERABLE(zmk_rpc_transport, name) = {                                           \
+        .transport = _transport,                                                                   \
+        .rx_start = _rx_start,                                                                     \
+        .rx_stop = _rx_stop,                                                                       \
+        .tx_user_data = _tx_user_data,                                                             \
+        .tx_notify = _tx_notify,                                                                   \
+    }

app/src/hid.c

@@ -249,8 +249,9 @@ static inline int check_keyboard_usage(zmk_key_t usage) {
 #endif
 
 #define TOGGLE_CONSUMER(match, val)                                                                \
-    COND_CODE_1(IS_ENABLED(CONFIG_ZMK_HID_CONSUMER_REPORT_USAGES_BASIC),                           \
-                (if (val > 0xFF) { return -ENOTSUP; }), ())                                        \
+    if (val > ZMK_HID_CONSUMER_MAX_USAGE) {                                                        \
+        return -ENOTSUP;                                                                           \
+    }                                                                                              \
     for (int idx = 0; idx < CONFIG_ZMK_HID_CONSUMER_REPORT_SIZE; idx++) {                          \
         if (consumer_report.body.keys[idx] != match) {                                             \
             continue;                                                                              \

app/src/studio/CMakeLists.txt

@@ -0,0 +1,15 @@
+# Copyright (c) 2024 The ZMK Contributors
+# SPDX-License-Identifier: MIT
+
+zephyr_linker_sources(DATA_SECTIONS ../../include/linker/zmk-rpc-subsystems.ld)
+zephyr_linker_sources(SECTIONS ../../include/linker/zmk-rpc-subsystem-handlers.ld)
+zephyr_linker_sources(SECTIONS ../../include/linker/zmk-rpc-event-mappers.ld)
+zephyr_linker_sources(SECTIONS ../../include/linker/zmk-rpc-transport.ld)
+
+target_sources(app PRIVATE msg_framing.c)
+target_sources(app PRIVATE rpc.c)
+target_sources(app PRIVATE core.c)
+target_sources(app PRIVATE behavior_subsystem.c)
+target_sources(app PRIVATE core_subsystem.c)
+target_sources_ifdef(CONFIG_ZMK_STUDIO_TRANSPORT_UART app PRIVATE uart_rpc_transport.c)
+target_sources_ifdef(CONFIG_ZMK_STUDIO_TRANSPORT_BLE app PRIVATE gatt_rpc_transport.c)