From 4c53a843c8d413f788d3a94ca13bb33ec7d4a355 Mon Sep 17 00:00:00 2001 From: Ken Van Hoeylandt Date: Sun, 16 Aug 2026 23:12:34 +0200 Subject: [PATCH] Create proper keyboard device when keyboard HID is attached + fix issue with software keyboard popping up when hardware keyboard was present --- .../lvgl-module/source/devices/keyboard.cpp | 43 +++--- .../source/drivers/usb/esp32_usbhost_hid.cpp | 141 +++++++++++++++++- Platforms/platform-esp32/source/module.cpp | 3 + .../Tactility/lvgl/KeyboardDeviceListener.h | 21 +++ Tactility/Source/Tactility.cpp | 3 + .../Source/lvgl/KeyboardDeviceListener.cpp | 115 ++++++++++++++ .../include/tactility/device_listener.h | 1 + TactilityKernel/source/device.cpp | 2 + 8 files changed, 304 insertions(+), 25 deletions(-) create mode 100644 Tactility/Include/Tactility/lvgl/KeyboardDeviceListener.h create mode 100644 Tactility/Source/lvgl/KeyboardDeviceListener.cpp diff --git a/Modules/lvgl-module/source/devices/keyboard.cpp b/Modules/lvgl-module/source/devices/keyboard.cpp index e3319a983..02bee3d89 100644 --- a/Modules/lvgl-module/source/devices/keyboard.cpp +++ b/Modules/lvgl-module/source/devices/keyboard.cpp @@ -4,9 +4,12 @@ #include #include +#include #include +constexpr auto* TAG = "lvgl_keyboard"; + static LvglSoftwareKeyboard last_software_keyboard = { .object = nullptr }; @@ -106,16 +109,23 @@ void lvgl_keyboard_disable(lv_indev_t* indev) { lv_indev_set_group(indev, nullptr); } -bool lvgl_hardware_keyboard_is_available() { - Device* keyboard_device; - if (device_get_first_active_by_type(&KEYBOARD_TYPE, &keyboard_device) != ERROR_NONE) { - return false; +static bool lvgl_hardware_keyboard_check_present(Device* device, void* context) { + bool ready = device_is_ready(device); + bool present = ready && keyboard_is_present(device); + LOG_D(TAG, "keyboard device %s: ready=%d present=%d", device->name, (int)ready, (int)present); + if (!present) { + return true; // keep looking } + *static_cast(context) = true; + return false; // found one, stop iterating +} +bool lvgl_hardware_keyboard_is_available() { // TODO: Refactor the driver subsystem to so it does proper probing/releasing of such devices // This work-around exists for the Tab5 keyboard driver. - bool present = keyboard_is_present(keyboard_device); - device_put(keyboard_device); + bool present = false; + device_for_each_of_type(&KEYBOARD_TYPE, &present, lvgl_hardware_keyboard_check_present); + LOG_D(TAG, "lvgl_hardware_keyboard_is_available() -> %d", (int)present); return present; } @@ -138,6 +148,11 @@ void lvgl_hardware_keyboard_remove_custom(lv_indev_t* indev) { } static void textarea_show_keyboard(lv_event_t* event) { + // Re-checked here rather than gated once at lvgl_keyboard_add_textarea() time, so a hardware + // keyboard that connects/disconnects after the textarea was created is honored immediately. + if (!lvgl_software_keyboard_is_enabled()) { + return; + } lv_obj_t* target = lv_event_get_current_target_obj(event); if (last_software_keyboard.object != nullptr) { lvgl_software_keyboard_show(&last_software_keyboard, target); @@ -192,18 +207,10 @@ LvglSoftwareKeyboard* lvgl_software_keyboard_get_last() { } void lvgl_keyboard_add_textarea(LvglSoftwareKeyboard* keyboard, lv_obj_t* textarea) { - // Only the on-screen keyboard's show/hide wiring is specific to "no hardware keyboard" - // mode. Group membership must NOT be gated on it: a hardware keypad indev (see - // lvgl_keyboard_enable()/lvgl_software_keyboard_activate()) is bound to keyboard_group - // regardless of whether a software keyboard is in use, so skipping lv_group_add_obj() - // here left every textarea unreachable from a hardware keyboard - it was never a member - // of the group its indev delivers key events through. - if (lvgl_software_keyboard_is_enabled()) { - lv_obj_add_event_cb(textarea, textarea_show_keyboard, LV_EVENT_FOCUSED, nullptr); - lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_DEFOCUSED, nullptr); - lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_READY, nullptr); - lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_DELETE, nullptr); - } + lv_obj_add_event_cb(textarea, textarea_show_keyboard, LV_EVENT_FOCUSED, nullptr); + lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_DEFOCUSED, nullptr); + lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_READY, nullptr); + lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_DELETE, nullptr); // lv_obj_t auto-remove themselves from the group when they are destroyed (last checked in LVGL 8.3) lv_group_add_obj(keyboard_group, textarea); diff --git a/Platforms/platform-esp32/source/drivers/usb/esp32_usbhost_hid.cpp b/Platforms/platform-esp32/source/drivers/usb/esp32_usbhost_hid.cpp index dd55b1f73..2e2b63319 100644 --- a/Platforms/platform-esp32/source/drivers/usb/esp32_usbhost_hid.cpp +++ b/Platforms/platform-esp32/source/drivers/usb/esp32_usbhost_hid.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -26,6 +27,7 @@ constexpr auto HID_PROC_TASK_STACK = 4096; constexpr auto HID_PROC_TASK_PRIORITY = 5; constexpr auto HID_STOP_TIMEOUT_MS = 2000; constexpr auto MAX_SUBSCRIBERS = 4; +constexpr auto USB_HID_KB_QUEUE_SIZE = 16; typedef struct { hid_host_device_handle_t handle; @@ -53,8 +55,20 @@ struct UsbHidContext { QueueHandle_t subscribers[MAX_SUBSCRIBERS] = {}; SemaphoreHandle_t sub_mutex = nullptr; + + // The controller Device itself (set in start_device()), used as the parent for kb_device. + Device* controller_device = nullptr; + // Dynamic KEYBOARD_TYPE child device, constructed while a physical USB keyboard is connected. + Device kb_device{}; + bool kb_device_active = false; }; +extern "C" { +static void usb_hid_keyboard_device_construct(UsbHidContext* ctx); +static void usb_hid_keyboard_device_destruct(UsbHidContext* ctx); +static void usb_hid_keyboard_publish_key(UsbHidContext* ctx, uint32_t lv_key, bool pressed, bool ctrl, bool alt, uint8_t hid_keycode, uint8_t hid_modifier); +} + static const uint8_t keycode2ascii[57][2] = { {0, 0}, {0, 0}, {0, 0}, {0, 0}, {'a', 'A'}, {'b', 'B'}, {'c', 'C'}, {'d', 'D'}, {'e', 'E'}, @@ -182,8 +196,7 @@ static void hid_interface_callback(hid_host_device_handle_t handle, uint32_t lv_key = ctx->pressed_lv_keys[prev_hid]; ctx->pressed_lv_keys[prev_hid] = 0; if (lv_key) { - UsbHidEvent evt = { .type = USB_HID_EVENT_KEY, .key = { lv_key, false, with_ctrl, with_alt } }; - publish_event(ctx, &evt); + usb_hid_keyboard_publish_key(ctx, lv_key, false, with_ctrl, with_alt, prev_hid, kb->modifier.val); } } } @@ -221,11 +234,7 @@ static void hid_interface_callback(hid_host_device_handle_t handle, uint32_t lv_key = hid_keycode_to_key(kb->modifier.val, hid_code, ctx->caps_lock_active, ctx->num_lock_active); if (lv_key) { - UsbHidEvent evt = { - .type = USB_HID_EVENT_KEY, - .key = { lv_key, true, with_ctrl, with_alt } - }; - publish_event(ctx, &evt); + usb_hid_keyboard_publish_key(ctx, lv_key, true, with_ctrl, with_alt, hid_code, kb->modifier.val); ctx->pressed_lv_keys[hid_code] = lv_key; } } @@ -271,6 +280,7 @@ static void hid_interface_callback(hid_host_device_handle_t handle, memset(ctx->pressed_lv_keys, 0, sizeof(ctx->pressed_lv_keys)); ctx->kb_handle.store(nullptr); ctx->kb_led_pending.store(false); + usb_hid_keyboard_device_destruct(ctx); } else if (params.proto == HID_PROTOCOL_MOUSE) { ctx->mouse_connected = false; } @@ -349,6 +359,7 @@ static void hid_proc_task(void* arg) { | (ctx->caps_lock_active ? 0x02 : 0) | (ctx->scroll_lock_active ? 0x04 : 0); hid_class_request_set_report(dev_evt.handle, HID_REPORT_TYPE_OUTPUT, 0, &leds, 1); + usb_hid_keyboard_device_construct(ctx); } else if (params.proto == HID_PROTOCOL_MOUSE) { ctx->mouse_connected = true; } @@ -419,8 +430,122 @@ static const UsbHidApi hid_api = { extern "C" { +// region Dynamic KEYBOARD_TYPE device +// +// While a physical USB keyboard is connected, a KEYBOARD_TYPE child device is constructed so the +// rest of the system (lvgl_hardware_keyboard_is_available(), Tactility's KeyboardDeviceListener) +// sees a real hardware keyboard through the same generic device model as any other keyboard, e.g. +// Devices/m5stack-tab5/Source/devices/tab5_keyboard.cpp. Real key events are delivered exclusively +// through this device (kb_handle's hid_interface_callback pushes into its queue below); the +// generic UsbHidEvent publish/subscribe channel above is unaffected for mouse move/button/scroll +// and the right-click-as-ESC synthesis it already does. + +static error_t usb_hid_kb_device_start(Device* device) { + auto* queue = xQueueCreate(USB_HID_KB_QUEUE_SIZE, sizeof(KeyboardKeyData)); + if (queue == nullptr) { + return ERROR_RESOURCE; + } + device_set_driver_data(device, queue); + return ERROR_NONE; +} + +static error_t usb_hid_kb_device_stop(Device* device) { + auto* queue = static_cast(device_get_driver_data(device)); + vQueueDelete(queue); + device_set_driver_data(device, nullptr); + return ERROR_NONE; +} + +static error_t usb_hid_kb_device_read_key(Device* device, KeyboardKeyData* data) { + auto* queue = static_cast(device_get_driver_data(device)); + if (queue == nullptr) { + *data = {}; + return ERROR_NONE; + } + if (xQueueReceive(queue, data, 0) != pdTRUE) { + *data = {}; + return ERROR_NONE; + } + data->continue_reading = uxQueueMessagesWaiting(queue) > 0; + return ERROR_NONE; +} + +static const KeyboardApi esp32_usbhost_hid_keyboard_api = { + .read_key = usb_hid_kb_device_read_key, + .get_backlight = nullptr, + .is_present = nullptr, +}; + +Driver esp32_usbhost_hid_keyboard_driver = { + .name = "esp32_usbhost_hid_keyboard", + .compatible = (const char*[]) { nullptr }, + .start_device = usb_hid_kb_device_start, + .stop_device = usb_hid_kb_device_stop, + .api = &esp32_usbhost_hid_keyboard_api, + .device_type = &KEYBOARD_TYPE, + .owner = nullptr, + .internal = nullptr, +}; + +static void usb_hid_keyboard_device_construct(UsbHidContext* ctx) { + if (ctx->kb_device_active) { + return; + } + + ctx->kb_device = Device { + .address = 0, + .name = "usb_keyboard0", + .config = nullptr, + .parent = nullptr, + .internal = nullptr, + }; + + if (device_construct(&ctx->kb_device) != ERROR_NONE) { + LOG_E(TAG, "failed to construct USB keyboard device"); + return; + } + device_set_parent(&ctx->kb_device, ctx->controller_device); + device_set_driver(&ctx->kb_device, &esp32_usbhost_hid_keyboard_driver); + if (device_add(&ctx->kb_device) != ERROR_NONE) { + LOG_E(TAG, "failed to add USB keyboard device"); + device_destruct(&ctx->kb_device); + return; + } + if (device_start(&ctx->kb_device) != ERROR_NONE) { + LOG_E(TAG, "failed to start USB keyboard device"); + device_remove(&ctx->kb_device); + device_destruct(&ctx->kb_device); + return; + } + + ctx->kb_device_active = true; +} + +static void usb_hid_keyboard_device_destruct(UsbHidContext* ctx) { + if (!ctx->kb_device_active) { + return; + } + ctx->kb_device_active = false; + + device_stop(&ctx->kb_device); + device_remove(&ctx->kb_device); + device_destruct(&ctx->kb_device); +} + +static void usb_hid_keyboard_publish_key(UsbHidContext* ctx, uint32_t lv_key, bool pressed, bool ctrl, bool alt, uint8_t hid_keycode, uint8_t hid_modifier) { + if (!ctx->kb_device_active) { + return; + } + auto* queue = static_cast(device_get_driver_data(&ctx->kb_device)); + KeyboardKeyData data = { lv_key, pressed, false, ctrl, alt, hid_keycode, hid_modifier }; + xQueueSend(queue, &data, 0); +} + +// endregion + static error_t start_device(struct Device* device) { auto* ctx = new UsbHidContext(); + ctx->controller_device = device; ctx->sub_mutex = xSemaphoreCreateMutex(); if (!ctx->sub_mutex) { @@ -486,6 +611,8 @@ static error_t stop_device(struct Device* device) { auto* ctx = static_cast(device_get_driver_data(device)); if (!ctx) return ERROR_NONE; + usb_hid_keyboard_device_destruct(ctx); + ctx->hid_proc_running = false; if (xSemaphoreTake(ctx->hid_proc_task_done, pdMS_TO_TICKS(HID_STOP_TIMEOUT_MS)) != pdTRUE) { diff --git a/Platforms/platform-esp32/source/module.cpp b/Platforms/platform-esp32/source/module.cpp index 557be5474..8e2b492d3 100644 --- a/Platforms/platform-esp32/source/module.cpp +++ b/Platforms/platform-esp32/source/module.cpp @@ -40,6 +40,7 @@ extern Driver esp32_ble_hid_device_driver; #if SOC_USB_OTG_SUPPORTED extern Driver esp32_usbhost_driver; extern Driver esp32_usbhost_hid_driver; +extern Driver esp32_usbhost_hid_keyboard_driver; extern Driver esp32_usbhost_midi_driver; extern Driver esp32_usbhost_msc_driver; #endif @@ -91,6 +92,7 @@ static error_t start() { #if SOC_USB_OTG_SUPPORTED check(driver_construct_add(&esp32_usbhost_driver) == ERROR_NONE); check(driver_construct_add(&esp32_usbhost_hid_driver) == ERROR_NONE); + check(driver_construct_add(&esp32_usbhost_hid_keyboard_driver) == ERROR_NONE); check(driver_construct_add(&esp32_usbhost_midi_driver) == ERROR_NONE); check(driver_construct_add(&esp32_usbhost_msc_driver) == ERROR_NONE); #endif @@ -141,6 +143,7 @@ static error_t stop() { #if SOC_USB_OTG_SUPPORTED check(driver_remove_destruct(&esp32_usbhost_msc_driver) == ERROR_NONE); check(driver_remove_destruct(&esp32_usbhost_midi_driver) == ERROR_NONE); + check(driver_remove_destruct(&esp32_usbhost_hid_keyboard_driver) == ERROR_NONE); check(driver_remove_destruct(&esp32_usbhost_hid_driver) == ERROR_NONE); check(driver_remove_destruct(&esp32_usbhost_driver) == ERROR_NONE); #endif diff --git a/Tactility/Include/Tactility/lvgl/KeyboardDeviceListener.h b/Tactility/Include/Tactility/lvgl/KeyboardDeviceListener.h new file mode 100644 index 000000000..59c250a84 --- /dev/null +++ b/Tactility/Include/Tactility/lvgl/KeyboardDeviceListener.h @@ -0,0 +1,21 @@ +#pragma once + +namespace tt::lvgl { + +/** + * @brief Starts listening for KEYBOARD_TYPE kernel devices being started/stopped at runtime + * (e.g. a USB HID keyboard being plugged in/out) and binds/unbinds each one to LVGL via + * lvgl_keyboard_add()/lvgl_keyboard_remove(). + * + * Devices already started before this is called are picked up by lvgl-module's own boot-time + * scan instead (see Modules/lvgl-module/source/devices/devices.cpp's lvgl_devices_attach()) - + * this only reacts to devices that start/stop afterwards. + */ +void startKeyboardDeviceListener(); + +/** + * @brief Stops listening and unbinds every KEYBOARD_TYPE device this listener bound to LVGL. + */ +void stopKeyboardDeviceListener(); + +} // namespace tt::lvgl diff --git a/Tactility/Source/Tactility.cpp b/Tactility/Source/Tactility.cpp index 5c1181dde..d2e239edd 100644 --- a/Tactility/Source/Tactility.cpp +++ b/Tactility/Source/Tactility.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -441,6 +442,7 @@ static void onLvglStarted() { #endif lvgl::startUsbHidInput(); + lvgl::startKeyboardDeviceListener(); lvgl::initTrackball(); #ifdef CONFIG_TT_TOUCH_CALIBRATION_SUPPORTED @@ -457,6 +459,7 @@ static void onLvglStopped() { module_stop(&lvgl_window_manager_module); + lvgl::stopKeyboardDeviceListener(); lvgl::stopUsbHidInput(); #if TT_FEATURE_SCREENSHOT_ENABLED diff --git a/Tactility/Source/lvgl/KeyboardDeviceListener.cpp b/Tactility/Source/lvgl/KeyboardDeviceListener.cpp new file mode 100644 index 000000000..c46b9042a --- /dev/null +++ b/Tactility/Source/lvgl/KeyboardDeviceListener.cpp @@ -0,0 +1,115 @@ +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +namespace tt::lvgl { + +constexpr auto* TAG = "KeyboardDeviceListener"; + +namespace { + +struct KeyboardBinding { + Device* device; + lv_indev_t* indev; +}; + +Mutex& bindingsMutex() { + static Mutex mutex; + return mutex; +} + +std::vector& bindings() { + static std::vector list; + return list; +} + +void onKeyboardDeviceStarted(Device* device) { + lv_indev_t* indev = nullptr; + lvgl_lock(); + error_t error = lvgl_keyboard_add(device, lv_display_get_default(), &indev); + lvgl_unlock(); + if (error != ERROR_NONE) { + LOG_E(TAG, "failed to bind keyboard device %s to LVGL", device->name); + return; + } + + auto lock = bindingsMutex().asScopedLock(); + lock.lock(); + bindings().push_back({ device, indev }); +} + +void onKeyboardDeviceStopped(Device* device) { + lv_indev_t* indev = nullptr; + { + auto lock = bindingsMutex().asScopedLock(); + lock.lock(); + auto& list = bindings(); + for (auto it = list.begin(); it != list.end(); ++it) { + if (it->device == device) { + indev = it->indev; + list.erase(it); + break; + } + } + } + if (indev == nullptr) { + return; + } + + lvgl_lock(); + lvgl_keyboard_remove(indev); + lvgl_unlock(); +} + +void onDeviceEvent(Device* device, DeviceEvent event, void* context) { + (void)context; + if (device_get_type(device) != &KEYBOARD_TYPE) { + return; + } + // Detach on STOPPING (before stop_device() frees the device's resources), not STOPPED (which + // only fires after stop_device() already ran) - see DEVICE_EVENT_STOPPING's doc comment. + if (event == DEVICE_EVENT_STARTED) { + onKeyboardDeviceStarted(device); + } else if (event == DEVICE_EVENT_STOPPING) { + onKeyboardDeviceStopped(device); + } +} + +} // namespace + +void startKeyboardDeviceListener() { + device_listener_add(onDeviceEvent, nullptr); +} + +void stopKeyboardDeviceListener() { + device_listener_remove(onDeviceEvent); + + std::vector remaining; + { + auto lock = bindingsMutex().asScopedLock(); + lock.lock(); + remaining = std::move(bindings()); + bindings().clear(); + } + + if (remaining.empty()) { + return; + } + + lvgl_lock(); + for (auto& binding : remaining) { + lvgl_keyboard_remove(binding.indev); + } + lvgl_unlock(); +} + +} // namespace tt::lvgl diff --git a/TactilityKernel/include/tactility/device_listener.h b/TactilityKernel/include/tactility/device_listener.h index 94001cb78..9fcd9098c 100644 --- a/TactilityKernel/include/tactility/device_listener.h +++ b/TactilityKernel/include/tactility/device_listener.h @@ -8,6 +8,7 @@ struct Device; enum DeviceEvent { DEVICE_EVENT_STARTED, + DEVICE_EVENT_STOPPING, DEVICE_EVENT_STOPPED, }; diff --git a/TactilityKernel/source/device.cpp b/TactilityKernel/source/device.cpp index baa4af77d..b852a68e6 100644 --- a/TactilityKernel/source/device.cpp +++ b/TactilityKernel/source/device.cpp @@ -256,6 +256,8 @@ error_t device_stop(Device* device) { internal->state.stopping = true; unlock_internal(internal); + device_listener_notify(device, DEVICE_EVENT_STOPPING); + error_t unbind_error = driver_unbind(internal->driver, device); lock_internal(internal);