Create proper keyboard device when keyboard HID is attached

+ fix issue with software keyboard popping up when hardware keyboard was
present

Modules/lvgl-module/source/devices/keyboard.cpp

@@ -4,9 +4,12 @@
 #include <lvgl/lvgl.h>
 
 #include <tactility/drivers/keyboard.h>
+#include <tactility/log.h>
 
 #include <vector>
 
+constexpr auto* TAG = "lvgl_keyboard";
+
 static LvglSoftwareKeyboard last_software_keyboard = {
     .object = nullptr
 };
@@ -106,16 +109,23 @@ void lvgl_keyboard_disable(lv_indev_t* indev) {
     lv_indev_set_group(indev, nullptr);
 }
 
-bool lvgl_hardware_keyboard_is_available() {
+static bool lvgl_hardware_keyboard_check_present(Device* device, void* context) {
-    Device* keyboard_device;
+    bool ready = device_is_ready(device);
-    if (device_get_first_active_by_type(&KEYBOARD_TYPE, &keyboard_device) != ERROR_NONE) {
+    bool present = ready && keyboard_is_present(device);
-        return false;
+    LOG_D(TAG, "keyboard device %s: ready=%d present=%d", device->name, (int)ready, (int)present);
+    if (!present) {
+        return true; // keep looking
     }
+    *static_cast<bool*>(context) = true;
+    return false; // found one, stop iterating
+}
 
+bool lvgl_hardware_keyboard_is_available() {
     // TODO: Refactor the driver subsystem to so it does proper probing/releasing of such devices
     // This work-around exists for the Tab5 keyboard driver.
-    bool present = keyboard_is_present(keyboard_device);
+    bool present = false;
-    device_put(keyboard_device);
+    device_for_each_of_type(&KEYBOARD_TYPE, &present, lvgl_hardware_keyboard_check_present);
+    LOG_D(TAG, "lvgl_hardware_keyboard_is_available() -> %d", (int)present);
     return present;
 }
 
@@ -138,6 +148,11 @@ void lvgl_hardware_keyboard_remove_custom(lv_indev_t* indev) {
 }
 
 static void textarea_show_keyboard(lv_event_t* event) {
+    // Re-checked here rather than gated once at lvgl_keyboard_add_textarea() time, so a hardware
+    // keyboard that connects/disconnects after the textarea was created is honored immediately.
+    if (!lvgl_software_keyboard_is_enabled()) {
+        return;
+    }
     lv_obj_t* target = lv_event_get_current_target_obj(event);
     if (last_software_keyboard.object != nullptr) {
         lvgl_software_keyboard_show(&last_software_keyboard, target);
@@ -192,18 +207,10 @@ LvglSoftwareKeyboard* lvgl_software_keyboard_get_last() {
 }
 
 void lvgl_keyboard_add_textarea(LvglSoftwareKeyboard* keyboard, lv_obj_t* textarea) {
-    // Only the on-screen keyboard's show/hide wiring is specific to "no hardware keyboard"
+    lv_obj_add_event_cb(textarea, textarea_show_keyboard, LV_EVENT_FOCUSED, nullptr);
-    // mode. Group membership must NOT be gated on it: a hardware keypad indev (see
+    lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_DEFOCUSED, nullptr);
-    // lvgl_keyboard_enable()/lvgl_software_keyboard_activate()) is bound to keyboard_group
+    lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_READY, nullptr);
-    // regardless of whether a software keyboard is in use, so skipping lv_group_add_obj()
+    lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_DELETE, nullptr);
-    // here left every textarea unreachable from a hardware keyboard - it was never a member
-    // of the group its indev delivers key events through.
-    if (lvgl_software_keyboard_is_enabled()) {
-        lv_obj_add_event_cb(textarea, textarea_show_keyboard, LV_EVENT_FOCUSED, nullptr);
-        lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_DEFOCUSED, nullptr);
-        lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_READY, nullptr);
-        lv_obj_add_event_cb(textarea, textarea_hide_keyboard, LV_EVENT_DELETE, nullptr);
-    }
 
     // lv_obj_t auto-remove themselves from the group when they are destroyed (last checked in LVGL 8.3)
     lv_group_add_obj(keyboard_group, textarea);

Platforms/platform-esp32/source/drivers/usb/esp32_usbhost_hid.cpp

@@ -3,6 +3,7 @@
 
 #include <tactility/device.h>
 #include <tactility/driver.h>
+#include <tactility/drivers/keyboard.h>
 #include <tactility/drivers/usb_host_hid.h>
 #include <tactility/log.h>
 
@@ -26,6 +27,7 @@ constexpr auto HID_PROC_TASK_STACK    = 4096;
 constexpr auto HID_PROC_TASK_PRIORITY = 5;
 constexpr auto HID_STOP_TIMEOUT_MS    = 2000;
 constexpr auto MAX_SUBSCRIBERS        = 4;
+constexpr auto USB_HID_KB_QUEUE_SIZE  = 16;
 
 typedef struct {
     hid_host_device_handle_t handle;
@@ -53,8 +55,20 @@ struct UsbHidContext {
 
     QueueHandle_t    subscribers[MAX_SUBSCRIBERS] = {};
     SemaphoreHandle_t sub_mutex                   = nullptr;
+
+    // The controller Device itself (set in start_device()), used as the parent for kb_device.
+    Device* controller_device = nullptr;
+    // Dynamic KEYBOARD_TYPE child device, constructed while a physical USB keyboard is connected.
+    Device kb_device{};
+    bool kb_device_active = false;
 };
 
+extern "C" {
+static void usb_hid_keyboard_device_construct(UsbHidContext* ctx);
+static void usb_hid_keyboard_device_destruct(UsbHidContext* ctx);
+static void usb_hid_keyboard_publish_key(UsbHidContext* ctx, uint32_t lv_key, bool pressed, bool ctrl, bool alt, uint8_t hid_keycode, uint8_t hid_modifier);
+}
+
 static const uint8_t keycode2ascii[57][2] = {
     {0, 0}, {0, 0}, {0, 0}, {0, 0},
     {'a', 'A'}, {'b', 'B'}, {'c', 'C'}, {'d', 'D'}, {'e', 'E'},
@@ -182,8 +196,7 @@ static void hid_interface_callback(hid_host_device_handle_t handle,
                         uint32_t lv_key = ctx->pressed_lv_keys[prev_hid];
                         ctx->pressed_lv_keys[prev_hid] = 0;
                         if (lv_key) {
-                            UsbHidEvent evt = { .type = USB_HID_EVENT_KEY, .key = { lv_key, false, with_ctrl, with_alt } };
+                            usb_hid_keyboard_publish_key(ctx, lv_key, false, with_ctrl, with_alt, prev_hid, kb->modifier.val);
-                            publish_event(ctx, &evt);
                         }
                     }
                 }
@@ -221,11 +234,7 @@ static void hid_interface_callback(hid_host_device_handle_t handle,
                         uint32_t lv_key = hid_keycode_to_key(kb->modifier.val, hid_code,
                                                                   ctx->caps_lock_active, ctx->num_lock_active);
                         if (lv_key) {
-                            UsbHidEvent evt = {
+                            usb_hid_keyboard_publish_key(ctx, lv_key, true, with_ctrl, with_alt, hid_code, kb->modifier.val);
-                                .type = USB_HID_EVENT_KEY,
-                                .key = { lv_key, true, with_ctrl, with_alt }
-                            };
-                            publish_event(ctx, &evt);
                             ctx->pressed_lv_keys[hid_code] = lv_key;
                         }
                     }
@@ -271,6 +280,7 @@ static void hid_interface_callback(hid_host_device_handle_t handle,
             memset(ctx->pressed_lv_keys, 0, sizeof(ctx->pressed_lv_keys));
             ctx->kb_handle.store(nullptr);
             ctx->kb_led_pending.store(false);
+            usb_hid_keyboard_device_destruct(ctx);
         } else if (params.proto == HID_PROTOCOL_MOUSE) {
             ctx->mouse_connected = false;
         }
@@ -349,6 +359,7 @@ static void hid_proc_task(void* arg) {
                              | (ctx->caps_lock_active   ? 0x02 : 0)
                              | (ctx->scroll_lock_active ? 0x04 : 0);
                 hid_class_request_set_report(dev_evt.handle, HID_REPORT_TYPE_OUTPUT, 0, &leds, 1);
+                usb_hid_keyboard_device_construct(ctx);
             } else if (params.proto == HID_PROTOCOL_MOUSE) {
                 ctx->mouse_connected = true;
             }
@@ -419,8 +430,122 @@ static const UsbHidApi hid_api = {
 
 extern "C" {
 
+// region Dynamic KEYBOARD_TYPE device
+//
+// While a physical USB keyboard is connected, a KEYBOARD_TYPE child device is constructed so the
+// rest of the system (lvgl_hardware_keyboard_is_available(), Tactility's KeyboardDeviceListener)
+// sees a real hardware keyboard through the same generic device model as any other keyboard, e.g.
+// Devices/m5stack-tab5/Source/devices/tab5_keyboard.cpp. Real key events are delivered exclusively
+// through this device (kb_handle's hid_interface_callback pushes into its queue below); the
+// generic UsbHidEvent publish/subscribe channel above is unaffected for mouse move/button/scroll
+// and the right-click-as-ESC synthesis it already does.
+
+static error_t usb_hid_kb_device_start(Device* device) {
+    auto* queue = xQueueCreate(USB_HID_KB_QUEUE_SIZE, sizeof(KeyboardKeyData));
+    if (queue == nullptr) {
+        return ERROR_RESOURCE;
+    }
+    device_set_driver_data(device, queue);
+    return ERROR_NONE;
+}
+
+static error_t usb_hid_kb_device_stop(Device* device) {
+    auto* queue = static_cast<QueueHandle_t>(device_get_driver_data(device));
+    vQueueDelete(queue);
+    device_set_driver_data(device, nullptr);
+    return ERROR_NONE;
+}
+
+static error_t usb_hid_kb_device_read_key(Device* device, KeyboardKeyData* data) {
+    auto* queue = static_cast<QueueHandle_t>(device_get_driver_data(device));
+    if (queue == nullptr) {
+        *data = {};
+        return ERROR_NONE;
+    }
+    if (xQueueReceive(queue, data, 0) != pdTRUE) {
+        *data = {};
+        return ERROR_NONE;
+    }
+    data->continue_reading = uxQueueMessagesWaiting(queue) > 0;
+    return ERROR_NONE;
+}
+
+static const KeyboardApi esp32_usbhost_hid_keyboard_api = {
+    .read_key = usb_hid_kb_device_read_key,
+    .get_backlight = nullptr,
+    .is_present = nullptr,
+};
+
+Driver esp32_usbhost_hid_keyboard_driver = {
+    .name = "esp32_usbhost_hid_keyboard",
+    .compatible = (const char*[]) { nullptr },
+    .start_device = usb_hid_kb_device_start,
+    .stop_device = usb_hid_kb_device_stop,
+    .api = &esp32_usbhost_hid_keyboard_api,
+    .device_type = &KEYBOARD_TYPE,
+    .owner = nullptr,
+    .internal = nullptr,
+};
+
+static void usb_hid_keyboard_device_construct(UsbHidContext* ctx) {
+    if (ctx->kb_device_active) {
+        return;
+    }
+
+    ctx->kb_device = Device {
+        .address = 0,
+        .name = "usb_keyboard0",
+        .config = nullptr,
+        .parent = nullptr,
+        .internal = nullptr,
+    };
+
+    if (device_construct(&ctx->kb_device) != ERROR_NONE) {
+        LOG_E(TAG, "failed to construct USB keyboard device");
+        return;
+    }
+    device_set_parent(&ctx->kb_device, ctx->controller_device);
+    device_set_driver(&ctx->kb_device, &esp32_usbhost_hid_keyboard_driver);
+    if (device_add(&ctx->kb_device) != ERROR_NONE) {
+        LOG_E(TAG, "failed to add USB keyboard device");
+        device_destruct(&ctx->kb_device);
+        return;
+    }
+    if (device_start(&ctx->kb_device) != ERROR_NONE) {
+        LOG_E(TAG, "failed to start USB keyboard device");
+        device_remove(&ctx->kb_device);
+        device_destruct(&ctx->kb_device);
+        return;
+    }
+
+    ctx->kb_device_active = true;
+}
+
+static void usb_hid_keyboard_device_destruct(UsbHidContext* ctx) {
+    if (!ctx->kb_device_active) {
+        return;
+    }
+    ctx->kb_device_active = false;
+
+    device_stop(&ctx->kb_device);
+    device_remove(&ctx->kb_device);
+    device_destruct(&ctx->kb_device);
+}
+
+static void usb_hid_keyboard_publish_key(UsbHidContext* ctx, uint32_t lv_key, bool pressed, bool ctrl, bool alt, uint8_t hid_keycode, uint8_t hid_modifier) {
+    if (!ctx->kb_device_active) {
+        return;
+    }
+    auto* queue = static_cast<QueueHandle_t>(device_get_driver_data(&ctx->kb_device));
+    KeyboardKeyData data = { lv_key, pressed, false, ctrl, alt, hid_keycode, hid_modifier };
+    xQueueSend(queue, &data, 0);
+}
+
+// endregion
+
 static error_t start_device(struct Device* device) {
     auto* ctx = new UsbHidContext();
+    ctx->controller_device = device;
 
     ctx->sub_mutex = xSemaphoreCreateMutex();
     if (!ctx->sub_mutex) {
@@ -486,6 +611,8 @@ static error_t stop_device(struct Device* device) {
     auto* ctx = static_cast<UsbHidContext*>(device_get_driver_data(device));
     if (!ctx) return ERROR_NONE;
 
+    usb_hid_keyboard_device_destruct(ctx);
+
     ctx->hid_proc_running = false;
 
     if (xSemaphoreTake(ctx->hid_proc_task_done, pdMS_TO_TICKS(HID_STOP_TIMEOUT_MS)) != pdTRUE) {

Platforms/platform-esp32/source/module.cpp

@@ -40,6 +40,7 @@ extern Driver esp32_ble_hid_device_driver;
 #if SOC_USB_OTG_SUPPORTED
 extern Driver esp32_usbhost_driver;
 extern Driver esp32_usbhost_hid_driver;
+extern Driver esp32_usbhost_hid_keyboard_driver;
 extern Driver esp32_usbhost_midi_driver;
 extern Driver esp32_usbhost_msc_driver;
 #endif
@@ -91,6 +92,7 @@ static error_t start() {
 #if SOC_USB_OTG_SUPPORTED
     check(driver_construct_add(&esp32_usbhost_driver) == ERROR_NONE);
     check(driver_construct_add(&esp32_usbhost_hid_driver) == ERROR_NONE);
+    check(driver_construct_add(&esp32_usbhost_hid_keyboard_driver) == ERROR_NONE);
     check(driver_construct_add(&esp32_usbhost_midi_driver) == ERROR_NONE);
     check(driver_construct_add(&esp32_usbhost_msc_driver) == ERROR_NONE);
 #endif
@@ -141,6 +143,7 @@ static error_t stop() {
 #if SOC_USB_OTG_SUPPORTED
     check(driver_remove_destruct(&esp32_usbhost_msc_driver) == ERROR_NONE);
     check(driver_remove_destruct(&esp32_usbhost_midi_driver) == ERROR_NONE);
+    check(driver_remove_destruct(&esp32_usbhost_hid_keyboard_driver) == ERROR_NONE);
     check(driver_remove_destruct(&esp32_usbhost_hid_driver) == ERROR_NONE);
     check(driver_remove_destruct(&esp32_usbhost_driver) == ERROR_NONE);
 #endif

Tactility/Include/Tactility/lvgl/KeyboardDeviceListener.h

@@ -0,0 +1,21 @@
+#pragma once
+
+namespace tt::lvgl {
+
+/**
+ * @brief Starts listening for KEYBOARD_TYPE kernel devices being started/stopped at runtime
+ * (e.g. a USB HID keyboard being plugged in/out) and binds/unbinds each one to LVGL via
+ * lvgl_keyboard_add()/lvgl_keyboard_remove().
+ *
+ * Devices already started before this is called are picked up by lvgl-module's own boot-time
+ * scan instead (see Modules/lvgl-module/source/devices/devices.cpp's lvgl_devices_attach()) -
+ * this only reacts to devices that start/stop afterwards.
+ */
+void startKeyboardDeviceListener();
+
+/**
+ * @brief Stops listening and unbinds every KEYBOARD_TYPE device this listener bound to LVGL.
+ */
+void stopKeyboardDeviceListener();
+
+} // namespace tt::lvgl

Tactility/Source/Tactility.cpp

@@ -25,6 +25,7 @@
 #include <Tactility/bluetooth/Bluetooth.h>
 #include <Tactility/file/File.h>
 #include <Tactility/hal/SdCard.h>
+#include <Tactility/lvgl/KeyboardDeviceListener.h>
 #include <Tactility/lvgl/Statusbar.h>
 #include <Tactility/lvgl/TrackballInit.h>
 #include <Tactility/lvgl/UsbHidInput.h>
@@ -441,6 +442,7 @@ static void onLvglStarted() {
 #endif
 
     lvgl::startUsbHidInput();
+    lvgl::startKeyboardDeviceListener();
     lvgl::initTrackball();
 
 #ifdef CONFIG_TT_TOUCH_CALIBRATION_SUPPORTED
@@ -457,6 +459,7 @@ static void onLvglStopped() {
 
     module_stop(&lvgl_window_manager_module);
 
+    lvgl::stopKeyboardDeviceListener();
     lvgl::stopUsbHidInput();
 
 #if TT_FEATURE_SCREENSHOT_ENABLED

Tactility/Source/lvgl/KeyboardDeviceListener.cpp

@@ -0,0 +1,115 @@
+#include <Tactility/lvgl/KeyboardDeviceListener.h>
+#include <Tactility/Mutex.h>
+
+#include <tactility/device.h>
+#include <tactility/device_listener.h>
+#include <tactility/drivers/keyboard.h>
+#include <tactility/log.h>
+
+#include <lvgl/devices/keyboard.h>
+#include <lvgl/lvgl.h>
+
+#include <vector>
+
+namespace tt::lvgl {
+
+constexpr auto* TAG = "KeyboardDeviceListener";
+
+namespace {
+
+struct KeyboardBinding {
+    Device* device;
+    lv_indev_t* indev;
+};
+
+Mutex& bindingsMutex() {
+    static Mutex mutex;
+    return mutex;
+}
+
+std::vector<KeyboardBinding>& bindings() {
+    static std::vector<KeyboardBinding> list;
+    return list;
+}
+
+void onKeyboardDeviceStarted(Device* device) {
+    lv_indev_t* indev = nullptr;
+    lvgl_lock();
+    error_t error = lvgl_keyboard_add(device, lv_display_get_default(), &indev);
+    lvgl_unlock();
+    if (error != ERROR_NONE) {
+        LOG_E(TAG, "failed to bind keyboard device %s to LVGL", device->name);
+        return;
+    }
+
+    auto lock = bindingsMutex().asScopedLock();
+    lock.lock();
+    bindings().push_back({ device, indev });
+}
+
+void onKeyboardDeviceStopped(Device* device) {
+    lv_indev_t* indev = nullptr;
+    {
+        auto lock = bindingsMutex().asScopedLock();
+        lock.lock();
+        auto& list = bindings();
+        for (auto it = list.begin(); it != list.end(); ++it) {
+            if (it->device == device) {
+                indev = it->indev;
+                list.erase(it);
+                break;
+            }
+        }
+    }
+    if (indev == nullptr) {
+        return;
+    }
+
+    lvgl_lock();
+    lvgl_keyboard_remove(indev);
+    lvgl_unlock();
+}
+
+void onDeviceEvent(Device* device, DeviceEvent event, void* context) {
+    (void)context;
+    if (device_get_type(device) != &KEYBOARD_TYPE) {
+        return;
+    }
+    // Detach on STOPPING (before stop_device() frees the device's resources), not STOPPED (which
+    // only fires after stop_device() already ran) - see DEVICE_EVENT_STOPPING's doc comment.
+    if (event == DEVICE_EVENT_STARTED) {
+        onKeyboardDeviceStarted(device);
+    } else if (event == DEVICE_EVENT_STOPPING) {
+        onKeyboardDeviceStopped(device);
+    }
+}
+
+} // namespace
+
+void startKeyboardDeviceListener() {
+    device_listener_add(onDeviceEvent, nullptr);
+}
+
+void stopKeyboardDeviceListener() {
+    device_listener_remove(onDeviceEvent);
+
+    std::vector<KeyboardBinding> remaining;
+    {
+        auto lock = bindingsMutex().asScopedLock();
+        lock.lock();
+        remaining = std::move(bindings());
+        bindings().clear();
+    }
+
+    if (remaining.empty()) {
+        return;
+    }
+
+    lvgl_lock();
+    for (auto& binding : remaining) {
+        lvgl_keyboard_remove(binding.indev);
+    }
+    lvgl_unlock();
+}
+
+} // namespace tt::lvgl

TactilityKernel/include/tactility/device_listener.h

@@ -8,6 +8,7 @@ struct Device;
 
 enum DeviceEvent {
     DEVICE_EVENT_STARTED,
+    DEVICE_EVENT_STOPPING,
     DEVICE_EVENT_STOPPED,
 };
 

TactilityKernel/source/device.cpp

@@ -256,6 +256,8 @@ error_t device_stop(Device* device) {
     internal->state.stopping = true;
     unlock_internal(internal);
 
+    device_listener_notify(device, DEVICE_EVENT_STOPPING);
+
     error_t unbind_error = driver_unbind(internal->driver, device);
 
     lock_internal(internal);