From 8c30ed6d090be2ad9e03bfc570ee940795488dc9 Mon Sep 17 00:00:00 2001 From: german77 Date: Sat, 1 May 2021 17:02:35 -0500 Subject: hid: Improve hardware accuracy of gestures --- src/core/hle/service/hid/controllers/gesture.cpp | 347 +++++++++++++++++++---- src/core/hle/service/hid/controllers/gesture.h | 69 ++++- 2 files changed, 340 insertions(+), 76 deletions(-) diff --git a/src/core/hle/service/hid/controllers/gesture.cpp b/src/core/hle/service/hid/controllers/gesture.cpp index bb77d8959..9e5df3bb7 100644 --- a/src/core/hle/service/hid/controllers/gesture.cpp +++ b/src/core/hle/service/hid/controllers/gesture.cpp @@ -1,10 +1,9 @@ -// Copyright 2018 yuzu emulator team +// Copyright 2021 yuzu Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. -#include -#include "common/common_types.h" #include "common/logging/log.h" +#include "common/math_util.h" #include "common/settings.h" #include "core/core_timing.h" #include "core/frontend/emu_window.h" @@ -12,10 +11,19 @@ namespace Service::HID { constexpr std::size_t SHARED_MEMORY_OFFSET = 0x3BA00; -constexpr f32 angle_threshold = 0.08f; -constexpr f32 pinch_threshold = 100.0f; -Controller_Gesture::Controller_Gesture(Core::System& system_) : ControllerBase{system_} {} +// HW is around 700, value is set to 400 to make it easier to trigger with mouse +constexpr f32 swipe_threshold = 400.0f; // Threshold in pixels/s +constexpr f32 angle_threshold = 0.015f; // Threshold in radians +constexpr f32 pinch_threshold = 0.5f; // Threshold in pixels +constexpr f32 press_delay = 0.5f; // Time in seconds +constexpr f32 double_tap_delay = 0.35f; // Time in seconds + +constexpr f32 Square(s32 num) { + return static_cast(num * num); +} + +Controller_Gesture::Controller_Gesture(Core::System& system) : ControllerBase(system) {} Controller_Gesture::~Controller_Gesture() = default; void Controller_Gesture::OnInit() { @@ -24,6 +32,8 @@ void Controller_Gesture::OnInit() { keyboard_finger_id[id] = MAX_POINTS; udp_finger_id[id] = MAX_POINTS; } + shared_memory.header.entry_count = 0; + force_update = true; } void Controller_Gesture::OnRelease() {} @@ -38,17 +48,23 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u shared_memory.header.last_entry_index = 0; return; } - shared_memory.header.entry_count = 16; - const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index]; - shared_memory.header.last_entry_index = (shared_memory.header.last_entry_index + 1) % 17; - auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index]; + ReadTouchInput(); - cur_entry.sampling_number = last_entry.sampling_number + 1; - cur_entry.sampling_number2 = cur_entry.sampling_number; + GestureProperties gesture = GetGestureProperties(); + f32 time_difference = static_cast(shared_memory.header.timestamp - last_update_timestamp) / + (1000 * 1000 * 1000); - // TODO(german77): Implement all gesture types + // Only update if necesary + if (!ShouldUpdateGesture(gesture, time_difference)) { + return; + } + last_update_timestamp = shared_memory.header.timestamp; + UpdateGestureSharedMemory(data, size, gesture, time_difference); +} + +void Controller_Gesture::ReadTouchInput() { const Input::TouchStatus& mouse_status = touch_mouse_device->GetStatus(); const Input::TouchStatus& udp_status = touch_udp_device->GetStatus(); for (std::size_t id = 0; id < mouse_status.size(); ++id) { @@ -63,50 +79,71 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u UpdateTouchInputEvent(keyboard_status[id], keyboard_finger_id[id]); } } +} - TouchType type = TouchType::Idle; - Attribute attributes{}; - GestureProperties gesture = GetGestureProperties(); - if (last_gesture.active_points != gesture.active_points) { - ++last_gesture.detection_count; +bool Controller_Gesture::ShouldUpdateGesture(const GestureProperties& gesture, + f32 time_difference) { + const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index]; + if (force_update) { + force_update = false; + return true; } - if (gesture.active_points > 0) { - if (last_gesture.active_points == 0) { - attributes.is_new_touch.Assign(true); - last_gesture.average_distance = gesture.average_distance; - last_gesture.angle = gesture.angle; - } - type = TouchType::Touch; - if (gesture.mid_point.x != last_entry.x || gesture.mid_point.y != last_entry.y) { - type = TouchType::Pan; - } - if (std::abs(gesture.average_distance - last_gesture.average_distance) > pinch_threshold) { - type = TouchType::Pinch; - } - if (std::abs(gesture.angle - last_gesture.angle) > angle_threshold) { - type = TouchType::Rotate; + // Update if coordinates change + for (size_t id = 0; id < MAX_POINTS; id++) { + if (gesture.points[id].x != last_gesture.points[id].x || + gesture.points[id].y != last_gesture.points[id].y) { + return true; } + } + + // Update on press and hold event after 0.5 seconds + if (last_entry.type == TouchType::Touch && last_entry.point_count == 1 && + time_difference > press_delay) { + return enable_press_and_tap; + } + + return false; +} + +void Controller_Gesture::UpdateGestureSharedMemory(u8* data, std::size_t size, + GestureProperties& gesture, + f32 time_difference) { + TouchType type = TouchType::Idle; + Attribute attributes{}; + + const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index]; + shared_memory.header.last_entry_index = (shared_memory.header.last_entry_index + 1) % 17; + auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index]; - cur_entry.delta_x = gesture.mid_point.x - last_entry.x; - cur_entry.delta_y = gesture.mid_point.y - last_entry.y; - // TODO: Find how velocities are calculated - cur_entry.vel_x = static_cast(cur_entry.delta_x) * 150.1f; - cur_entry.vel_y = static_cast(cur_entry.delta_y) * 150.1f; + if (shared_memory.header.entry_count < 16) { + shared_memory.header.entry_count++; + } - // Slowdown the rate of change for less flapping - last_gesture.average_distance = - (last_gesture.average_distance * 0.9f) + (gesture.average_distance * 0.1f); - last_gesture.angle = (last_gesture.angle * 0.9f) + (gesture.angle * 0.1f); + cur_entry.sampling_number = last_entry.sampling_number + 1; + cur_entry.sampling_number2 = cur_entry.sampling_number; + // Reset values to default + cur_entry.delta_x = 0; + cur_entry.delta_y = 0; + cur_entry.vel_x = 0; + cur_entry.vel_y = 0; + cur_entry.direction = Direction::None; + cur_entry.rotation_angle = 0; + cur_entry.scale = 0; + + if (gesture.active_points > 0) { + if (last_gesture.active_points == 0) { + NewGesture(gesture, type, attributes); + } else { + UpdateExistingGesture(gesture, type, time_difference); + } } else { - cur_entry.delta_x = 0; - cur_entry.delta_y = 0; - cur_entry.vel_x = 0; - cur_entry.vel_y = 0; + EndGesture(gesture, last_gesture, type, attributes, time_difference); } - last_gesture.active_points = gesture.active_points; - cur_entry.detection_count = last_gesture.detection_count; + + // Apply attributes + cur_entry.detection_count = gesture.detection_count; cur_entry.type = type; cur_entry.attributes = attributes; cur_entry.x = gesture.mid_point.x; @@ -116,12 +153,190 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u cur_entry.points[id].x = gesture.points[id].x; cur_entry.points[id].y = gesture.points[id].y; } - cur_entry.rotation_angle = 0; - cur_entry.scale = 0; + last_gesture = gesture; std::memcpy(data + SHARED_MEMORY_OFFSET, &shared_memory, sizeof(SharedMemory)); } +void Controller_Gesture::NewGesture(GestureProperties& gesture, TouchType& type, + Attribute& attributes) { + const auto& last_entry = + shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17]; + gesture.detection_count++; + type = TouchType::Touch; + + // New touch after cancel is not considered new + if (last_entry.type != TouchType::Cancel) { + attributes.is_new_touch.Assign(1); + enable_press_and_tap = true; + } +} + +void Controller_Gesture::UpdateExistingGesture(GestureProperties& gesture, TouchType& type, + f32 time_difference) { + const auto& last_entry = + shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17]; + + // Promote to pan type if touch moved + for (size_t id = 0; id < MAX_POINTS; id++) { + if (gesture.points[id].x != last_gesture.points[id].x || + gesture.points[id].y != last_gesture.points[id].y) { + type = TouchType::Pan; + break; + } + } + + // Number of fingers changed cancel the last event and clear data + if (gesture.active_points != last_gesture.active_points) { + type = TouchType::Cancel; + enable_press_and_tap = false; + gesture.active_points = 0; + gesture.mid_point = {}; + for (size_t id = 0; id < MAX_POINTS; id++) { + gesture.points[id].x = 0; + gesture.points[id].y = 0; + } + return; + } + + // Calculate extra parameters of panning + if (type == TouchType::Pan) { + UpdatePanEvent(gesture, last_gesture, type, time_difference); + return; + } + + // Promote to press type + if (last_entry.type == TouchType::Touch) { + type = TouchType::Press; + } +} + +void Controller_Gesture::EndGesture(GestureProperties& gesture, GestureProperties& last_gesture, + TouchType& type, Attribute& attributes, f32 time_difference) { + const auto& last_entry = + shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17]; + if (last_gesture.active_points != 0) { + switch (last_entry.type) { + case TouchType::Touch: + if (enable_press_and_tap) { + SetTapEvent(gesture, last_gesture, type, attributes); + return; + } + type = TouchType::Cancel; + force_update = true; + break; + case TouchType::Press: + case TouchType::Tap: + case TouchType::Swipe: + case TouchType::Pinch: + case TouchType::Rotate: + type = TouchType::Complete; + force_update = true; + break; + case TouchType::Pan: + EndPanEvent(gesture, last_gesture, type, time_difference); + break; + default: + break; + } + return; + } + if (last_entry.type == TouchType::Complete || last_entry.type == TouchType::Cancel) { + gesture.detection_count++; + } +} + +void Controller_Gesture::SetTapEvent(GestureProperties& gesture, GestureProperties& last_gesture, + TouchType& type, Attribute& attributes) { + type = TouchType::Tap; + gesture = last_gesture; + force_update = true; + f32 tap_time_difference = + static_cast(last_update_timestamp - last_tap_timestamp) / (1000 * 1000 * 1000); + last_tap_timestamp = last_update_timestamp; + if (tap_time_difference < double_tap_delay) { + attributes.is_double_tap.Assign(1); + } +} + +void Controller_Gesture::UpdatePanEvent(GestureProperties& gesture, GestureProperties& last_gesture, + TouchType& type, f32 time_difference) { + auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index]; + const auto& last_entry = + shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17]; + cur_entry.delta_x = gesture.mid_point.x - last_entry.x; + cur_entry.delta_y = gesture.mid_point.y - last_entry.y; + + cur_entry.vel_x = static_cast(cur_entry.delta_x) / time_difference; + cur_entry.vel_y = static_cast(cur_entry.delta_y) / time_difference; + last_pan_time_difference = time_difference; + + // Promote to pinch type + if (std::abs(gesture.average_distance - last_gesture.average_distance) > pinch_threshold) { + type = TouchType::Pinch; + cur_entry.scale = gesture.average_distance / last_gesture.average_distance; + } + + const f32 angle_between_two_lines = std::atan((gesture.angle - last_gesture.angle) / + (1 + (gesture.angle * last_gesture.angle))); + // Promote to rotate type + if (std::abs(angle_between_two_lines) > angle_threshold) { + type = TouchType::Rotate; + cur_entry.scale = 0; + cur_entry.rotation_angle = angle_between_two_lines * 180.0f / Common::PI; + } +} + +void Controller_Gesture::EndPanEvent(GestureProperties& gesture, GestureProperties& last_gesture, + TouchType& type, f32 time_difference) { + auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index]; + const auto& last_entry = + shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17]; + cur_entry.vel_x = + static_cast(last_entry.delta_x) / (last_pan_time_difference + time_difference); + cur_entry.vel_y = + static_cast(last_entry.delta_y) / (last_pan_time_difference + time_difference); + const f32 curr_vel = + std::sqrt((cur_entry.vel_x * cur_entry.vel_x) + (cur_entry.vel_y * cur_entry.vel_y)); + + // Set swipe event with parameters + if (curr_vel > swipe_threshold) { + SetSwipeEvent(gesture, last_gesture, type); + return; + } + + // End panning without swipe + type = TouchType::Complete; + cur_entry.vel_x = 0; + cur_entry.vel_y = 0; + force_update = true; +} + +void Controller_Gesture::SetSwipeEvent(GestureProperties& gesture, GestureProperties& last_gesture, + TouchType& type) { + auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index]; + const auto& last_entry = + shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17]; + type = TouchType::Swipe; + gesture = last_gesture; + force_update = true; + cur_entry.delta_x = last_entry.delta_x; + cur_entry.delta_y = last_entry.delta_y; + if (std::abs(cur_entry.delta_x) > std::abs(cur_entry.delta_y)) { + if (cur_entry.delta_x > 0) { + cur_entry.direction = Direction::Right; + return; + } + cur_entry.direction = Direction::Left; + return; + } + if (cur_entry.delta_y > 0) { + cur_entry.direction = Direction::Down; + return; + } + cur_entry.direction = Direction::Up; +} + void Controller_Gesture::OnLoadInputDevices() { touch_mouse_device = Input::CreateDevice("engine:emu_window"); touch_udp_device = Input::CreateDevice("engine:cemuhookudp"); @@ -183,23 +398,33 @@ Controller_Gesture::GestureProperties Controller_Gesture::GetGestureProperties() for (size_t id = 0; id < gesture.active_points; ++id) { gesture.points[id].x = - static_cast(active_fingers[id].x * Layout::ScreenUndocked::Width); + static_cast(active_fingers[id].x * Layout::ScreenUndocked::Width); gesture.points[id].y = - static_cast(active_fingers[id].y * Layout::ScreenUndocked::Height); - gesture.mid_point.x += static_cast(gesture.points[id].x / gesture.active_points); - gesture.mid_point.y += static_cast(gesture.points[id].y / gesture.active_points); + static_cast(active_fingers[id].y * Layout::ScreenUndocked::Height); + + // Hack: There is no touch in docked but games still allow it + if (Settings::values.use_docked_mode.GetValue()) { + gesture.points[id].x = + static_cast(active_fingers[id].x * Layout::ScreenDocked::Width); + gesture.points[id].y = + static_cast(active_fingers[id].y * Layout::ScreenDocked::Height); + } + + gesture.mid_point.x += static_cast(gesture.points[id].x / gesture.active_points); + gesture.mid_point.y += static_cast(gesture.points[id].y / gesture.active_points); } for (size_t id = 0; id < gesture.active_points; ++id) { - const double distance = - std::pow(static_cast(gesture.mid_point.x - gesture.points[id].x), 2) + - std::pow(static_cast(gesture.mid_point.y - gesture.points[id].y), 2); - gesture.average_distance += - static_cast(distance) / static_cast(gesture.active_points); + const f32 distance = std::sqrt(Square(gesture.mid_point.x - gesture.points[id].x) + + Square(gesture.mid_point.y - gesture.points[id].y)); + gesture.average_distance += distance / static_cast(gesture.active_points); } - gesture.angle = std::atan2(static_cast(gesture.mid_point.y - gesture.points[0].y), - static_cast(gesture.mid_point.x - gesture.points[0].x)); + gesture.angle = std::atan2(static_cast(gesture.mid_point.y - gesture.points[0].y), + static_cast(gesture.mid_point.x - gesture.points[0].x)); + + gesture.detection_count = last_gesture.detection_count; + return gesture; } diff --git a/src/core/hle/service/hid/controllers/gesture.h b/src/core/hle/service/hid/controllers/gesture.h index 7c357b977..18110a6ad 100644 --- a/src/core/hle/service/hid/controllers/gesture.h +++ b/src/core/hle/service/hid/controllers/gesture.h @@ -1,4 +1,4 @@ -// Copyright 2018 yuzu emulator team +// Copyright 2021 yuzu Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. @@ -7,7 +7,6 @@ #include #include "common/bit_field.h" #include "common/common_types.h" -#include "common/swap.h" #include "core/frontend/input.h" #include "core/hle/service/hid/controllers/controller_base.h" @@ -35,10 +34,10 @@ private: enum class TouchType : u32 { Idle, // Nothing touching the screen - Complete, // Unknown. End of touch? - Cancel, // Never triggered - Touch, // Pressing without movement - Press, // Never triggered + Complete, // Set at the end of a touch event + Cancel, // Set when the number of fingers change + Touch, // A finger just touched the screen + Press, // Set if last type is touch and the finger hasn't moved Tap, // Fast press then release Pan, // All points moving together across the screen Swipe, // Fast press movement and release of a single point @@ -58,8 +57,8 @@ private: union { u32_le raw{}; - BitField<0, 1, u32> is_new_touch; - BitField<1, 1, u32> is_double_tap; + BitField<4, 1, u32> is_new_touch; + BitField<8, 1, u32> is_double_tap; }; }; static_assert(sizeof(Attribute) == 4, "Attribute is an invalid size"); @@ -73,10 +72,9 @@ private: struct GestureState { s64_le sampling_number; s64_le sampling_number2; - s64_le detection_count; TouchType type; - Direction dir; + Direction direction; s32_le x; s32_le y; s32_le delta_x; @@ -84,8 +82,8 @@ private: f32 vel_x; f32 vel_y; Attribute attributes; - u32 scale; - u32 rotation_angle; + f32 scale; + f32 rotation_angle; s32_le point_count; std::array points; }; @@ -109,10 +107,46 @@ private: Points mid_point{}; s64_le detection_count{}; u64_le delta_time{}; - float average_distance{}; - float angle{}; + f32 average_distance{}; + f32 angle{}; }; + // Reads input from all available input engines + void ReadTouchInput(); + + // Returns true if gesture state needs to be updated + bool ShouldUpdateGesture(const GestureProperties& gesture, f32 time_difference); + + // Updates the shared memory to the next state + void UpdateGestureSharedMemory(u8* data, std::size_t size, GestureProperties& gesture, + f32 time_difference); + + // Initializes new gesture + void NewGesture(GestureProperties& gesture, TouchType& type, Attribute& attributes); + + // Updates existing gesture state + void UpdateExistingGesture(GestureProperties& gesture, TouchType& type, f32 time_difference); + + // Terminates exiting gesture + void EndGesture(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type, + Attribute& attributes, f32 time_difference); + + // Set current event to a tap event + void SetTapEvent(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type, + Attribute& attributes); + + // Calculates and set the extra parameters related to a pan event + void UpdatePanEvent(GestureProperties& gesture, GestureProperties& last_gesture, + TouchType& type, f32 time_difference); + + // Terminates the pan event + void EndPanEvent(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type, + f32 time_difference); + + // Set current event to a swipe event + void SetSwipeEvent(GestureProperties& gesture, GestureProperties& last_gesture, + TouchType& type); + // Returns an unused finger id, if there is no fingers avaliable MAX_FINGERS will be returned std::optional GetUnusedFingerID() const; @@ -134,6 +168,11 @@ private: std::array keyboard_finger_id; std::array udp_finger_id; std::array fingers; - GestureProperties last_gesture; + GestureProperties last_gesture{}; + s64_le last_update_timestamp{}; + s64_le last_tap_timestamp{}; + f32 last_pan_time_difference{}; + bool force_update{false}; + bool enable_press_and_tap{false}; }; } // namespace Service::HID -- cgit v1.2.3