diff options
author | bunnei <bunneidev@gmail.com> | 2023-03-03 23:42:00 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-03-03 23:42:00 +0100 |
commit | 1f98634371838cc94d01613497660937f70ff78b (patch) | |
tree | 7760b21a8fe3a407d49b531994784e43fcd87894 /src/core/hle/kernel | |
parent | ci: Actually enable LTO on MSVC (#9887) (diff) | |
parent | kernel: be more careful about kernel address keys (diff) | |
download | yuzu-1f98634371838cc94d01613497660937f70ff78b.tar yuzu-1f98634371838cc94d01613497660937f70ff78b.tar.gz yuzu-1f98634371838cc94d01613497660937f70ff78b.tar.bz2 yuzu-1f98634371838cc94d01613497660937f70ff78b.tar.lz yuzu-1f98634371838cc94d01613497660937f70ff78b.tar.xz yuzu-1f98634371838cc94d01613497660937f70ff78b.tar.zst yuzu-1f98634371838cc94d01613497660937f70ff78b.zip |
Diffstat (limited to 'src/core/hle/kernel')
-rw-r--r-- | src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp | 21 | ||||
-rw-r--r-- | src/core/hle/kernel/init/init_slab_setup.cpp | 8 | ||||
-rw-r--r-- | src/core/hle/kernel/k_address_arbiter.cpp | 8 | ||||
-rw-r--r-- | src/core/hle/kernel/k_address_space_info.cpp | 79 | ||||
-rw-r--r-- | src/core/hle/kernel/k_condition_variable.cpp | 50 | ||||
-rw-r--r-- | src/core/hle/kernel/k_light_lock.cpp | 8 | ||||
-rw-r--r-- | src/core/hle/kernel/k_process.cpp | 6 | ||||
-rw-r--r-- | src/core/hle/kernel/k_scheduler_lock.h | 11 | ||||
-rw-r--r-- | src/core/hle/kernel/k_thread.cpp | 271 | ||||
-rw-r--r-- | src/core/hle/kernel/k_thread.h | 175 | ||||
-rw-r--r-- | src/core/hle/kernel/kernel.cpp | 93 | ||||
-rw-r--r-- | src/core/hle/kernel/kernel.h | 67 | ||||
-rw-r--r-- | src/core/hle/kernel/svc_types.h | 1 |
13 files changed, 510 insertions, 288 deletions
diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp index c10b7bf30..5b8a248c8 100644 --- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp +++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp @@ -14,9 +14,12 @@ namespace Kernel::Board::Nintendo::Nx { namespace impl { -constexpr const std::size_t RequiredNonSecureSystemMemorySizeVi = 0x2238 * 4 * 1024; -constexpr const std::size_t RequiredNonSecureSystemMemorySizeNvservices = 0x710 * 4 * 1024; -constexpr const std::size_t RequiredNonSecureSystemMemorySizeMisc = 0x80 * 4 * 1024; +using namespace Common::Literals; + +constexpr const std::size_t RequiredNonSecureSystemMemorySizeVi = 0x2280 * 4_KiB; +constexpr const std::size_t RequiredNonSecureSystemMemorySizeViFatal = 0x200 * 4_KiB; +constexpr const std::size_t RequiredNonSecureSystemMemorySizeNvservices = 0x704 * 4_KiB; +constexpr const std::size_t RequiredNonSecureSystemMemorySizeMisc = 0x80 * 4_KiB; } // namespace impl @@ -24,6 +27,9 @@ constexpr const std::size_t RequiredNonSecureSystemMemorySize = impl::RequiredNonSecureSystemMemorySizeVi + impl::RequiredNonSecureSystemMemorySizeNvservices + impl::RequiredNonSecureSystemMemorySizeMisc; +constexpr const std::size_t RequiredNonSecureSystemMemorySizeWithFatal = + RequiredNonSecureSystemMemorySize + impl::RequiredNonSecureSystemMemorySizeViFatal; + namespace { using namespace Common::Literals; @@ -120,10 +126,13 @@ size_t KSystemControl::Init::GetAppletPoolSize() { size_t KSystemControl::Init::GetMinimumNonSecureSystemPoolSize() { // Verify that our minimum is at least as large as Nintendo's. - constexpr size_t MinimumSize = RequiredNonSecureSystemMemorySize; - static_assert(MinimumSize >= 0x29C8000); + constexpr size_t MinimumSizeWithFatal = RequiredNonSecureSystemMemorySizeWithFatal; + static_assert(MinimumSizeWithFatal >= 0x2C04000); + + constexpr size_t MinimumSizeWithoutFatal = RequiredNonSecureSystemMemorySize; + static_assert(MinimumSizeWithoutFatal >= 0x2A00000); - return MinimumSize; + return MinimumSizeWithFatal; } namespace { diff --git a/src/core/hle/kernel/init/init_slab_setup.cpp b/src/core/hle/kernel/init/init_slab_setup.cpp index abdb5639f..5e4090e2b 100644 --- a/src/core/hle/kernel/init/init_slab_setup.cpp +++ b/src/core/hle/kernel/init/init_slab_setup.cpp @@ -33,6 +33,9 @@ namespace Kernel::Init { +// For macro convenience. +using KThreadLockInfo = KThread::LockWithPriorityInheritanceInfo; + #define SLAB_COUNT(CLASS) kernel.SlabResourceCounts().num_##CLASS #define FOREACH_SLAB_TYPE(HANDLER, ...) \ @@ -54,7 +57,8 @@ namespace Kernel::Init { HANDLER(KResourceLimit, (SLAB_COUNT(KResourceLimit)), ##__VA_ARGS__) \ HANDLER(KEventInfo, (SLAB_COUNT(KThread) + SLAB_COUNT(KDebug)), ##__VA_ARGS__) \ HANDLER(KDebug, (SLAB_COUNT(KDebug)), ##__VA_ARGS__) \ - HANDLER(KSecureSystemResource, (SLAB_COUNT(KProcess)), ##__VA_ARGS__) + HANDLER(KSecureSystemResource, (SLAB_COUNT(KProcess)), ##__VA_ARGS__) \ + HANDLER(KThreadLockInfo, (SLAB_COUNT(KThread)), ##__VA_ARGS__) namespace { @@ -131,7 +135,7 @@ VAddr InitializeSlabHeap(Core::System& system, KMemoryLayout& memory_layout, VAd } size_t CalculateSlabHeapGapSize() { - constexpr size_t KernelSlabHeapGapSize = 2_MiB - 320_KiB; + constexpr size_t KernelSlabHeapGapSize = 2_MiB - 356_KiB; static_assert(KernelSlabHeapGapSize <= KernelSlabHeapGapsSizeMax); return KernelSlabHeapGapSize; } diff --git a/src/core/hle/kernel/k_address_arbiter.cpp b/src/core/hle/kernel/k_address_arbiter.cpp index a442a3b98..fb86451ea 100644 --- a/src/core/hle/kernel/k_address_arbiter.cpp +++ b/src/core/hle/kernel/k_address_arbiter.cpp @@ -29,7 +29,9 @@ bool DecrementIfLessThan(Core::System& system, s32* out, VAddr address, s32 valu auto& monitor = system.Monitor(); const auto current_core = system.Kernel().CurrentPhysicalCoreIndex(); - // TODO(bunnei): We should disable interrupts here via KScopedInterruptDisable. + // NOTE: If scheduler lock is not held here, interrupt disable is required. + // KScopedInterruptDisable di; + // TODO(bunnei): We should call CanAccessAtomic(..) here. // Load the value from the address. @@ -59,7 +61,9 @@ bool UpdateIfEqual(Core::System& system, s32* out, VAddr address, s32 value, s32 auto& monitor = system.Monitor(); const auto current_core = system.Kernel().CurrentPhysicalCoreIndex(); - // TODO(bunnei): We should disable interrupts here via KScopedInterruptDisable. + // NOTE: If scheduler lock is not held here, interrupt disable is required. + // KScopedInterruptDisable di; + // TODO(bunnei): We should call CanAccessAtomic(..) here. // Load the value from the address. diff --git a/src/core/hle/kernel/k_address_space_info.cpp b/src/core/hle/kernel/k_address_space_info.cpp index 3e612a207..97972ebae 100644 --- a/src/core/hle/kernel/k_address_space_info.cpp +++ b/src/core/hle/kernel/k_address_space_info.cpp @@ -23,86 +23,33 @@ constexpr std::array<KAddressSpaceInfo, 13> AddressSpaceInfos{{ { .bit_width = 32, .address = Size_Invalid, .size = 1_GiB , .type = KAddressSpaceInfo::Type::Heap, }, { .bit_width = 36, .address = 128_MiB , .size = 2_GiB - 128_MiB, .type = KAddressSpaceInfo::Type::MapSmall, }, { .bit_width = 36, .address = 2_GiB , .size = 64_GiB - 2_GiB , .type = KAddressSpaceInfo::Type::MapLarge, }, - { .bit_width = 36, .address = Size_Invalid, .size = 6_GiB , .type = KAddressSpaceInfo::Type::Heap, }, + { .bit_width = 36, .address = Size_Invalid, .size = 8_GiB , .type = KAddressSpaceInfo::Type::Heap, }, { .bit_width = 36, .address = Size_Invalid, .size = 6_GiB , .type = KAddressSpaceInfo::Type::Alias, }, { .bit_width = 39, .address = 128_MiB , .size = 512_GiB - 128_MiB, .type = KAddressSpaceInfo::Type::Map39Bit, }, { .bit_width = 39, .address = Size_Invalid, .size = 64_GiB , .type = KAddressSpaceInfo::Type::MapSmall }, - { .bit_width = 39, .address = Size_Invalid, .size = 6_GiB , .type = KAddressSpaceInfo::Type::Heap, }, + { .bit_width = 39, .address = Size_Invalid, .size = 8_GiB , .type = KAddressSpaceInfo::Type::Heap, }, { .bit_width = 39, .address = Size_Invalid, .size = 64_GiB , .type = KAddressSpaceInfo::Type::Alias, }, { .bit_width = 39, .address = Size_Invalid, .size = 2_GiB , .type = KAddressSpaceInfo::Type::Stack, }, }}; // clang-format on -constexpr bool IsAllowedIndexForAddress(std::size_t index) { - return index < AddressSpaceInfos.size() && AddressSpaceInfos[index].address != Size_Invalid; -} - -using IndexArray = - std::array<std::size_t, static_cast<std::size_t>(KAddressSpaceInfo::Type::Count)>; - -constexpr IndexArray AddressSpaceIndices32Bit{ - 0, 1, 0, 2, 0, 3, -}; - -constexpr IndexArray AddressSpaceIndices36Bit{ - 4, 5, 4, 6, 4, 7, -}; - -constexpr IndexArray AddressSpaceIndices39Bit{ - 9, 8, 8, 10, 12, 11, -}; - -constexpr bool IsAllowed32BitType(KAddressSpaceInfo::Type type) { - return type < KAddressSpaceInfo::Type::Count && type != KAddressSpaceInfo::Type::Map39Bit && - type != KAddressSpaceInfo::Type::Stack; -} - -constexpr bool IsAllowed36BitType(KAddressSpaceInfo::Type type) { - return type < KAddressSpaceInfo::Type::Count && type != KAddressSpaceInfo::Type::Map39Bit && - type != KAddressSpaceInfo::Type::Stack; -} - -constexpr bool IsAllowed39BitType(KAddressSpaceInfo::Type type) { - return type < KAddressSpaceInfo::Type::Count && type != KAddressSpaceInfo::Type::MapLarge; +const KAddressSpaceInfo& GetAddressSpaceInfo(size_t width, KAddressSpaceInfo::Type type) { + for (auto& info : AddressSpaceInfos) { + if (info.bit_width == width && info.type == type) { + return info; + } + } + UNREACHABLE_MSG("Could not find AddressSpaceInfo"); } } // namespace -u64 KAddressSpaceInfo::GetAddressSpaceStart(std::size_t width, Type type) { - const std::size_t index{static_cast<std::size_t>(type)}; - switch (width) { - case 32: - ASSERT(IsAllowed32BitType(type)); - ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices32Bit[index])); - return AddressSpaceInfos[AddressSpaceIndices32Bit[index]].address; - case 36: - ASSERT(IsAllowed36BitType(type)); - ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices36Bit[index])); - return AddressSpaceInfos[AddressSpaceIndices36Bit[index]].address; - case 39: - ASSERT(IsAllowed39BitType(type)); - ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices39Bit[index])); - return AddressSpaceInfos[AddressSpaceIndices39Bit[index]].address; - } - ASSERT(false); - return 0; +uintptr_t KAddressSpaceInfo::GetAddressSpaceStart(size_t width, KAddressSpaceInfo::Type type) { + return GetAddressSpaceInfo(width, type).address; } -std::size_t KAddressSpaceInfo::GetAddressSpaceSize(std::size_t width, Type type) { - const std::size_t index{static_cast<std::size_t>(type)}; - switch (width) { - case 32: - ASSERT(IsAllowed32BitType(type)); - return AddressSpaceInfos[AddressSpaceIndices32Bit[index]].size; - case 36: - ASSERT(IsAllowed36BitType(type)); - return AddressSpaceInfos[AddressSpaceIndices36Bit[index]].size; - case 39: - ASSERT(IsAllowed39BitType(type)); - return AddressSpaceInfos[AddressSpaceIndices39Bit[index]].size; - } - ASSERT(false); - return 0; +size_t KAddressSpaceInfo::GetAddressSpaceSize(size_t width, KAddressSpaceInfo::Type type) { + return GetAddressSpaceInfo(width, type).size; } } // namespace Kernel diff --git a/src/core/hle/kernel/k_condition_variable.cpp b/src/core/hle/kernel/k_condition_variable.cpp index 3f0be1c3f..f40cf92b1 100644 --- a/src/core/hle/kernel/k_condition_variable.cpp +++ b/src/core/hle/kernel/k_condition_variable.cpp @@ -111,36 +111,36 @@ Result KConditionVariable::SignalToAddress(VAddr addr) { KScopedSchedulerLock sl(kernel); // Remove waiter thread. - s32 num_waiters{}; - KThread* next_owner_thread = - owner_thread->RemoveWaiterByKey(std::addressof(num_waiters), addr); + bool has_waiters{}; + KThread* const next_owner_thread = + owner_thread->RemoveUserWaiterByKey(std::addressof(has_waiters), addr); // Determine the next tag. u32 next_value{}; if (next_owner_thread != nullptr) { next_value = next_owner_thread->GetAddressKeyValue(); - if (num_waiters > 1) { + if (has_waiters) { next_value |= Svc::HandleWaitMask; } + } - // Write the value to userspace. - Result result{ResultSuccess}; - if (WriteToUser(system, addr, std::addressof(next_value))) [[likely]] { - result = ResultSuccess; - } else { - result = ResultInvalidCurrentMemory; - } + // Synchronize memory before proceeding. + std::atomic_thread_fence(std::memory_order_seq_cst); - // Signal the next owner thread. - next_owner_thread->EndWait(result); - return result; + // Write the value to userspace. + Result result{ResultSuccess}; + if (WriteToUser(system, addr, std::addressof(next_value))) [[likely]] { + result = ResultSuccess; } else { - // Just write the value to userspace. - R_UNLESS(WriteToUser(system, addr, std::addressof(next_value)), - ResultInvalidCurrentMemory); + result = ResultInvalidCurrentMemory; + } - return ResultSuccess; + // If necessary, signal the next owner thread. + if (next_owner_thread != nullptr) { + next_owner_thread->EndWait(result); } + + R_RETURN(result); } } @@ -198,7 +198,9 @@ void KConditionVariable::SignalImpl(KThread* thread) { u32 prev_tag{}; bool can_access{}; { - // TODO(bunnei): We should disable interrupts here via KScopedInterruptDisable. + // NOTE: If scheduler lock is not held here, interrupt disable is required. + // KScopedInterruptDisable di; + // TODO(bunnei): We should call CanAccessAtomic(..) here. can_access = true; if (can_access) [[likely]] { @@ -245,9 +247,11 @@ void KConditionVariable::Signal(u64 cv_key, s32 count) { (it->GetConditionVariableKey() == cv_key)) { KThread* target_thread = std::addressof(*it); - this->SignalImpl(target_thread); it = thread_tree.erase(it); target_thread->ClearConditionVariable(); + + this->SignalImpl(target_thread); + ++num_waiters; } @@ -277,16 +281,16 @@ Result KConditionVariable::Wait(VAddr addr, u64 key, u32 value, s64 timeout) { // Update the value and process for the next owner. { // Remove waiter thread. - s32 num_waiters{}; + bool has_waiters{}; KThread* next_owner_thread = - cur_thread->RemoveWaiterByKey(std::addressof(num_waiters), addr); + cur_thread->RemoveUserWaiterByKey(std::addressof(has_waiters), addr); // Update for the next owner thread. u32 next_value{}; if (next_owner_thread != nullptr) { // Get the next tag value. next_value = next_owner_thread->GetAddressKeyValue(); - if (num_waiters > 1) { + if (has_waiters) { next_value |= Svc::HandleWaitMask; } diff --git a/src/core/hle/kernel/k_light_lock.cpp b/src/core/hle/kernel/k_light_lock.cpp index d791acbe3..14cb615da 100644 --- a/src/core/hle/kernel/k_light_lock.cpp +++ b/src/core/hle/kernel/k_light_lock.cpp @@ -90,15 +90,15 @@ void KLightLock::UnlockSlowPath(uintptr_t _cur_thread) { KScopedSchedulerLock sl(kernel); // Get the next owner. - s32 num_waiters; - KThread* next_owner = owner_thread->RemoveWaiterByKey( - std::addressof(num_waiters), reinterpret_cast<uintptr_t>(std::addressof(tag))); + bool has_waiters; + KThread* next_owner = owner_thread->RemoveKernelWaiterByKey( + std::addressof(has_waiters), reinterpret_cast<uintptr_t>(std::addressof(tag))); // Pass the lock to the next owner. uintptr_t next_tag = 0; if (next_owner != nullptr) { next_tag = - reinterpret_cast<uintptr_t>(next_owner) | static_cast<uintptr_t>(num_waiters > 1); + reinterpret_cast<uintptr_t>(next_owner) | static_cast<uintptr_t>(has_waiters); next_owner->EndWait(ResultSuccess); diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index d9c1a0eb3..d44f6e921 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -156,9 +156,9 @@ bool KProcess::ReleaseUserException(KThread* thread) { exception_thread = nullptr; // Remove waiter thread. - s32 num_waiters{}; - if (KThread* next = thread->RemoveWaiterByKey( - std::addressof(num_waiters), + bool has_waiters{}; + if (KThread* next = thread->RemoveKernelWaiterByKey( + std::addressof(has_waiters), reinterpret_cast<uintptr_t>(std::addressof(exception_thread))); next != nullptr) { next->EndWait(ResultSuccess); diff --git a/src/core/hle/kernel/k_scheduler_lock.h b/src/core/hle/kernel/k_scheduler_lock.h index 129d60472..13463717f 100644 --- a/src/core/hle/kernel/k_scheduler_lock.h +++ b/src/core/hle/kernel/k_scheduler_lock.h @@ -31,22 +31,23 @@ public: } if (IsLockedByCurrentThread()) { - // If we already own the lock, we can just increment the count. + // If we already own the lock, the lock count should be > 0. + // For debug, ensure this is true. ASSERT(lock_count > 0); - lock_count++; } else { // Otherwise, we want to disable scheduling and acquire the spinlock. SchedulerType::DisableScheduling(kernel); spin_lock.Lock(); - // For debug, ensure that our state is valid. ASSERT(lock_count == 0); ASSERT(owner_thread == nullptr); - // Increment count, take ownership. - lock_count = 1; + // Take ownership of the lock. owner_thread = GetCurrentThreadPointer(kernel); } + + // Increment the lock count. + lock_count++; } void Unlock() { diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp index 599d05947..8c403f5fd 100644 --- a/src/core/hle/kernel/k_thread.cpp +++ b/src/core/hle/kernel/k_thread.cpp @@ -191,7 +191,7 @@ Result KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack light_ipc_data = nullptr; // We're not waiting for a lock, and we haven't disabled migration. - lock_owner = nullptr; + waiting_lock_info = nullptr; num_core_migration_disables = 0; // We have no waiters, but we do have an entrypoint. @@ -341,25 +341,39 @@ void KThread::Finalize() { // Release any waiters. { - ASSERT(lock_owner == nullptr); + ASSERT(waiting_lock_info == nullptr); KScopedSchedulerLock sl{kernel}; - auto it = waiter_list.begin(); - while (it != waiter_list.end()) { - // Get the thread. - KThread* const waiter = std::addressof(*it); + // Check that we have no kernel waiters. + ASSERT(num_kernel_waiters == 0); - // The thread shouldn't be a kernel waiter. - ASSERT(!waiter->GetAddressKeyIsKernel()); + auto it = held_lock_info_list.begin(); + while (it != held_lock_info_list.end()) { + // Get the lock info. + auto* const lock_info = std::addressof(*it); - // Clear the lock owner. - waiter->SetLockOwner(nullptr); + // The lock shouldn't have a kernel waiter. + ASSERT(!lock_info->GetIsKernelAddressKey()); - // Erase the waiter from our list. - it = waiter_list.erase(it); + // Remove all waiters. + while (lock_info->GetWaiterCount() != 0) { + // Get the front waiter. + KThread* const waiter = lock_info->GetHighestPriorityWaiter(); - // Cancel the thread's wait. - waiter->CancelWait(ResultInvalidState, true); + // Remove it from the lock. + if (lock_info->RemoveWaiter(waiter)) { + ASSERT(lock_info->GetWaiterCount() == 0); + } + + // Cancel the thread's wait. + waiter->CancelWait(ResultInvalidState, true); + } + + // Remove the held lock from our list. + it = held_lock_info_list.erase(it); + + // Free the lock info. + LockWithPriorityInheritanceInfo::Free(kernel, lock_info); } } @@ -708,6 +722,24 @@ void KThread::SetBasePriority(s32 value) { RestorePriority(kernel, this); } +KThread* KThread::GetLockOwner() const { + return waiting_lock_info != nullptr ? waiting_lock_info->GetOwner() : nullptr; +} + +void KThread::IncreaseBasePriority(s32 priority_) { + ASSERT(Svc::HighestThreadPriority <= priority_ && priority_ <= Svc::LowestThreadPriority); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); + ASSERT(!this->GetStackParameters().is_pinned); + + // Set our base priority. + if (base_priority > priority_) { + base_priority = priority_; + + // Perform a priority restoration. + RestorePriority(kernel, this); + } +} + void KThread::RequestSuspend(SuspendType type) { KScopedSchedulerLock sl{kernel}; @@ -891,51 +923,89 @@ Result KThread::GetThreadContext3(std::vector<u8>& out) { R_SUCCEED(); } -void KThread::AddWaiterImpl(KThread* thread) { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); +void KThread::AddHeldLock(LockWithPriorityInheritanceInfo* lock_info) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); + + // Set ourselves as the lock's owner. + lock_info->SetOwner(this); - // Find the right spot to insert the waiter. - auto it = waiter_list.begin(); - while (it != waiter_list.end()) { - if (it->GetPriority() > thread->GetPriority()) { - break; + // Add the lock to our held list. + held_lock_info_list.push_front(*lock_info); +} + +KThread::LockWithPriorityInheritanceInfo* KThread::FindHeldLock(VAddr address_key_, + bool is_kernel_address_key_) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); + + // Try to find an existing held lock. + for (auto& held_lock : held_lock_info_list) { + if (held_lock.GetAddressKey() == address_key_ && + held_lock.GetIsKernelAddressKey() == is_kernel_address_key_) { + return std::addressof(held_lock); } - it++; } + return nullptr; +} + +void KThread::AddWaiterImpl(KThread* thread) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); + ASSERT(thread->GetConditionVariableTree() == nullptr); + + // Get the thread's address key. + const auto address_key_ = thread->GetAddressKey(); + const auto is_kernel_address_key_ = thread->GetIsKernelAddressKey(); + // Keep track of how many kernel waiters we have. - if (thread->GetAddressKeyIsKernel()) { + if (is_kernel_address_key_) { ASSERT((num_kernel_waiters++) >= 0); KScheduler::SetSchedulerUpdateNeeded(kernel); } - // Insert the waiter. - waiter_list.insert(it, *thread); - thread->SetLockOwner(this); + // Get the relevant lock info. + auto* lock_info = this->FindHeldLock(address_key_, is_kernel_address_key_); + if (lock_info == nullptr) { + // Create a new lock for the address key. + lock_info = + LockWithPriorityInheritanceInfo::Create(kernel, address_key_, is_kernel_address_key_); + + // Add the new lock to our list. + this->AddHeldLock(lock_info); + } + + // Add the thread as waiter to the lock info. + lock_info->AddWaiter(thread); } void KThread::RemoveWaiterImpl(KThread* thread) { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); // Keep track of how many kernel waiters we have. - if (thread->GetAddressKeyIsKernel()) { + if (thread->GetIsKernelAddressKey()) { ASSERT((num_kernel_waiters--) > 0); KScheduler::SetSchedulerUpdateNeeded(kernel); } + // Get the info for the lock the thread is waiting on. + auto* lock_info = thread->GetWaitingLockInfo(); + ASSERT(lock_info->GetOwner() == this); + // Remove the waiter. - waiter_list.erase(waiter_list.iterator_to(*thread)); - thread->SetLockOwner(nullptr); + if (lock_info->RemoveWaiter(thread)) { + held_lock_info_list.erase(held_lock_info_list.iterator_to(*lock_info)); + LockWithPriorityInheritanceInfo::Free(kernel, lock_info); + } } -void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) { - ASSERT(kernel_ctx.GlobalSchedulerContext().IsLocked()); +void KThread::RestorePriority(KernelCore& kernel, KThread* thread) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); - while (true) { + while (thread != nullptr) { // We want to inherit priority where possible. s32 new_priority = thread->GetBasePriority(); - if (thread->HasWaiters()) { - new_priority = std::min(new_priority, thread->waiter_list.front().GetPriority()); + for (const auto& held_lock : thread->held_lock_info_list) { + new_priority = + std::min(new_priority, held_lock.GetHighestPriorityWaiter()->GetPriority()); } // If the priority we would inherit is not different from ours, don't do anything. @@ -943,9 +1013,18 @@ void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) { return; } + // Get the owner of whatever lock this thread is waiting on. + KThread* const lock_owner = thread->GetLockOwner(); + + // If the thread is waiting on some lock, remove it as a waiter to prevent violating red + // black tree invariants. + if (lock_owner != nullptr) { + lock_owner->RemoveWaiterImpl(thread); + } + // Ensure we don't violate condition variable red black tree invariants. if (auto* cv_tree = thread->GetConditionVariableTree(); cv_tree != nullptr) { - BeforeUpdatePriority(kernel_ctx, cv_tree, thread); + BeforeUpdatePriority(kernel, cv_tree, thread); } // Change the priority. @@ -954,73 +1033,99 @@ void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) { // Restore the condition variable, if relevant. if (auto* cv_tree = thread->GetConditionVariableTree(); cv_tree != nullptr) { - AfterUpdatePriority(kernel_ctx, cv_tree, thread); + AfterUpdatePriority(kernel, cv_tree, thread); } - // Update the scheduler. - KScheduler::OnThreadPriorityChanged(kernel_ctx, thread, old_priority); - - // Keep the lock owner up to date. - KThread* lock_owner = thread->GetLockOwner(); - if (lock_owner == nullptr) { - return; + // If we removed the thread from some lock's waiting list, add it back. + if (lock_owner != nullptr) { + lock_owner->AddWaiterImpl(thread); } - // Update the thread in the lock owner's sorted list, and continue inheriting. - lock_owner->RemoveWaiterImpl(thread); - lock_owner->AddWaiterImpl(thread); + // Update the scheduler. + KScheduler::OnThreadPriorityChanged(kernel, thread, old_priority); + + // Continue inheriting priority. thread = lock_owner; } } void KThread::AddWaiter(KThread* thread) { - AddWaiterImpl(thread); - RestorePriority(kernel, this); + this->AddWaiterImpl(thread); + + // If the thread has a higher priority than us, we should inherit. + if (thread->GetPriority() < this->GetPriority()) { + RestorePriority(kernel, this); + } } void KThread::RemoveWaiter(KThread* thread) { - RemoveWaiterImpl(thread); - RestorePriority(kernel, this); + this->RemoveWaiterImpl(thread); + + // If our priority is the same as the thread's (and we've inherited), we may need to restore to + // lower priority. + if (this->GetPriority() == thread->GetPriority() && + this->GetPriority() < this->GetBasePriority()) { + RestorePriority(kernel, this); + } } -KThread* KThread::RemoveWaiterByKey(s32* out_num_waiters, VAddr key) { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); +KThread* KThread::RemoveWaiterByKey(bool* out_has_waiters, VAddr key, bool is_kernel_address_key_) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); - s32 num_waiters{}; - KThread* next_lock_owner{}; - auto it = waiter_list.begin(); - while (it != waiter_list.end()) { - if (it->GetAddressKey() == key) { - KThread* thread = std::addressof(*it); - - // Keep track of how many kernel waiters we have. - if (thread->GetAddressKeyIsKernel()) { - ASSERT((num_kernel_waiters--) > 0); - KScheduler::SetSchedulerUpdateNeeded(kernel); - } - it = waiter_list.erase(it); + // Get the relevant lock info. + auto* lock_info = this->FindHeldLock(key, is_kernel_address_key_); + if (lock_info == nullptr) { + *out_has_waiters = false; + return nullptr; + } - // Update the next lock owner. - if (next_lock_owner == nullptr) { - next_lock_owner = thread; - next_lock_owner->SetLockOwner(nullptr); - } else { - next_lock_owner->AddWaiterImpl(thread); - } - num_waiters++; - } else { - it++; + // Remove the lock info from our held list. + held_lock_info_list.erase(held_lock_info_list.iterator_to(*lock_info)); + + // Keep track of how many kernel waiters we have. + if (lock_info->GetIsKernelAddressKey()) { + num_kernel_waiters -= lock_info->GetWaiterCount(); + ASSERT(num_kernel_waiters >= 0); + KScheduler::SetSchedulerUpdateNeeded(kernel); + } + + ASSERT(lock_info->GetWaiterCount() > 0); + + // Remove the highest priority waiter from the lock to be the next owner. + KThread* next_lock_owner = lock_info->GetHighestPriorityWaiter(); + if (lock_info->RemoveWaiter(next_lock_owner)) { + // The new owner was the only waiter. + *out_has_waiters = false; + + // Free the lock info, since it has no waiters. + LockWithPriorityInheritanceInfo::Free(kernel, lock_info); + } else { + // There are additional waiters on the lock. + *out_has_waiters = true; + + // Add the lock to the new owner's held list. + next_lock_owner->AddHeldLock(lock_info); + + // Keep track of any kernel waiters for the new owner. + if (lock_info->GetIsKernelAddressKey()) { + next_lock_owner->num_kernel_waiters += lock_info->GetWaiterCount(); + ASSERT(next_lock_owner->num_kernel_waiters > 0); + + // NOTE: No need to set scheduler update needed, because we will have already done so + // when removing earlier. } } - // Do priority updates, if we have a next owner. - if (next_lock_owner) { + // If our priority is the same as the next owner's (and we've inherited), we may need to restore + // to lower priority. + if (this->GetPriority() == next_lock_owner->GetPriority() && + this->GetPriority() < this->GetBasePriority()) { RestorePriority(kernel, this); - RestorePriority(kernel, next_lock_owner); + // NOTE: No need to restore priority on the next lock owner, because it was already the + // highest priority waiter on the lock. } - // Return output. - *out_num_waiters = num_waiters; + // Return the next lock owner. return next_lock_owner; } @@ -1137,9 +1242,7 @@ ThreadState KThread::RequestTerminate() { } // Change the thread's priority to be higher than any system thread's. - if (this->GetBasePriority() >= Svc::SystemThreadPriorityHighest) { - this->SetBasePriority(TerminatingThreadPriority); - } + this->IncreaseBasePriority(TerminatingThreadPriority); // If the thread is runnable, send a termination interrupt to other cores. if (this->GetState() == ThreadState::Runnable) { diff --git a/src/core/hle/kernel/k_thread.h b/src/core/hle/kernel/k_thread.h index a04de21bc..bd125f5f1 100644 --- a/src/core/hle/kernel/k_thread.h +++ b/src/core/hle/kernel/k_thread.h @@ -339,13 +339,7 @@ public: void SetInterruptFlag(); void ClearInterruptFlag(); - [[nodiscard]] KThread* GetLockOwner() const { - return lock_owner; - } - - void SetLockOwner(KThread* owner) { - lock_owner = owner; - } + KThread* GetLockOwner() const; [[nodiscard]] const KAffinityMask& GetAffinityMask() const { return physical_affinity_mask; @@ -601,7 +595,13 @@ public: [[nodiscard]] Result GetThreadContext3(std::vector<u8>& out); - [[nodiscard]] KThread* RemoveWaiterByKey(s32* out_num_waiters, VAddr key); + [[nodiscard]] KThread* RemoveUserWaiterByKey(bool* out_has_waiters, VAddr key) { + return this->RemoveWaiterByKey(out_has_waiters, key, false); + } + + [[nodiscard]] KThread* RemoveKernelWaiterByKey(bool* out_has_waiters, VAddr key) { + return this->RemoveWaiterByKey(out_has_waiters, key, true); + } [[nodiscard]] VAddr GetAddressKey() const { return address_key; @@ -611,8 +611,8 @@ public: return address_key_value; } - [[nodiscard]] bool GetAddressKeyIsKernel() const { - return address_key_is_kernel; + [[nodiscard]] bool GetIsKernelAddressKey() const { + return is_kernel_address_key; } //! NB: intentional deviation from official kernel. @@ -621,20 +621,17 @@ public: // to cope with arbitrary host pointers making their way // into things. - void SetUserAddressKey(VAddr key) { - address_key = key; - address_key_is_kernel = false; - } - void SetUserAddressKey(VAddr key, u32 val) { + ASSERT(waiting_lock_info == nullptr); address_key = key; address_key_value = val; - address_key_is_kernel = false; + is_kernel_address_key = false; } void SetKernelAddressKey(VAddr key) { + ASSERT(waiting_lock_info == nullptr); address_key = key; - address_key_is_kernel = true; + is_kernel_address_key = true; } void ClearWaitQueue() { @@ -646,10 +643,6 @@ public: void EndWait(Result wait_result_); void CancelWait(Result wait_result_, bool cancel_timer_task); - [[nodiscard]] bool HasWaiters() const { - return !waiter_list.empty(); - } - [[nodiscard]] s32 GetNumKernelWaiters() const { return num_kernel_waiters; } @@ -679,6 +672,9 @@ public: } private: + [[nodiscard]] KThread* RemoveWaiterByKey(bool* out_has_waiters, VAddr key, + bool is_kernel_address_key); + static constexpr size_t PriorityInheritanceCountMax = 10; union SyncObjectBuffer { std::array<KSynchronizationObject*, Svc::ArgumentHandleCountMax> sync_objects{}; @@ -722,13 +718,14 @@ private: }; void AddWaiterImpl(KThread* thread); - void RemoveWaiterImpl(KThread* thread); + static void RestorePriority(KernelCore& kernel, KThread* thread); void StartTermination(); - void FinishTermination(); + void IncreaseBasePriority(s32 priority); + [[nodiscard]] Result Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack_top, s32 prio, s32 virt_core, KProcess* owner, ThreadType type); @@ -737,8 +734,6 @@ private: s32 core, KProcess* owner, ThreadType type, std::function<void()>&& init_func); - static void RestorePriority(KernelCore& kernel_ctx, KThread* thread); - // For core KThread implementation ThreadContext32 thread_context_32{}; ThreadContext64 thread_context_64{}; @@ -749,6 +744,127 @@ private: &KThread::condvar_arbiter_tree_node>; using ConditionVariableThreadTree = ConditionVariableThreadTreeTraits::TreeType<ConditionVariableComparator>; + +private: + struct LockWithPriorityInheritanceComparator { + struct RedBlackKeyType { + s32 m_priority; + + constexpr s32 GetPriority() const { + return m_priority; + } + }; + + template <typename T> + requires(std::same_as<T, KThread> || std::same_as<T, RedBlackKeyType>) + static constexpr int Compare(const T& lhs, const KThread& rhs) { + if (lhs.GetPriority() < rhs.GetPriority()) { + // Sort by priority. + return -1; + } else { + return 1; + } + } + }; + static_assert(std::same_as<Common::RedBlackKeyType<LockWithPriorityInheritanceComparator, void>, + LockWithPriorityInheritanceComparator::RedBlackKeyType>); + + using LockWithPriorityInheritanceThreadTreeTraits = + Common::IntrusiveRedBlackTreeMemberTraitsDeferredAssert< + &KThread::condvar_arbiter_tree_node>; + using LockWithPriorityInheritanceThreadTree = + ConditionVariableThreadTreeTraits::TreeType<LockWithPriorityInheritanceComparator>; + +public: + class LockWithPriorityInheritanceInfo : public KSlabAllocated<LockWithPriorityInheritanceInfo>, + public boost::intrusive::list_base_hook<> { + public: + explicit LockWithPriorityInheritanceInfo(KernelCore&) {} + + static LockWithPriorityInheritanceInfo* Create(KernelCore& kernel, VAddr address_key, + bool is_kernel_address_key) { + // Create a new lock info. + auto* new_lock = LockWithPriorityInheritanceInfo::Allocate(kernel); + ASSERT(new_lock != nullptr); + + // Set the new lock's address key. + new_lock->m_address_key = address_key; + new_lock->m_is_kernel_address_key = is_kernel_address_key; + + return new_lock; + } + + void SetOwner(KThread* new_owner) { + // Set new owner. + m_owner = new_owner; + } + + void AddWaiter(KThread* waiter) { + // Insert the waiter. + m_tree.insert(*waiter); + m_waiter_count++; + + waiter->SetWaitingLockInfo(this); + } + + [[nodiscard]] bool RemoveWaiter(KThread* waiter) { + m_tree.erase(m_tree.iterator_to(*waiter)); + + waiter->SetWaitingLockInfo(nullptr); + + return (--m_waiter_count) == 0; + } + + KThread* GetHighestPriorityWaiter() { + return std::addressof(m_tree.front()); + } + const KThread* GetHighestPriorityWaiter() const { + return std::addressof(m_tree.front()); + } + + LockWithPriorityInheritanceThreadTree& GetThreadTree() { + return m_tree; + } + const LockWithPriorityInheritanceThreadTree& GetThreadTree() const { + return m_tree; + } + + VAddr GetAddressKey() const { + return m_address_key; + } + bool GetIsKernelAddressKey() const { + return m_is_kernel_address_key; + } + KThread* GetOwner() const { + return m_owner; + } + u32 GetWaiterCount() const { + return m_waiter_count; + } + + private: + LockWithPriorityInheritanceThreadTree m_tree{}; + VAddr m_address_key{}; + KThread* m_owner{}; + u32 m_waiter_count{}; + bool m_is_kernel_address_key{}; + }; + + void SetWaitingLockInfo(LockWithPriorityInheritanceInfo* lock) { + waiting_lock_info = lock; + } + + LockWithPriorityInheritanceInfo* GetWaitingLockInfo() { + return waiting_lock_info; + } + + void AddHeldLock(LockWithPriorityInheritanceInfo* lock_info); + LockWithPriorityInheritanceInfo* FindHeldLock(VAddr address_key, bool is_kernel_address_key); + +private: + using LockWithPriorityInheritanceInfoList = + boost::intrusive::list<LockWithPriorityInheritanceInfo>; + ConditionVariableThreadTree* condvar_tree{}; u64 condvar_key{}; u64 virtual_affinity_mask{}; @@ -765,9 +881,9 @@ private: s64 last_scheduled_tick{}; std::array<QueueEntry, Core::Hardware::NUM_CPU_CORES> per_core_priority_queue_entry{}; KThreadQueue* wait_queue{}; - WaiterList waiter_list{}; + LockWithPriorityInheritanceInfoList held_lock_info_list{}; + LockWithPriorityInheritanceInfo* waiting_lock_info{}; WaiterList pinned_waiter_list{}; - KThread* lock_owner{}; u32 address_key_value{}; u32 suspend_request_flags{}; u32 suspend_allowed_flags{}; @@ -791,7 +907,7 @@ private: bool debug_attached{}; s8 priority_inheritance_count{}; bool resource_limit_release_hint{}; - bool address_key_is_kernel{}; + bool is_kernel_address_key{}; StackParameters stack_parameters{}; Common::SpinLock context_guard{}; @@ -814,10 +930,12 @@ public: void SetConditionVariable(ConditionVariableThreadTree* tree, VAddr address, u64 cv_key, u32 value) { + ASSERT(waiting_lock_info == nullptr); condvar_tree = tree; condvar_key = cv_key; address_key = address; address_key_value = value; + is_kernel_address_key = false; } void ClearConditionVariable() { @@ -829,6 +947,7 @@ public: } void SetAddressArbiter(ConditionVariableThreadTree* tree, u64 address) { + ASSERT(waiting_lock_info == nullptr); condvar_tree = tree; condvar_key = address; } diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index ce94d3605..ef7057ff7 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -1318,4 +1318,97 @@ const Core::System& KernelCore::System() const { return impl->system; } +struct KernelCore::SlabHeapContainer { + KSlabHeap<KClientSession> client_session; + KSlabHeap<KEvent> event; + KSlabHeap<KLinkedListNode> linked_list_node; + KSlabHeap<KPort> port; + KSlabHeap<KProcess> process; + KSlabHeap<KResourceLimit> resource_limit; + KSlabHeap<KSession> session; + KSlabHeap<KSharedMemory> shared_memory; + KSlabHeap<KSharedMemoryInfo> shared_memory_info; + KSlabHeap<KThread> thread; + KSlabHeap<KTransferMemory> transfer_memory; + KSlabHeap<KCodeMemory> code_memory; + KSlabHeap<KDeviceAddressSpace> device_address_space; + KSlabHeap<KPageBuffer> page_buffer; + KSlabHeap<KThreadLocalPage> thread_local_page; + KSlabHeap<KObjectName> object_name; + KSlabHeap<KSessionRequest> session_request; + KSlabHeap<KSecureSystemResource> secure_system_resource; + KSlabHeap<KThread::LockWithPriorityInheritanceInfo> lock_info; + KSlabHeap<KEventInfo> event_info; + KSlabHeap<KDebug> debug; +}; + +template <typename T> +KSlabHeap<T>& KernelCore::SlabHeap() { + if constexpr (std::is_same_v<T, KClientSession>) { + return slab_heap_container->client_session; + } else if constexpr (std::is_same_v<T, KEvent>) { + return slab_heap_container->event; + } else if constexpr (std::is_same_v<T, KLinkedListNode>) { + return slab_heap_container->linked_list_node; + } else if constexpr (std::is_same_v<T, KPort>) { + return slab_heap_container->port; + } else if constexpr (std::is_same_v<T, KProcess>) { + return slab_heap_container->process; + } else if constexpr (std::is_same_v<T, KResourceLimit>) { + return slab_heap_container->resource_limit; + } else if constexpr (std::is_same_v<T, KSession>) { + return slab_heap_container->session; + } else if constexpr (std::is_same_v<T, KSharedMemory>) { + return slab_heap_container->shared_memory; + } else if constexpr (std::is_same_v<T, KSharedMemoryInfo>) { + return slab_heap_container->shared_memory_info; + } else if constexpr (std::is_same_v<T, KThread>) { + return slab_heap_container->thread; + } else if constexpr (std::is_same_v<T, KTransferMemory>) { + return slab_heap_container->transfer_memory; + } else if constexpr (std::is_same_v<T, KCodeMemory>) { + return slab_heap_container->code_memory; + } else if constexpr (std::is_same_v<T, KDeviceAddressSpace>) { + return slab_heap_container->device_address_space; + } else if constexpr (std::is_same_v<T, KPageBuffer>) { + return slab_heap_container->page_buffer; + } else if constexpr (std::is_same_v<T, KThreadLocalPage>) { + return slab_heap_container->thread_local_page; + } else if constexpr (std::is_same_v<T, KObjectName>) { + return slab_heap_container->object_name; + } else if constexpr (std::is_same_v<T, KSessionRequest>) { + return slab_heap_container->session_request; + } else if constexpr (std::is_same_v<T, KSecureSystemResource>) { + return slab_heap_container->secure_system_resource; + } else if constexpr (std::is_same_v<T, KThread::LockWithPriorityInheritanceInfo>) { + return slab_heap_container->lock_info; + } else if constexpr (std::is_same_v<T, KEventInfo>) { + return slab_heap_container->event_info; + } else if constexpr (std::is_same_v<T, KDebug>) { + return slab_heap_container->debug; + } +} + +template KSlabHeap<KClientSession>& KernelCore::SlabHeap(); +template KSlabHeap<KEvent>& KernelCore::SlabHeap(); +template KSlabHeap<KLinkedListNode>& KernelCore::SlabHeap(); +template KSlabHeap<KPort>& KernelCore::SlabHeap(); +template KSlabHeap<KProcess>& KernelCore::SlabHeap(); +template KSlabHeap<KResourceLimit>& KernelCore::SlabHeap(); +template KSlabHeap<KSession>& KernelCore::SlabHeap(); +template KSlabHeap<KSharedMemory>& KernelCore::SlabHeap(); +template KSlabHeap<KSharedMemoryInfo>& KernelCore::SlabHeap(); +template KSlabHeap<KThread>& KernelCore::SlabHeap(); +template KSlabHeap<KTransferMemory>& KernelCore::SlabHeap(); +template KSlabHeap<KCodeMemory>& KernelCore::SlabHeap(); +template KSlabHeap<KDeviceAddressSpace>& KernelCore::SlabHeap(); +template KSlabHeap<KPageBuffer>& KernelCore::SlabHeap(); +template KSlabHeap<KThreadLocalPage>& KernelCore::SlabHeap(); +template KSlabHeap<KObjectName>& KernelCore::SlabHeap(); +template KSlabHeap<KSessionRequest>& KernelCore::SlabHeap(); +template KSlabHeap<KSecureSystemResource>& KernelCore::SlabHeap(); +template KSlabHeap<KThread::LockWithPriorityInheritanceInfo>& KernelCore::SlabHeap(); +template KSlabHeap<KEventInfo>& KernelCore::SlabHeap(); +template KSlabHeap<KDebug>& KernelCore::SlabHeap(); + } // namespace Kernel diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h index 4449f6949..1b380a07b 100644 --- a/src/core/hle/kernel/kernel.h +++ b/src/core/hle/kernel/kernel.h @@ -305,49 +305,7 @@ public: /// Gets the slab heap for the specified kernel object type. template <typename T> - KSlabHeap<T>& SlabHeap() { - if constexpr (std::is_same_v<T, KClientSession>) { - return slab_heap_container->client_session; - } else if constexpr (std::is_same_v<T, KEvent>) { - return slab_heap_container->event; - } else if constexpr (std::is_same_v<T, KLinkedListNode>) { - return slab_heap_container->linked_list_node; - } else if constexpr (std::is_same_v<T, KPort>) { - return slab_heap_container->port; - } else if constexpr (std::is_same_v<T, KProcess>) { - return slab_heap_container->process; - } else if constexpr (std::is_same_v<T, KResourceLimit>) { - return slab_heap_container->resource_limit; - } else if constexpr (std::is_same_v<T, KSession>) { - return slab_heap_container->session; - } else if constexpr (std::is_same_v<T, KSharedMemory>) { - return slab_heap_container->shared_memory; - } else if constexpr (std::is_same_v<T, KSharedMemoryInfo>) { - return slab_heap_container->shared_memory_info; - } else if constexpr (std::is_same_v<T, KThread>) { - return slab_heap_container->thread; - } else if constexpr (std::is_same_v<T, KTransferMemory>) { - return slab_heap_container->transfer_memory; - } else if constexpr (std::is_same_v<T, KCodeMemory>) { - return slab_heap_container->code_memory; - } else if constexpr (std::is_same_v<T, KDeviceAddressSpace>) { - return slab_heap_container->device_address_space; - } else if constexpr (std::is_same_v<T, KPageBuffer>) { - return slab_heap_container->page_buffer; - } else if constexpr (std::is_same_v<T, KThreadLocalPage>) { - return slab_heap_container->thread_local_page; - } else if constexpr (std::is_same_v<T, KObjectName>) { - return slab_heap_container->object_name; - } else if constexpr (std::is_same_v<T, KSessionRequest>) { - return slab_heap_container->session_request; - } else if constexpr (std::is_same_v<T, KSecureSystemResource>) { - return slab_heap_container->secure_system_resource; - } else if constexpr (std::is_same_v<T, KEventInfo>) { - return slab_heap_container->event_info; - } else if constexpr (std::is_same_v<T, KDebug>) { - return slab_heap_container->debug; - } - } + KSlabHeap<T>& SlabHeap(); /// Gets the current slab resource counts. Init::KSlabResourceCounts& SlabResourceCounts(); @@ -393,28 +351,7 @@ private: private: /// Helper to encapsulate all slab heaps in a single heap allocated container - struct SlabHeapContainer { - KSlabHeap<KClientSession> client_session; - KSlabHeap<KEvent> event; - KSlabHeap<KLinkedListNode> linked_list_node; - KSlabHeap<KPort> port; - KSlabHeap<KProcess> process; - KSlabHeap<KResourceLimit> resource_limit; - KSlabHeap<KSession> session; - KSlabHeap<KSharedMemory> shared_memory; - KSlabHeap<KSharedMemoryInfo> shared_memory_info; - KSlabHeap<KThread> thread; - KSlabHeap<KTransferMemory> transfer_memory; - KSlabHeap<KCodeMemory> code_memory; - KSlabHeap<KDeviceAddressSpace> device_address_space; - KSlabHeap<KPageBuffer> page_buffer; - KSlabHeap<KThreadLocalPage> thread_local_page; - KSlabHeap<KObjectName> object_name; - KSlabHeap<KSessionRequest> session_request; - KSlabHeap<KSecureSystemResource> secure_system_resource; - KSlabHeap<KEventInfo> event_info; - KSlabHeap<KDebug> debug; - }; + struct SlabHeapContainer; std::unique_ptr<SlabHeapContainer> slab_heap_container; }; diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h index 542c13461..39355d9c4 100644 --- a/src/core/hle/kernel/svc_types.h +++ b/src/core/hle/kernel/svc_types.h @@ -151,6 +151,7 @@ enum class InfoType : u32 { FreeThreadCount = 24, ThreadTickCount = 25, IsSvcPermitted = 26, + IoRegionHint = 27, MesosphereMeta = 65000, MesosphereCurrentProcess = 65001, |