From 65e0178cc09299550aee949d7b89e211017bddee Mon Sep 17 00:00:00 2001 From: bunnei Date: Fri, 12 Feb 2021 17:02:51 -0800 Subject: hle: kernel: Migrate to KMemoryBlock, KMemoryBlockManager, and others. --- src/core/hle/kernel/k_memory_block.h | 332 +++++++++++++++ src/core/hle/kernel/k_memory_block_manager.cpp | 223 ++++++++++ src/core/hle/kernel/k_memory_block_manager.h | 67 +++ src/core/hle/kernel/k_shared_memory.cpp | 12 +- src/core/hle/kernel/k_shared_memory.h | 12 +- src/core/hle/kernel/kernel.cpp | 32 +- src/core/hle/kernel/kernel.h | 6 +- src/core/hle/kernel/memory/memory_block.h | 335 --------------- .../hle/kernel/memory/memory_block_manager.cpp | 223 ---------- src/core/hle/kernel/memory/memory_block_manager.h | 66 --- src/core/hle/kernel/memory/memory_types.h | 18 - src/core/hle/kernel/memory/page_heap.h | 2 +- src/core/hle/kernel/memory/page_linked_list.h | 2 +- src/core/hle/kernel/memory/page_table.cpp | 456 +++++++++++---------- src/core/hle/kernel/memory/page_table.h | 81 ++-- src/core/hle/kernel/memory_types.h | 18 + src/core/hle/kernel/process.cpp | 41 +- src/core/hle/kernel/svc.cpp | 28 +- src/core/hle/kernel/transfer_memory.cpp | 2 +- src/core/hle/kernel/transfer_memory.h | 6 +- 20 files changed, 983 insertions(+), 979 deletions(-) create mode 100644 src/core/hle/kernel/k_memory_block.h create mode 100644 src/core/hle/kernel/k_memory_block_manager.cpp create mode 100644 src/core/hle/kernel/k_memory_block_manager.h delete mode 100644 src/core/hle/kernel/memory/memory_block.h delete mode 100644 src/core/hle/kernel/memory/memory_block_manager.cpp delete mode 100644 src/core/hle/kernel/memory/memory_block_manager.h delete mode 100644 src/core/hle/kernel/memory/memory_types.h create mode 100644 src/core/hle/kernel/memory_types.h (limited to 'src/core/hle/kernel') diff --git a/src/core/hle/kernel/k_memory_block.h b/src/core/hle/kernel/k_memory_block.h new file mode 100644 index 000000000..c5b9c5e85 --- /dev/null +++ b/src/core/hle/kernel/k_memory_block.h @@ -0,0 +1,332 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/common_types.h" +#include "core/hle/kernel/memory_types.h" +#include "core/hle/kernel/svc_types.h" + +namespace Kernel { + +enum class KMemoryState : u32 { + None = 0, + Mask = 0xFF, + All = ~None, + + FlagCanReprotect = (1 << 8), + FlagCanDebug = (1 << 9), + FlagCanUseIpc = (1 << 10), + FlagCanUseNonDeviceIpc = (1 << 11), + FlagCanUseNonSecureIpc = (1 << 12), + FlagMapped = (1 << 13), + FlagCode = (1 << 14), + FlagCanAlias = (1 << 15), + FlagCanCodeAlias = (1 << 16), + FlagCanTransfer = (1 << 17), + FlagCanQueryPhysical = (1 << 18), + FlagCanDeviceMap = (1 << 19), + FlagCanAlignedDeviceMap = (1 << 20), + FlagCanIpcUserBuffer = (1 << 21), + FlagReferenceCounted = (1 << 22), + FlagCanMapProcess = (1 << 23), + FlagCanChangeAttribute = (1 << 24), + FlagCanCodeMemory = (1 << 25), + + FlagsData = FlagCanReprotect | FlagCanUseIpc | FlagCanUseNonDeviceIpc | FlagCanUseNonSecureIpc | + FlagMapped | FlagCanAlias | FlagCanTransfer | FlagCanQueryPhysical | + FlagCanDeviceMap | FlagCanAlignedDeviceMap | FlagCanIpcUserBuffer | + FlagReferenceCounted | FlagCanChangeAttribute, + + FlagsCode = FlagCanDebug | FlagCanUseIpc | FlagCanUseNonDeviceIpc | FlagCanUseNonSecureIpc | + FlagMapped | FlagCode | FlagCanQueryPhysical | FlagCanDeviceMap | + FlagCanAlignedDeviceMap | FlagReferenceCounted, + + FlagsMisc = FlagMapped | FlagReferenceCounted | FlagCanQueryPhysical | FlagCanDeviceMap, + + Free = static_cast(Svc::MemoryState::Free), + Io = static_cast(Svc::MemoryState::Io) | FlagMapped, + Static = static_cast(Svc::MemoryState::Static) | FlagMapped | FlagCanQueryPhysical, + Code = static_cast(Svc::MemoryState::Code) | FlagsCode | FlagCanMapProcess, + CodeData = static_cast(Svc::MemoryState::CodeData) | FlagsData | FlagCanMapProcess | + FlagCanCodeMemory, + Shared = static_cast(Svc::MemoryState::Shared) | FlagMapped | FlagReferenceCounted, + Normal = static_cast(Svc::MemoryState::Normal) | FlagsData | FlagCanCodeMemory, + + AliasCode = static_cast(Svc::MemoryState::AliasCode) | FlagsCode | FlagCanMapProcess | + FlagCanCodeAlias, + AliasCodeData = static_cast(Svc::MemoryState::AliasCodeData) | FlagsData | + FlagCanMapProcess | FlagCanCodeAlias | FlagCanCodeMemory, + + Ipc = static_cast(Svc::MemoryState::Ipc) | FlagsMisc | FlagCanAlignedDeviceMap | + FlagCanUseIpc | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + Stack = static_cast(Svc::MemoryState::Stack) | FlagsMisc | FlagCanAlignedDeviceMap | + FlagCanUseIpc | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + ThreadLocal = + static_cast(Svc::MemoryState::ThreadLocal) | FlagMapped | FlagReferenceCounted, + + Transferred = static_cast(Svc::MemoryState::Transferred) | FlagsMisc | + FlagCanAlignedDeviceMap | FlagCanChangeAttribute | FlagCanUseIpc | + FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + SharedTransferred = static_cast(Svc::MemoryState::SharedTransferred) | FlagsMisc | + FlagCanAlignedDeviceMap | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + SharedCode = static_cast(Svc::MemoryState::SharedCode) | FlagMapped | + FlagReferenceCounted | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + Inaccessible = static_cast(Svc::MemoryState::Inaccessible), + + NonSecureIpc = static_cast(Svc::MemoryState::NonSecureIpc) | FlagsMisc | + FlagCanAlignedDeviceMap | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + NonDeviceIpc = + static_cast(Svc::MemoryState::NonDeviceIpc) | FlagsMisc | FlagCanUseNonDeviceIpc, + + Kernel = static_cast(Svc::MemoryState::Kernel) | FlagMapped, + + GeneratedCode = static_cast(Svc::MemoryState::GeneratedCode) | FlagMapped | + FlagReferenceCounted | FlagCanDebug, + CodeOut = static_cast(Svc::MemoryState::CodeOut) | FlagMapped | FlagReferenceCounted, +}; +DECLARE_ENUM_FLAG_OPERATORS(KMemoryState); + +static_assert(static_cast(KMemoryState::Free) == 0x00000000); +static_assert(static_cast(KMemoryState::Io) == 0x00002001); +static_assert(static_cast(KMemoryState::Static) == 0x00042002); +static_assert(static_cast(KMemoryState::Code) == 0x00DC7E03); +static_assert(static_cast(KMemoryState::CodeData) == 0x03FEBD04); +static_assert(static_cast(KMemoryState::Normal) == 0x037EBD05); +static_assert(static_cast(KMemoryState::Shared) == 0x00402006); +static_assert(static_cast(KMemoryState::AliasCode) == 0x00DD7E08); +static_assert(static_cast(KMemoryState::AliasCodeData) == 0x03FFBD09); +static_assert(static_cast(KMemoryState::Ipc) == 0x005C3C0A); +static_assert(static_cast(KMemoryState::Stack) == 0x005C3C0B); +static_assert(static_cast(KMemoryState::ThreadLocal) == 0x0040200C); +static_assert(static_cast(KMemoryState::Transferred) == 0x015C3C0D); +static_assert(static_cast(KMemoryState::SharedTransferred) == 0x005C380E); +static_assert(static_cast(KMemoryState::SharedCode) == 0x0040380F); +static_assert(static_cast(KMemoryState::Inaccessible) == 0x00000010); +static_assert(static_cast(KMemoryState::NonSecureIpc) == 0x005C3811); +static_assert(static_cast(KMemoryState::NonDeviceIpc) == 0x004C2812); +static_assert(static_cast(KMemoryState::Kernel) == 0x00002013); +static_assert(static_cast(KMemoryState::GeneratedCode) == 0x00402214); +static_assert(static_cast(KMemoryState::CodeOut) == 0x00402015); + +enum class KMemoryPermission : u8 { + None = 0, + Mask = static_cast(~None), + + Read = 1 << 0, + Write = 1 << 1, + Execute = 1 << 2, + + ReadAndWrite = Read | Write, + ReadAndExecute = Read | Execute, + + UserMask = static_cast(Svc::MemoryPermission::Read | Svc::MemoryPermission::Write | + Svc::MemoryPermission::Execute), +}; +DECLARE_ENUM_FLAG_OPERATORS(KMemoryPermission); + +enum class KMemoryAttribute : u8 { + None = 0x00, + Mask = 0x7F, + All = Mask, + DontCareMask = 0x80, + + Locked = static_cast(Svc::MemoryAttribute::Locked), + IpcLocked = static_cast(Svc::MemoryAttribute::IpcLocked), + DeviceShared = static_cast(Svc::MemoryAttribute::DeviceShared), + Uncached = static_cast(Svc::MemoryAttribute::Uncached), + + IpcAndDeviceMapped = IpcLocked | DeviceShared, + LockedAndIpcLocked = Locked | IpcLocked, + DeviceSharedAndUncached = DeviceShared | Uncached +}; +DECLARE_ENUM_FLAG_OPERATORS(KMemoryAttribute); + +static_assert((static_cast(KMemoryAttribute::Mask) & + static_cast(KMemoryAttribute::DontCareMask)) == 0); + +struct KMemoryInfo { + VAddr addr{}; + std::size_t size{}; + KMemoryState state{}; + KMemoryPermission perm{}; + KMemoryAttribute attribute{}; + KMemoryPermission original_perm{}; + u16 ipc_lock_count{}; + u16 device_use_count{}; + + constexpr Svc::MemoryInfo GetSvcMemoryInfo() const { + return { + addr, + size, + static_cast(state & KMemoryState::Mask), + static_cast(attribute & KMemoryAttribute::Mask), + static_cast(perm & KMemoryPermission::UserMask), + ipc_lock_count, + device_use_count, + }; + } + + constexpr VAddr GetAddress() const { + return addr; + } + constexpr std::size_t GetSize() const { + return size; + } + constexpr std::size_t GetNumPages() const { + return GetSize() / PageSize; + } + constexpr VAddr GetEndAddress() const { + return GetAddress() + GetSize(); + } + constexpr VAddr GetLastAddress() const { + return GetEndAddress() - 1; + } +}; + +class KMemoryBlock final { + friend class KMemoryBlockManager; + +private: + VAddr addr{}; + std::size_t num_pages{}; + KMemoryState state{KMemoryState::None}; + u16 ipc_lock_count{}; + u16 device_use_count{}; + KMemoryPermission perm{KMemoryPermission::None}; + KMemoryPermission original_perm{KMemoryPermission::None}; + KMemoryAttribute attribute{KMemoryAttribute::None}; + +public: + static constexpr int Compare(const KMemoryBlock& lhs, const KMemoryBlock& rhs) { + if (lhs.GetAddress() < rhs.GetAddress()) { + return -1; + } else if (lhs.GetAddress() <= rhs.GetLastAddress()) { + return 0; + } else { + return 1; + } + } + +public: + constexpr KMemoryBlock() = default; + constexpr KMemoryBlock(VAddr addr_, std::size_t num_pages_, KMemoryState state_, + KMemoryPermission perm_, KMemoryAttribute attribute_) + : addr{addr_}, num_pages(num_pages_), state{state_}, perm{perm_}, attribute{attribute_} {} + + constexpr VAddr GetAddress() const { + return addr; + } + + constexpr std::size_t GetNumPages() const { + return num_pages; + } + + constexpr std::size_t GetSize() const { + return GetNumPages() * PageSize; + } + + constexpr VAddr GetEndAddress() const { + return GetAddress() + GetSize(); + } + + constexpr VAddr GetLastAddress() const { + return GetEndAddress() - 1; + } + + constexpr KMemoryInfo GetMemoryInfo() const { + return { + GetAddress(), GetSize(), state, perm, + attribute, original_perm, ipc_lock_count, device_use_count, + }; + } + + void ShareToDevice(KMemoryPermission /*new_perm*/) { + ASSERT((attribute & KMemoryAttribute::DeviceShared) == KMemoryAttribute::DeviceShared || + device_use_count == 0); + attribute |= KMemoryAttribute::DeviceShared; + const u16 new_use_count{++device_use_count}; + ASSERT(new_use_count > 0); + } + + void UnshareToDevice(KMemoryPermission /*new_perm*/) { + ASSERT((attribute & KMemoryAttribute::DeviceShared) == KMemoryAttribute::DeviceShared); + const u16 prev_use_count{device_use_count--}; + ASSERT(prev_use_count > 0); + if (prev_use_count == 1) { + attribute &= ~KMemoryAttribute::DeviceShared; + } + } + +private: + constexpr bool HasProperties(KMemoryState s, KMemoryPermission p, KMemoryAttribute a) const { + constexpr KMemoryAttribute AttributeIgnoreMask{KMemoryAttribute::DontCareMask | + KMemoryAttribute::IpcLocked | + KMemoryAttribute::DeviceShared}; + return state == s && perm == p && + (attribute | AttributeIgnoreMask) == (a | AttributeIgnoreMask); + } + + constexpr bool HasSameProperties(const KMemoryBlock& rhs) const { + return state == rhs.state && perm == rhs.perm && original_perm == rhs.original_perm && + attribute == rhs.attribute && ipc_lock_count == rhs.ipc_lock_count && + device_use_count == rhs.device_use_count; + } + + constexpr bool Contains(VAddr start) const { + return GetAddress() <= start && start <= GetEndAddress(); + } + + constexpr void Add(std::size_t count) { + ASSERT(count > 0); + ASSERT(GetAddress() + count * PageSize - 1 < GetEndAddress() + count * PageSize - 1); + + num_pages += count; + } + + constexpr void Update(KMemoryState new_state, KMemoryPermission new_perm, + KMemoryAttribute new_attribute) { + ASSERT(original_perm == KMemoryPermission::None); + ASSERT((attribute & KMemoryAttribute::IpcLocked) == KMemoryAttribute::None); + + state = new_state; + perm = new_perm; + + attribute = static_cast( + new_attribute | + (attribute & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared))); + } + + constexpr KMemoryBlock Split(VAddr split_addr) { + ASSERT(GetAddress() < split_addr); + ASSERT(Contains(split_addr)); + ASSERT(Common::IsAligned(split_addr, PageSize)); + + KMemoryBlock block; + block.addr = addr; + block.num_pages = (split_addr - GetAddress()) / PageSize; + block.state = state; + block.ipc_lock_count = ipc_lock_count; + block.device_use_count = device_use_count; + block.perm = perm; + block.original_perm = original_perm; + block.attribute = attribute; + + addr = split_addr; + num_pages -= block.num_pages; + + return block; + } +}; +static_assert(std::is_trivially_destructible::value); + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_memory_block_manager.cpp b/src/core/hle/kernel/k_memory_block_manager.cpp new file mode 100644 index 000000000..4a2d88008 --- /dev/null +++ b/src/core/hle/kernel/k_memory_block_manager.cpp @@ -0,0 +1,223 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "core/hle/kernel/k_memory_block_manager.h" +#include "core/hle/kernel/memory_types.h" + +namespace Kernel { + +KMemoryBlockManager::KMemoryBlockManager(VAddr start_addr, VAddr end_addr) + : start_addr{start_addr}, end_addr{end_addr} { + const u64 num_pages{(end_addr - start_addr) / PageSize}; + memory_block_tree.emplace_back(start_addr, num_pages, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None); +} + +KMemoryBlockManager::iterator KMemoryBlockManager::FindIterator(VAddr addr) { + auto node{memory_block_tree.begin()}; + while (node != end()) { + const VAddr end_addr{node->GetNumPages() * PageSize + node->GetAddress()}; + if (node->GetAddress() <= addr && end_addr - 1 >= addr) { + return node; + } + node = std::next(node); + } + return end(); +} + +VAddr KMemoryBlockManager::FindFreeArea(VAddr region_start, std::size_t region_num_pages, + std::size_t num_pages, std::size_t align, + std::size_t offset, std::size_t guard_pages) { + if (num_pages == 0) { + return {}; + } + + const VAddr region_end{region_start + region_num_pages * PageSize}; + const VAddr region_last{region_end - 1}; + for (auto it{FindIterator(region_start)}; it != memory_block_tree.cend(); it++) { + const auto info{it->GetMemoryInfo()}; + if (region_last < info.GetAddress()) { + break; + } + + if (info.state != KMemoryState::Free) { + continue; + } + + VAddr area{(info.GetAddress() <= region_start) ? region_start : info.GetAddress()}; + area += guard_pages * PageSize; + + const VAddr offset_area{Common::AlignDown(area, align) + offset}; + area = (area <= offset_area) ? offset_area : offset_area + align; + + const VAddr area_end{area + num_pages * PageSize + guard_pages * PageSize}; + const VAddr area_last{area_end - 1}; + + if (info.GetAddress() <= area && area < area_last && area_last <= region_last && + area_last <= info.GetLastAddress()) { + return area; + } + } + + return {}; +} + +void KMemoryBlockManager::Update(VAddr addr, std::size_t num_pages, KMemoryState prev_state, + KMemoryPermission prev_perm, KMemoryAttribute prev_attribute, + KMemoryState state, KMemoryPermission perm, + KMemoryAttribute attribute) { + const VAddr end_addr{addr + num_pages * PageSize}; + iterator node{memory_block_tree.begin()}; + + prev_attribute |= KMemoryAttribute::IpcAndDeviceMapped; + + while (node != memory_block_tree.end()) { + KMemoryBlock* block{&(*node)}; + iterator next_node{std::next(node)}; + const VAddr cur_addr{block->GetAddress()}; + const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + + if (addr < cur_end_addr && cur_addr < end_addr) { + if (!block->HasProperties(prev_state, prev_perm, prev_attribute)) { + node = next_node; + continue; + } + + iterator new_node{node}; + if (addr > cur_addr) { + memory_block_tree.insert(node, block->Split(addr)); + } + + if (end_addr < cur_end_addr) { + new_node = memory_block_tree.insert(node, block->Split(end_addr)); + } + + new_node->Update(state, perm, attribute); + + MergeAdjacent(new_node, next_node); + } + + if (cur_end_addr - 1 >= end_addr - 1) { + break; + } + + node = next_node; + } +} + +void KMemoryBlockManager::Update(VAddr addr, std::size_t num_pages, KMemoryState state, + KMemoryPermission perm, KMemoryAttribute attribute) { + const VAddr end_addr{addr + num_pages * PageSize}; + iterator node{memory_block_tree.begin()}; + + while (node != memory_block_tree.end()) { + KMemoryBlock* block{&(*node)}; + iterator next_node{std::next(node)}; + const VAddr cur_addr{block->GetAddress()}; + const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + + if (addr < cur_end_addr && cur_addr < end_addr) { + iterator new_node{node}; + + if (addr > cur_addr) { + memory_block_tree.insert(node, block->Split(addr)); + } + + if (end_addr < cur_end_addr) { + new_node = memory_block_tree.insert(node, block->Split(end_addr)); + } + + new_node->Update(state, perm, attribute); + + MergeAdjacent(new_node, next_node); + } + + if (cur_end_addr - 1 >= end_addr - 1) { + break; + } + + node = next_node; + } +} + +void KMemoryBlockManager::UpdateLock(VAddr addr, std::size_t num_pages, LockFunc&& lock_func, + KMemoryPermission perm) { + const VAddr end_addr{addr + num_pages * PageSize}; + iterator node{memory_block_tree.begin()}; + + while (node != memory_block_tree.end()) { + KMemoryBlock* block{&(*node)}; + iterator next_node{std::next(node)}; + const VAddr cur_addr{block->GetAddress()}; + const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + + if (addr < cur_end_addr && cur_addr < end_addr) { + iterator new_node{node}; + + if (addr > cur_addr) { + memory_block_tree.insert(node, block->Split(addr)); + } + + if (end_addr < cur_end_addr) { + new_node = memory_block_tree.insert(node, block->Split(end_addr)); + } + + lock_func(new_node, perm); + + MergeAdjacent(new_node, next_node); + } + + if (cur_end_addr - 1 >= end_addr - 1) { + break; + } + + node = next_node; + } +} + +void KMemoryBlockManager::IterateForRange(VAddr start, VAddr end, IterateFunc&& func) { + const_iterator it{FindIterator(start)}; + KMemoryInfo info{}; + do { + info = it->GetMemoryInfo(); + func(info); + it = std::next(it); + } while (info.addr + info.size - 1 < end - 1 && it != cend()); +} + +void KMemoryBlockManager::MergeAdjacent(iterator it, iterator& next_it) { + KMemoryBlock* block{&(*it)}; + + auto EraseIt = [&](const iterator it_to_erase) { + if (next_it == it_to_erase) { + next_it = std::next(next_it); + } + memory_block_tree.erase(it_to_erase); + }; + + if (it != memory_block_tree.begin()) { + KMemoryBlock* prev{&(*std::prev(it))}; + + if (block->HasSameProperties(*prev)) { + const iterator prev_it{std::prev(it)}; + + prev->Add(block->GetNumPages()); + EraseIt(it); + + it = prev_it; + block = prev; + } + } + + if (it != cend()) { + const KMemoryBlock* const next{&(*std::next(it))}; + + if (block->HasSameProperties(*next)) { + block->Add(next->GetNumPages()); + EraseIt(std::next(it)); + } + } +} + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_memory_block_manager.h b/src/core/hle/kernel/k_memory_block_manager.h new file mode 100644 index 000000000..e11cc70c8 --- /dev/null +++ b/src/core/hle/kernel/k_memory_block_manager.h @@ -0,0 +1,67 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include + +#include "common/common_types.h" +#include "core/hle/kernel/k_memory_block.h" + +namespace Kernel { + +class KMemoryBlockManager final { +public: + using MemoryBlockTree = std::list; + using iterator = MemoryBlockTree::iterator; + using const_iterator = MemoryBlockTree::const_iterator; + +public: + KMemoryBlockManager(VAddr start_addr, VAddr end_addr); + + iterator end() { + return memory_block_tree.end(); + } + const_iterator end() const { + return memory_block_tree.end(); + } + const_iterator cend() const { + return memory_block_tree.cend(); + } + + iterator FindIterator(VAddr addr); + + VAddr FindFreeArea(VAddr region_start, std::size_t region_num_pages, std::size_t num_pages, + std::size_t align, std::size_t offset, std::size_t guard_pages); + + void Update(VAddr addr, std::size_t num_pages, KMemoryState prev_state, + KMemoryPermission prev_perm, KMemoryAttribute prev_attribute, KMemoryState state, + KMemoryPermission perm, KMemoryAttribute attribute); + + void Update(VAddr addr, std::size_t num_pages, KMemoryState state, + KMemoryPermission perm = KMemoryPermission::None, + KMemoryAttribute attribute = KMemoryAttribute::None); + + using LockFunc = std::function; + void UpdateLock(VAddr addr, std::size_t num_pages, LockFunc&& lock_func, + KMemoryPermission perm); + + using IterateFunc = std::function; + void IterateForRange(VAddr start, VAddr end, IterateFunc&& func); + + KMemoryBlock& FindBlock(VAddr addr) { + return *FindIterator(addr); + } + +private: + void MergeAdjacent(iterator it, iterator& next_it); + + [[maybe_unused]] const VAddr start_addr; + [[maybe_unused]] const VAddr end_addr; + + MemoryBlockTree memory_block_tree; +}; + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_shared_memory.cpp b/src/core/hle/kernel/k_shared_memory.cpp index dd82e0217..eb48afe9d 100644 --- a/src/core/hle/kernel/k_shared_memory.cpp +++ b/src/core/hle/kernel/k_shared_memory.cpp @@ -20,8 +20,8 @@ KSharedMemory::~KSharedMemory() { std::shared_ptr KSharedMemory::Create( KernelCore& kernel, Core::DeviceMemory& device_memory, Process* owner_process, - Memory::PageLinkedList&& page_list, Memory::MemoryPermission owner_permission, - Memory::MemoryPermission user_permission, PAddr physical_address, std::size_t size, + Memory::PageLinkedList&& page_list, KMemoryPermission owner_permission, + KMemoryPermission user_permission, PAddr physical_address, std::size_t size, std::string name) { const auto resource_limit = kernel.GetSystemResourceLimit(); @@ -45,21 +45,21 @@ std::shared_ptr KSharedMemory::Create( } ResultCode KSharedMemory::Map(Process& target_process, VAddr address, std::size_t size, - Memory::MemoryPermission permissions) { - const u64 page_count{(size + Memory::PageSize - 1) / Memory::PageSize}; + KMemoryPermission permissions) { + const u64 page_count{(size + PageSize - 1) / PageSize}; if (page_list.GetNumPages() != page_count) { UNIMPLEMENTED_MSG("Page count does not match"); } - const Memory::MemoryPermission expected = + const KMemoryPermission expected = &target_process == owner_process ? owner_permission : user_permission; if (permissions != expected) { UNIMPLEMENTED_MSG("Permission does not match"); } - return target_process.PageTable().MapPages(address, page_list, Memory::MemoryState::Shared, + return target_process.PageTable().MapPages(address, page_list, KMemoryState::Shared, permissions); } diff --git a/src/core/hle/kernel/k_shared_memory.h b/src/core/hle/kernel/k_shared_memory.h index 4d1354415..1ecb4f7dd 100644 --- a/src/core/hle/kernel/k_shared_memory.h +++ b/src/core/hle/kernel/k_shared_memory.h @@ -9,7 +9,7 @@ #include "common/common_types.h" #include "core/device_memory.h" -#include "core/hle/kernel/memory/memory_block.h" +#include "core/hle/kernel/k_memory_block.h" #include "core/hle/kernel/memory/page_linked_list.h" #include "core/hle/kernel/object.h" #include "core/hle/kernel/process.h" @@ -26,8 +26,8 @@ public: static std::shared_ptr Create( KernelCore& kernel, Core::DeviceMemory& device_memory, Process* owner_process, - Memory::PageLinkedList&& page_list, Memory::MemoryPermission owner_permission, - Memory::MemoryPermission user_permission, PAddr physical_address, std::size_t size, + Memory::PageLinkedList&& page_list, KMemoryPermission owner_permission, + KMemoryPermission user_permission, PAddr physical_address, std::size_t size, std::string name); std::string GetTypeName() const override { @@ -51,7 +51,7 @@ public: * @param permissions Memory block map permissions (specified by SVC field) */ ResultCode Map(Process& target_process, VAddr address, std::size_t size, - Memory::MemoryPermission permissions); + KMemoryPermission permissions); /** * Gets a pointer to the shared memory block @@ -77,8 +77,8 @@ private: Core::DeviceMemory& device_memory; Process* owner_process{}; Memory::PageLinkedList page_list; - Memory::MemoryPermission owner_permission{}; - Memory::MemoryPermission user_permission{}; + KMemoryPermission owner_permission{}; + KMemoryPermission user_permission{}; PAddr physical_address{}; std::size_t size{}; std::string name; diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index e4de3f3bf..eab73c46c 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -289,24 +289,24 @@ struct KernelCore::Impl { layout.System().EndAddress()); hid_shared_mem = Kernel::KSharedMemory::Create( - system.Kernel(), system.DeviceMemory(), nullptr, - {hid_addr, hid_size / Memory::PageSize}, Memory::MemoryPermission::None, - Memory::MemoryPermission::Read, hid_addr, hid_size, "HID:SharedMemory"); + system.Kernel(), system.DeviceMemory(), nullptr, {hid_addr, hid_size / PageSize}, + KMemoryPermission::None, KMemoryPermission::Read, hid_addr, hid_size, + "HID:SharedMemory"); font_shared_mem = Kernel::KSharedMemory::Create( - system.Kernel(), system.DeviceMemory(), nullptr, - {font_pa, font_size / Memory::PageSize}, Memory::MemoryPermission::None, - Memory::MemoryPermission::Read, font_pa, font_size, "Font:SharedMemory"); + system.Kernel(), system.DeviceMemory(), nullptr, {font_pa, font_size / PageSize}, + KMemoryPermission::None, KMemoryPermission::Read, font_pa, font_size, + "Font:SharedMemory"); irs_shared_mem = Kernel::KSharedMemory::Create( - system.Kernel(), system.DeviceMemory(), nullptr, - {irs_addr, irs_size / Memory::PageSize}, Memory::MemoryPermission::None, - Memory::MemoryPermission::Read, irs_addr, irs_size, "IRS:SharedMemory"); + system.Kernel(), system.DeviceMemory(), nullptr, {irs_addr, irs_size / PageSize}, + KMemoryPermission::None, KMemoryPermission::Read, irs_addr, irs_size, + "IRS:SharedMemory"); time_shared_mem = Kernel::KSharedMemory::Create( - system.Kernel(), system.DeviceMemory(), nullptr, - {time_addr, time_size / Memory::PageSize}, Memory::MemoryPermission::None, - Memory::MemoryPermission::Read, time_addr, time_size, "Time:SharedMemory"); + system.Kernel(), system.DeviceMemory(), nullptr, {time_addr, time_size / PageSize}, + KMemoryPermission::None, KMemoryPermission::Read, time_addr, time_size, + "Time:SharedMemory"); // Allocate slab heaps - user_slab_heap_pages = std::make_unique>(); + user_slab_heap_pages = std::make_unique>(); constexpr u64 user_slab_heap_size{0x1ef000}; // Reserve slab heaps @@ -349,7 +349,7 @@ struct KernelCore::Impl { // Kernel memory management std::unique_ptr memory_manager; - std::unique_ptr> user_slab_heap_pages; + std::unique_ptr> user_slab_heap_pages; // Shared memory for services std::shared_ptr hid_shared_mem; @@ -581,11 +581,11 @@ const Memory::MemoryManager& KernelCore::MemoryManager() const { return *impl->memory_manager; } -KSlabHeap& KernelCore::GetUserSlabHeapPages() { +KSlabHeap& KernelCore::GetUserSlabHeapPages() { return *impl->user_slab_heap_pages; } -const KSlabHeap& KernelCore::GetUserSlabHeapPages() const { +const KSlabHeap& KernelCore::GetUserSlabHeapPages() const { return *impl->user_slab_heap_pages; } diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h index 5488c962a..498f94417 100644 --- a/src/core/hle/kernel/kernel.h +++ b/src/core/hle/kernel/kernel.h @@ -11,7 +11,7 @@ #include #include "core/arm/cpu_interrupt_handler.h" #include "core/hardware_properties.h" -#include "core/hle/kernel/memory/memory_types.h" +#include "core/hle/kernel/memory_types.h" #include "core/hle/kernel/object.h" namespace Core { @@ -186,10 +186,10 @@ public: const Memory::MemoryManager& MemoryManager() const; /// Gets the slab heap allocated for user space pages. - KSlabHeap& GetUserSlabHeapPages(); + KSlabHeap& GetUserSlabHeapPages(); /// Gets the slab heap allocated for user space pages. - const KSlabHeap& GetUserSlabHeapPages() const; + const KSlabHeap& GetUserSlabHeapPages() const; /// Gets the shared memory object for HID services. Kernel::KSharedMemory& GetHidSharedMem(); diff --git a/src/core/hle/kernel/memory/memory_block.h b/src/core/hle/kernel/memory/memory_block.h deleted file mode 100644 index 83acece1e..000000000 --- a/src/core/hle/kernel/memory/memory_block.h +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -// This file references various implementation details from Atmosphere, an open-source firmware for -// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. - -#pragma once - -#include "common/alignment.h" -#include "common/assert.h" -#include "common/common_types.h" -#include "core/hle/kernel/memory/memory_types.h" -#include "core/hle/kernel/svc_types.h" - -namespace Kernel::Memory { - -enum class MemoryState : u32 { - None = 0, - Mask = 0xFF, - All = ~None, - - FlagCanReprotect = (1 << 8), - FlagCanDebug = (1 << 9), - FlagCanUseIpc = (1 << 10), - FlagCanUseNonDeviceIpc = (1 << 11), - FlagCanUseNonSecureIpc = (1 << 12), - FlagMapped = (1 << 13), - FlagCode = (1 << 14), - FlagCanAlias = (1 << 15), - FlagCanCodeAlias = (1 << 16), - FlagCanTransfer = (1 << 17), - FlagCanQueryPhysical = (1 << 18), - FlagCanDeviceMap = (1 << 19), - FlagCanAlignedDeviceMap = (1 << 20), - FlagCanIpcUserBuffer = (1 << 21), - FlagReferenceCounted = (1 << 22), - FlagCanMapProcess = (1 << 23), - FlagCanChangeAttribute = (1 << 24), - FlagCanCodeMemory = (1 << 25), - - FlagsData = FlagCanReprotect | FlagCanUseIpc | FlagCanUseNonDeviceIpc | FlagCanUseNonSecureIpc | - FlagMapped | FlagCanAlias | FlagCanTransfer | FlagCanQueryPhysical | - FlagCanDeviceMap | FlagCanAlignedDeviceMap | FlagCanIpcUserBuffer | - FlagReferenceCounted | FlagCanChangeAttribute, - - FlagsCode = FlagCanDebug | FlagCanUseIpc | FlagCanUseNonDeviceIpc | FlagCanUseNonSecureIpc | - FlagMapped | FlagCode | FlagCanQueryPhysical | FlagCanDeviceMap | - FlagCanAlignedDeviceMap | FlagReferenceCounted, - - FlagsMisc = FlagMapped | FlagReferenceCounted | FlagCanQueryPhysical | FlagCanDeviceMap, - - Free = static_cast(Svc::MemoryState::Free), - Io = static_cast(Svc::MemoryState::Io) | FlagMapped, - Static = static_cast(Svc::MemoryState::Static) | FlagMapped | FlagCanQueryPhysical, - Code = static_cast(Svc::MemoryState::Code) | FlagsCode | FlagCanMapProcess, - CodeData = static_cast(Svc::MemoryState::CodeData) | FlagsData | FlagCanMapProcess | - FlagCanCodeMemory, - Shared = static_cast(Svc::MemoryState::Shared) | FlagMapped | FlagReferenceCounted, - Normal = static_cast(Svc::MemoryState::Normal) | FlagsData | FlagCanCodeMemory, - - AliasCode = static_cast(Svc::MemoryState::AliasCode) | FlagsCode | FlagCanMapProcess | - FlagCanCodeAlias, - AliasCodeData = static_cast(Svc::MemoryState::AliasCodeData) | FlagsData | - FlagCanMapProcess | FlagCanCodeAlias | FlagCanCodeMemory, - - Ipc = static_cast(Svc::MemoryState::Ipc) | FlagsMisc | FlagCanAlignedDeviceMap | - FlagCanUseIpc | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, - - Stack = static_cast(Svc::MemoryState::Stack) | FlagsMisc | FlagCanAlignedDeviceMap | - FlagCanUseIpc | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, - - ThreadLocal = - static_cast(Svc::MemoryState::ThreadLocal) | FlagMapped | FlagReferenceCounted, - - Transferred = static_cast(Svc::MemoryState::Transferred) | FlagsMisc | - FlagCanAlignedDeviceMap | FlagCanChangeAttribute | FlagCanUseIpc | - FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, - - SharedTransferred = static_cast(Svc::MemoryState::SharedTransferred) | FlagsMisc | - FlagCanAlignedDeviceMap | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, - - SharedCode = static_cast(Svc::MemoryState::SharedCode) | FlagMapped | - FlagReferenceCounted | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, - - Inaccessible = static_cast(Svc::MemoryState::Inaccessible), - - NonSecureIpc = static_cast(Svc::MemoryState::NonSecureIpc) | FlagsMisc | - FlagCanAlignedDeviceMap | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, - - NonDeviceIpc = - static_cast(Svc::MemoryState::NonDeviceIpc) | FlagsMisc | FlagCanUseNonDeviceIpc, - - Kernel = static_cast(Svc::MemoryState::Kernel) | FlagMapped, - - GeneratedCode = static_cast(Svc::MemoryState::GeneratedCode) | FlagMapped | - FlagReferenceCounted | FlagCanDebug, - CodeOut = static_cast(Svc::MemoryState::CodeOut) | FlagMapped | FlagReferenceCounted, -}; -DECLARE_ENUM_FLAG_OPERATORS(MemoryState); - -static_assert(static_cast(MemoryState::Free) == 0x00000000); -static_assert(static_cast(MemoryState::Io) == 0x00002001); -static_assert(static_cast(MemoryState::Static) == 0x00042002); -static_assert(static_cast(MemoryState::Code) == 0x00DC7E03); -static_assert(static_cast(MemoryState::CodeData) == 0x03FEBD04); -static_assert(static_cast(MemoryState::Normal) == 0x037EBD05); -static_assert(static_cast(MemoryState::Shared) == 0x00402006); -static_assert(static_cast(MemoryState::AliasCode) == 0x00DD7E08); -static_assert(static_cast(MemoryState::AliasCodeData) == 0x03FFBD09); -static_assert(static_cast(MemoryState::Ipc) == 0x005C3C0A); -static_assert(static_cast(MemoryState::Stack) == 0x005C3C0B); -static_assert(static_cast(MemoryState::ThreadLocal) == 0x0040200C); -static_assert(static_cast(MemoryState::Transferred) == 0x015C3C0D); -static_assert(static_cast(MemoryState::SharedTransferred) == 0x005C380E); -static_assert(static_cast(MemoryState::SharedCode) == 0x0040380F); -static_assert(static_cast(MemoryState::Inaccessible) == 0x00000010); -static_assert(static_cast(MemoryState::NonSecureIpc) == 0x005C3811); -static_assert(static_cast(MemoryState::NonDeviceIpc) == 0x004C2812); -static_assert(static_cast(MemoryState::Kernel) == 0x00002013); -static_assert(static_cast(MemoryState::GeneratedCode) == 0x00402214); -static_assert(static_cast(MemoryState::CodeOut) == 0x00402015); - -enum class MemoryPermission : u8 { - None = 0, - Mask = static_cast(~None), - - Read = 1 << 0, - Write = 1 << 1, - Execute = 1 << 2, - - ReadAndWrite = Read | Write, - ReadAndExecute = Read | Execute, - - UserMask = static_cast(Svc::MemoryPermission::Read | Svc::MemoryPermission::Write | - Svc::MemoryPermission::Execute), -}; -DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission); - -enum class MemoryAttribute : u8 { - None = 0x00, - Mask = 0x7F, - All = Mask, - DontCareMask = 0x80, - - Locked = static_cast(Svc::MemoryAttribute::Locked), - IpcLocked = static_cast(Svc::MemoryAttribute::IpcLocked), - DeviceShared = static_cast(Svc::MemoryAttribute::DeviceShared), - Uncached = static_cast(Svc::MemoryAttribute::Uncached), - - IpcAndDeviceMapped = IpcLocked | DeviceShared, - LockedAndIpcLocked = Locked | IpcLocked, - DeviceSharedAndUncached = DeviceShared | Uncached -}; -DECLARE_ENUM_FLAG_OPERATORS(MemoryAttribute); - -static_assert((static_cast(MemoryAttribute::Mask) & - static_cast(MemoryAttribute::DontCareMask)) == 0); - -struct MemoryInfo { - VAddr addr{}; - std::size_t size{}; - MemoryState state{}; - MemoryPermission perm{}; - MemoryAttribute attribute{}; - MemoryPermission original_perm{}; - u16 ipc_lock_count{}; - u16 device_use_count{}; - - constexpr Svc::MemoryInfo GetSvcMemoryInfo() const { - return { - addr, - size, - static_cast(state & MemoryState::Mask), - static_cast(attribute & MemoryAttribute::Mask), - static_cast(perm & MemoryPermission::UserMask), - ipc_lock_count, - device_use_count, - }; - } - - constexpr VAddr GetAddress() const { - return addr; - } - constexpr std::size_t GetSize() const { - return size; - } - constexpr std::size_t GetNumPages() const { - return GetSize() / PageSize; - } - constexpr VAddr GetEndAddress() const { - return GetAddress() + GetSize(); - } - constexpr VAddr GetLastAddress() const { - return GetEndAddress() - 1; - } -}; - -class MemoryBlock final { - friend class MemoryBlockManager; - -private: - VAddr addr{}; - std::size_t num_pages{}; - MemoryState state{MemoryState::None}; - u16 ipc_lock_count{}; - u16 device_use_count{}; - MemoryPermission perm{MemoryPermission::None}; - MemoryPermission original_perm{MemoryPermission::None}; - MemoryAttribute attribute{MemoryAttribute::None}; - -public: - static constexpr int Compare(const MemoryBlock& lhs, const MemoryBlock& rhs) { - if (lhs.GetAddress() < rhs.GetAddress()) { - return -1; - } else if (lhs.GetAddress() <= rhs.GetLastAddress()) { - return 0; - } else { - return 1; - } - } - -public: - constexpr MemoryBlock() = default; - constexpr MemoryBlock(VAddr addr_, std::size_t num_pages_, MemoryState state_, - MemoryPermission perm_, MemoryAttribute attribute_) - : addr{addr_}, num_pages(num_pages_), state{state_}, perm{perm_}, attribute{attribute_} {} - - constexpr VAddr GetAddress() const { - return addr; - } - - constexpr std::size_t GetNumPages() const { - return num_pages; - } - - constexpr std::size_t GetSize() const { - return GetNumPages() * PageSize; - } - - constexpr VAddr GetEndAddress() const { - return GetAddress() + GetSize(); - } - - constexpr VAddr GetLastAddress() const { - return GetEndAddress() - 1; - } - - constexpr MemoryInfo GetMemoryInfo() const { - return { - GetAddress(), GetSize(), state, perm, - attribute, original_perm, ipc_lock_count, device_use_count, - }; - } - - void ShareToDevice(MemoryPermission /*new_perm*/) { - ASSERT((attribute & MemoryAttribute::DeviceShared) == MemoryAttribute::DeviceShared || - device_use_count == 0); - attribute |= MemoryAttribute::DeviceShared; - const u16 new_use_count{++device_use_count}; - ASSERT(new_use_count > 0); - } - - void UnshareToDevice(MemoryPermission /*new_perm*/) { - ASSERT((attribute & MemoryAttribute::DeviceShared) == MemoryAttribute::DeviceShared); - const u16 prev_use_count{device_use_count--}; - ASSERT(prev_use_count > 0); - if (prev_use_count == 1) { - attribute &= ~MemoryAttribute::DeviceShared; - } - } - -private: - constexpr bool HasProperties(MemoryState s, MemoryPermission p, MemoryAttribute a) const { - constexpr MemoryAttribute AttributeIgnoreMask{MemoryAttribute::DontCareMask | - MemoryAttribute::IpcLocked | - MemoryAttribute::DeviceShared}; - return state == s && perm == p && - (attribute | AttributeIgnoreMask) == (a | AttributeIgnoreMask); - } - - constexpr bool HasSameProperties(const MemoryBlock& rhs) const { - return state == rhs.state && perm == rhs.perm && original_perm == rhs.original_perm && - attribute == rhs.attribute && ipc_lock_count == rhs.ipc_lock_count && - device_use_count == rhs.device_use_count; - } - - constexpr bool Contains(VAddr start) const { - return GetAddress() <= start && start <= GetEndAddress(); - } - - constexpr void Add(std::size_t count) { - ASSERT(count > 0); - ASSERT(GetAddress() + count * PageSize - 1 < GetEndAddress() + count * PageSize - 1); - - num_pages += count; - } - - constexpr void Update(MemoryState new_state, MemoryPermission new_perm, - MemoryAttribute new_attribute) { - ASSERT(original_perm == MemoryPermission::None); - ASSERT((attribute & MemoryAttribute::IpcLocked) == MemoryAttribute::None); - - state = new_state; - perm = new_perm; - - attribute = static_cast( - new_attribute | - (attribute & (MemoryAttribute::IpcLocked | MemoryAttribute::DeviceShared))); - } - - constexpr MemoryBlock Split(VAddr split_addr) { - ASSERT(GetAddress() < split_addr); - ASSERT(Contains(split_addr)); - ASSERT(Common::IsAligned(split_addr, PageSize)); - - MemoryBlock block; - block.addr = addr; - block.num_pages = (split_addr - GetAddress()) / PageSize; - block.state = state; - block.ipc_lock_count = ipc_lock_count; - block.device_use_count = device_use_count; - block.perm = perm; - block.original_perm = original_perm; - block.attribute = attribute; - - addr = split_addr; - num_pages -= block.num_pages; - - return block; - } -}; -static_assert(std::is_trivially_destructible::value); - -} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_block_manager.cpp b/src/core/hle/kernel/memory/memory_block_manager.cpp deleted file mode 100644 index 0732fa5a1..000000000 --- a/src/core/hle/kernel/memory/memory_block_manager.cpp +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -#include "core/hle/kernel/memory/memory_block_manager.h" -#include "core/hle/kernel/memory/memory_types.h" - -namespace Kernel::Memory { - -MemoryBlockManager::MemoryBlockManager(VAddr start_addr, VAddr end_addr) - : start_addr{start_addr}, end_addr{end_addr} { - const u64 num_pages{(end_addr - start_addr) / PageSize}; - memory_block_tree.emplace_back(start_addr, num_pages, MemoryState::Free, MemoryPermission::None, - MemoryAttribute::None); -} - -MemoryBlockManager::iterator MemoryBlockManager::FindIterator(VAddr addr) { - auto node{memory_block_tree.begin()}; - while (node != end()) { - const VAddr end_addr{node->GetNumPages() * PageSize + node->GetAddress()}; - if (node->GetAddress() <= addr && end_addr - 1 >= addr) { - return node; - } - node = std::next(node); - } - return end(); -} - -VAddr MemoryBlockManager::FindFreeArea(VAddr region_start, std::size_t region_num_pages, - std::size_t num_pages, std::size_t align, std::size_t offset, - std::size_t guard_pages) { - if (num_pages == 0) { - return {}; - } - - const VAddr region_end{region_start + region_num_pages * PageSize}; - const VAddr region_last{region_end - 1}; - for (auto it{FindIterator(region_start)}; it != memory_block_tree.cend(); it++) { - const auto info{it->GetMemoryInfo()}; - if (region_last < info.GetAddress()) { - break; - } - - if (info.state != MemoryState::Free) { - continue; - } - - VAddr area{(info.GetAddress() <= region_start) ? region_start : info.GetAddress()}; - area += guard_pages * PageSize; - - const VAddr offset_area{Common::AlignDown(area, align) + offset}; - area = (area <= offset_area) ? offset_area : offset_area + align; - - const VAddr area_end{area + num_pages * PageSize + guard_pages * PageSize}; - const VAddr area_last{area_end - 1}; - - if (info.GetAddress() <= area && area < area_last && area_last <= region_last && - area_last <= info.GetLastAddress()) { - return area; - } - } - - return {}; -} - -void MemoryBlockManager::Update(VAddr addr, std::size_t num_pages, MemoryState prev_state, - MemoryPermission prev_perm, MemoryAttribute prev_attribute, - MemoryState state, MemoryPermission perm, - MemoryAttribute attribute) { - const VAddr end_addr{addr + num_pages * PageSize}; - iterator node{memory_block_tree.begin()}; - - prev_attribute |= MemoryAttribute::IpcAndDeviceMapped; - - while (node != memory_block_tree.end()) { - MemoryBlock* block{&(*node)}; - iterator next_node{std::next(node)}; - const VAddr cur_addr{block->GetAddress()}; - const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; - - if (addr < cur_end_addr && cur_addr < end_addr) { - if (!block->HasProperties(prev_state, prev_perm, prev_attribute)) { - node = next_node; - continue; - } - - iterator new_node{node}; - if (addr > cur_addr) { - memory_block_tree.insert(node, block->Split(addr)); - } - - if (end_addr < cur_end_addr) { - new_node = memory_block_tree.insert(node, block->Split(end_addr)); - } - - new_node->Update(state, perm, attribute); - - MergeAdjacent(new_node, next_node); - } - - if (cur_end_addr - 1 >= end_addr - 1) { - break; - } - - node = next_node; - } -} - -void MemoryBlockManager::Update(VAddr addr, std::size_t num_pages, MemoryState state, - MemoryPermission perm, MemoryAttribute attribute) { - const VAddr end_addr{addr + num_pages * PageSize}; - iterator node{memory_block_tree.begin()}; - - while (node != memory_block_tree.end()) { - MemoryBlock* block{&(*node)}; - iterator next_node{std::next(node)}; - const VAddr cur_addr{block->GetAddress()}; - const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; - - if (addr < cur_end_addr && cur_addr < end_addr) { - iterator new_node{node}; - - if (addr > cur_addr) { - memory_block_tree.insert(node, block->Split(addr)); - } - - if (end_addr < cur_end_addr) { - new_node = memory_block_tree.insert(node, block->Split(end_addr)); - } - - new_node->Update(state, perm, attribute); - - MergeAdjacent(new_node, next_node); - } - - if (cur_end_addr - 1 >= end_addr - 1) { - break; - } - - node = next_node; - } -} - -void MemoryBlockManager::UpdateLock(VAddr addr, std::size_t num_pages, LockFunc&& lock_func, - MemoryPermission perm) { - const VAddr end_addr{addr + num_pages * PageSize}; - iterator node{memory_block_tree.begin()}; - - while (node != memory_block_tree.end()) { - MemoryBlock* block{&(*node)}; - iterator next_node{std::next(node)}; - const VAddr cur_addr{block->GetAddress()}; - const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; - - if (addr < cur_end_addr && cur_addr < end_addr) { - iterator new_node{node}; - - if (addr > cur_addr) { - memory_block_tree.insert(node, block->Split(addr)); - } - - if (end_addr < cur_end_addr) { - new_node = memory_block_tree.insert(node, block->Split(end_addr)); - } - - lock_func(new_node, perm); - - MergeAdjacent(new_node, next_node); - } - - if (cur_end_addr - 1 >= end_addr - 1) { - break; - } - - node = next_node; - } -} - -void MemoryBlockManager::IterateForRange(VAddr start, VAddr end, IterateFunc&& func) { - const_iterator it{FindIterator(start)}; - MemoryInfo info{}; - do { - info = it->GetMemoryInfo(); - func(info); - it = std::next(it); - } while (info.addr + info.size - 1 < end - 1 && it != cend()); -} - -void MemoryBlockManager::MergeAdjacent(iterator it, iterator& next_it) { - MemoryBlock* block{&(*it)}; - - auto EraseIt = [&](const iterator it_to_erase) { - if (next_it == it_to_erase) { - next_it = std::next(next_it); - } - memory_block_tree.erase(it_to_erase); - }; - - if (it != memory_block_tree.begin()) { - MemoryBlock* prev{&(*std::prev(it))}; - - if (block->HasSameProperties(*prev)) { - const iterator prev_it{std::prev(it)}; - - prev->Add(block->GetNumPages()); - EraseIt(it); - - it = prev_it; - block = prev; - } - } - - if (it != cend()) { - const MemoryBlock* const next{&(*std::next(it))}; - - if (block->HasSameProperties(*next)) { - block->Add(next->GetNumPages()); - EraseIt(std::next(it)); - } - } -} - -} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_block_manager.h b/src/core/hle/kernel/memory/memory_block_manager.h deleted file mode 100644 index f57d1bbcc..000000000 --- a/src/core/hle/kernel/memory/memory_block_manager.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -#pragma once - -#include -#include - -#include "common/common_types.h" -#include "core/hle/kernel/memory/memory_block.h" - -namespace Kernel::Memory { - -class MemoryBlockManager final { -public: - using MemoryBlockTree = std::list; - using iterator = MemoryBlockTree::iterator; - using const_iterator = MemoryBlockTree::const_iterator; - -public: - MemoryBlockManager(VAddr start_addr, VAddr end_addr); - - iterator end() { - return memory_block_tree.end(); - } - const_iterator end() const { - return memory_block_tree.end(); - } - const_iterator cend() const { - return memory_block_tree.cend(); - } - - iterator FindIterator(VAddr addr); - - VAddr FindFreeArea(VAddr region_start, std::size_t region_num_pages, std::size_t num_pages, - std::size_t align, std::size_t offset, std::size_t guard_pages); - - void Update(VAddr addr, std::size_t num_pages, MemoryState prev_state, - MemoryPermission prev_perm, MemoryAttribute prev_attribute, MemoryState state, - MemoryPermission perm, MemoryAttribute attribute); - - void Update(VAddr addr, std::size_t num_pages, MemoryState state, - MemoryPermission perm = MemoryPermission::None, - MemoryAttribute attribute = MemoryAttribute::None); - - using LockFunc = std::function; - void UpdateLock(VAddr addr, std::size_t num_pages, LockFunc&& lock_func, MemoryPermission perm); - - using IterateFunc = std::function; - void IterateForRange(VAddr start, VAddr end, IterateFunc&& func); - - MemoryBlock& FindBlock(VAddr addr) { - return *FindIterator(addr); - } - -private: - void MergeAdjacent(iterator it, iterator& next_it); - - [[maybe_unused]] const VAddr start_addr; - [[maybe_unused]] const VAddr end_addr; - - MemoryBlockTree memory_block_tree; -}; - -} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_types.h b/src/core/hle/kernel/memory/memory_types.h deleted file mode 100644 index a75bf77c0..000000000 --- a/src/core/hle/kernel/memory/memory_types.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -#pragma once - -#include - -#include "common/common_types.h" - -namespace Kernel::Memory { - -constexpr std::size_t PageBits{12}; -constexpr std::size_t PageSize{1 << PageBits}; - -using Page = std::array; - -} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_heap.h b/src/core/hle/kernel/memory/page_heap.h index ee339f329..e21d60a54 100644 --- a/src/core/hle/kernel/memory/page_heap.h +++ b/src/core/hle/kernel/memory/page_heap.h @@ -16,7 +16,7 @@ #include "common/common_funcs.h" #include "common/common_types.h" #include "core/hle/kernel/k_page_bitmap.h" -#include "core/hle/kernel/memory/memory_types.h" +#include "core/hle/kernel/memory_types.h" namespace Kernel::Memory { diff --git a/src/core/hle/kernel/memory/page_linked_list.h b/src/core/hle/kernel/memory/page_linked_list.h index 45dc13eaf..9b871f15b 100644 --- a/src/core/hle/kernel/memory/page_linked_list.h +++ b/src/core/hle/kernel/memory/page_linked_list.h @@ -8,7 +8,7 @@ #include "common/assert.h" #include "common/common_types.h" -#include "core/hle/kernel/memory/memory_types.h" +#include "core/hle/kernel/memory_types.h" #include "core/hle/result.h" namespace Kernel::Memory { diff --git a/src/core/hle/kernel/memory/page_table.cpp b/src/core/hle/kernel/memory/page_table.cpp index 02a17a695..183482648 100644 --- a/src/core/hle/kernel/memory/page_table.cpp +++ b/src/core/hle/kernel/memory/page_table.cpp @@ -7,12 +7,12 @@ #include "common/scope_exit.h" #include "core/core.h" #include "core/hle/kernel/k_address_space_info.h" +#include "core/hle/kernel/k_memory_block.h" +#include "core/hle/kernel/k_memory_block_manager.h" #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_scoped_resource_reservation.h" #include "core/hle/kernel/k_system_control.h" #include "core/hle/kernel/kernel.h" -#include "core/hle/kernel/memory/memory_block.h" -#include "core/hle/kernel/memory/memory_block_manager.h" #include "core/hle/kernel/memory/page_linked_list.h" #include "core/hle/kernel/memory/page_table.h" #include "core/hle/kernel/process.h" @@ -38,14 +38,14 @@ constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceT } } -constexpr u64 GetAddressInRange(const MemoryInfo& info, VAddr addr) { +constexpr u64 GetAddressInRange(const KMemoryInfo& info, VAddr addr) { if (info.GetAddress() < addr) { return addr; } return info.GetAddress(); } -constexpr std::size_t GetSizeInRange(const MemoryInfo& info, VAddr start, VAddr end) { +constexpr std::size_t GetSizeInRange(const KMemoryInfo& info, VAddr start, VAddr end) { std::size_t size{info.GetSize()}; if (info.GetAddress() < start) { size -= start - info.GetAddress(); @@ -271,8 +271,8 @@ ResultCode PageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_t return InitializeMemoryLayout(start, end); } -ResultCode PageTable::MapProcessCode(VAddr addr, std::size_t num_pages, MemoryState state, - MemoryPermission perm) { +ResultCode PageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, + KMemoryPermission perm) { std::lock_guard lock{page_table_lock}; const u64 size{num_pages * PageSize}; @@ -300,12 +300,12 @@ ResultCode PageTable::MapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std:: const std::size_t num_pages{size / PageSize}; - MemoryState state{}; - MemoryPermission perm{}; - CASCADE_CODE(CheckMemoryState(&state, &perm, nullptr, src_addr, size, MemoryState::All, - MemoryState::Normal, MemoryPermission::Mask, - MemoryPermission::ReadAndWrite, MemoryAttribute::Mask, - MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + KMemoryState state{}; + KMemoryPermission perm{}; + CASCADE_CODE(CheckMemoryState(&state, &perm, nullptr, src_addr, size, KMemoryState::All, + KMemoryState::Normal, KMemoryPermission::Mask, + KMemoryPermission::ReadAndWrite, KMemoryAttribute::Mask, + KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); if (IsRegionMapped(dst_addr, size)) { return ResultInvalidCurrentMemory; @@ -318,16 +318,16 @@ ResultCode PageTable::MapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std:: auto block_guard = detail::ScopeExit( [&] { Operate(src_addr, num_pages, perm, OperationType::ChangePermissions); }); - CASCADE_CODE( - Operate(src_addr, num_pages, MemoryPermission::None, OperationType::ChangePermissions)); - CASCADE_CODE(MapPages(dst_addr, page_linked_list, MemoryPermission::None)); + CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None, + OperationType::ChangePermissions)); + CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::None)); block_guard.Cancel(); } - block_manager->Update(src_addr, num_pages, state, MemoryPermission::None, - MemoryAttribute::Locked); - block_manager->Update(dst_addr, num_pages, MemoryState::AliasCode); + block_manager->Update(src_addr, num_pages, state, KMemoryPermission::None, + KMemoryAttribute::Locked); + block_manager->Update(dst_addr, num_pages, KMemoryState::AliasCode); return RESULT_SUCCESS; } @@ -341,23 +341,24 @@ ResultCode PageTable::UnmapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std const std::size_t num_pages{size / PageSize}; - CASCADE_CODE(CheckMemoryState(nullptr, nullptr, nullptr, src_addr, size, MemoryState::All, - MemoryState::Normal, MemoryPermission::None, - MemoryPermission::None, MemoryAttribute::Mask, - MemoryAttribute::Locked, MemoryAttribute::IpcAndDeviceMapped)); + CASCADE_CODE(CheckMemoryState(nullptr, nullptr, nullptr, src_addr, size, KMemoryState::All, + KMemoryState::Normal, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::Mask, + KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); - MemoryState state{}; + KMemoryState state{}; CASCADE_CODE(CheckMemoryState( - &state, nullptr, nullptr, dst_addr, PageSize, MemoryState::FlagCanCodeAlias, - MemoryState::FlagCanCodeAlias, MemoryPermission::None, MemoryPermission::None, - MemoryAttribute::Mask, MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); - CASCADE_CODE(CheckMemoryState(dst_addr, size, MemoryState::All, state, MemoryPermission::None, - MemoryPermission::None, MemoryAttribute::Mask, - MemoryAttribute::None)); - CASCADE_CODE(Operate(dst_addr, num_pages, MemoryPermission::None, OperationType::Unmap)); - - block_manager->Update(dst_addr, num_pages, MemoryState::Free); - block_manager->Update(src_addr, num_pages, MemoryState::Normal, MemoryPermission::ReadAndWrite); + &state, nullptr, nullptr, dst_addr, PageSize, KMemoryState::FlagCanCodeAlias, + KMemoryState::FlagCanCodeAlias, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); + CASCADE_CODE(CheckMemoryState(dst_addr, size, KMemoryState::All, state, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::Mask, + KMemoryAttribute::None)); + CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); + + block_manager->Update(dst_addr, num_pages, KMemoryState::Free); + block_manager->Update(src_addr, num_pages, KMemoryState::Normal, + KMemoryPermission::ReadAndWrite); return RESULT_SUCCESS; } @@ -367,8 +368,8 @@ void PageTable::MapPhysicalMemory(PageLinkedList& page_linked_list, VAddr start, PAddr map_addr{node->GetAddress()}; std::size_t src_num_pages{node->GetNumPages()}; - block_manager->IterateForRange(start, end, [&](const MemoryInfo& info) { - if (info.state != MemoryState::Free) { + block_manager->IterateForRange(start, end, [&](const KMemoryInfo& info) { + if (info.state != KMemoryState::Free) { return; } @@ -383,7 +384,7 @@ void PageTable::MapPhysicalMemory(PageLinkedList& page_linked_list, VAddr start, } const std::size_t num_pages{std::min(src_num_pages, dst_num_pages)}; - Operate(dst_addr, num_pages, MemoryPermission::ReadAndWrite, OperationType::Map, + Operate(dst_addr, num_pages, KMemoryPermission::ReadAndWrite, OperationType::Map, map_addr); dst_addr += num_pages * PageSize; @@ -400,8 +401,8 @@ ResultCode PageTable::MapPhysicalMemory(VAddr addr, std::size_t size) { std::size_t mapped_size{}; const VAddr end_addr{addr + size}; - block_manager->IterateForRange(addr, end_addr, [&](const MemoryInfo& info) { - if (info.state != MemoryState::Free) { + block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { + if (info.state != KMemoryState::Free) { mapped_size += GetSizeInRange(info, addr, end_addr); } }); @@ -435,9 +436,9 @@ ResultCode PageTable::MapPhysicalMemory(VAddr addr, std::size_t size) { physical_memory_usage += remaining_size; const std::size_t num_pages{size / PageSize}; - block_manager->Update(addr, num_pages, MemoryState::Free, MemoryPermission::None, - MemoryAttribute::None, MemoryState::Normal, - MemoryPermission::ReadAndWrite, MemoryAttribute::None); + block_manager->Update(addr, num_pages, KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryState::Normal, + KMemoryPermission::ReadAndWrite, KMemoryAttribute::None); return RESULT_SUCCESS; } @@ -450,14 +451,14 @@ ResultCode PageTable::UnmapPhysicalMemory(VAddr addr, std::size_t size) { std::size_t mapped_size{}; // Verify that the region can be unmapped - block_manager->IterateForRange(addr, end_addr, [&](const MemoryInfo& info) { - if (info.state == MemoryState::Normal) { - if (info.attribute != MemoryAttribute::None) { + block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { + if (info.state == KMemoryState::Normal) { + if (info.attribute != KMemoryAttribute::None) { result = ResultInvalidCurrentMemory; return; } mapped_size += GetSizeInRange(info, addr, end_addr); - } else if (info.state != MemoryState::Free) { + } else if (info.state != KMemoryState::Free) { result = ResultInvalidCurrentMemory; } }); @@ -487,15 +488,15 @@ ResultCode PageTable::UnmapMemory(VAddr addr, std::size_t size) { PageLinkedList page_linked_list; // Unmap each region within the range - block_manager->IterateForRange(addr, end_addr, [&](const MemoryInfo& info) { - if (info.state == MemoryState::Normal) { + block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { + if (info.state == KMemoryState::Normal) { const std::size_t block_size{GetSizeInRange(info, addr, end_addr)}; const std::size_t block_num_pages{block_size / PageSize}; const VAddr block_addr{GetAddressInRange(info, addr)}; AddRegionToPages(block_addr, block_size / PageSize, page_linked_list); - if (result = Operate(block_addr, block_num_pages, MemoryPermission::None, + if (result = Operate(block_addr, block_num_pages, KMemoryPermission::None, OperationType::Unmap); result.IsError()) { return; @@ -510,7 +511,7 @@ ResultCode PageTable::UnmapMemory(VAddr addr, std::size_t size) { const std::size_t num_pages{size / PageSize}; system.Kernel().MemoryManager().Free(page_linked_list, num_pages, memory_pool); - block_manager->Update(addr, num_pages, MemoryState::Free); + block_manager->Update(addr, num_pages, KMemoryState::Free); return RESULT_SUCCESS; } @@ -518,11 +519,11 @@ ResultCode PageTable::UnmapMemory(VAddr addr, std::size_t size) { ResultCode PageTable::Map(VAddr dst_addr, VAddr src_addr, std::size_t size) { std::lock_guard lock{page_table_lock}; - MemoryState src_state{}; + KMemoryState src_state{}; CASCADE_CODE(CheckMemoryState( - &src_state, nullptr, nullptr, src_addr, size, MemoryState::FlagCanAlias, - MemoryState::FlagCanAlias, MemoryPermission::Mask, MemoryPermission::ReadAndWrite, - MemoryAttribute::Mask, MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + &src_state, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, + KMemoryState::FlagCanAlias, KMemoryPermission::Mask, KMemoryPermission::ReadAndWrite, + KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); if (IsRegionMapped(dst_addr, size)) { return ResultInvalidCurrentMemory; @@ -535,20 +536,21 @@ ResultCode PageTable::Map(VAddr dst_addr, VAddr src_addr, std::size_t size) { { auto block_guard = detail::ScopeExit([&] { - Operate(src_addr, num_pages, MemoryPermission::ReadAndWrite, + Operate(src_addr, num_pages, KMemoryPermission::ReadAndWrite, OperationType::ChangePermissions); }); - CASCADE_CODE( - Operate(src_addr, num_pages, MemoryPermission::None, OperationType::ChangePermissions)); - CASCADE_CODE(MapPages(dst_addr, page_linked_list, MemoryPermission::ReadAndWrite)); + CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None, + OperationType::ChangePermissions)); + CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::ReadAndWrite)); block_guard.Cancel(); } - block_manager->Update(src_addr, num_pages, src_state, MemoryPermission::None, - MemoryAttribute::Locked); - block_manager->Update(dst_addr, num_pages, MemoryState::Stack, MemoryPermission::ReadAndWrite); + block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::None, + KMemoryAttribute::Locked); + block_manager->Update(dst_addr, num_pages, KMemoryState::Stack, + KMemoryPermission::ReadAndWrite); return RESULT_SUCCESS; } @@ -556,17 +558,17 @@ ResultCode PageTable::Map(VAddr dst_addr, VAddr src_addr, std::size_t size) { ResultCode PageTable::Unmap(VAddr dst_addr, VAddr src_addr, std::size_t size) { std::lock_guard lock{page_table_lock}; - MemoryState src_state{}; + KMemoryState src_state{}; CASCADE_CODE(CheckMemoryState( - &src_state, nullptr, nullptr, src_addr, size, MemoryState::FlagCanAlias, - MemoryState::FlagCanAlias, MemoryPermission::Mask, MemoryPermission::None, - MemoryAttribute::Mask, MemoryAttribute::Locked, MemoryAttribute::IpcAndDeviceMapped)); + &src_state, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, + KMemoryState::FlagCanAlias, KMemoryPermission::Mask, KMemoryPermission::None, + KMemoryAttribute::Mask, KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); - MemoryPermission dst_perm{}; - CASCADE_CODE(CheckMemoryState(nullptr, &dst_perm, nullptr, dst_addr, size, MemoryState::All, - MemoryState::Stack, MemoryPermission::None, - MemoryPermission::None, MemoryAttribute::Mask, - MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + KMemoryPermission dst_perm{}; + CASCADE_CODE(CheckMemoryState(nullptr, &dst_perm, nullptr, dst_addr, size, KMemoryState::All, + KMemoryState::Stack, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::Mask, + KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); PageLinkedList src_pages; PageLinkedList dst_pages; @@ -582,21 +584,21 @@ ResultCode PageTable::Unmap(VAddr dst_addr, VAddr src_addr, std::size_t size) { { auto block_guard = detail::ScopeExit([&] { MapPages(dst_addr, dst_pages, dst_perm); }); - CASCADE_CODE(Operate(dst_addr, num_pages, MemoryPermission::None, OperationType::Unmap)); - CASCADE_CODE(Operate(src_addr, num_pages, MemoryPermission::ReadAndWrite, + CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); + CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::ReadAndWrite, OperationType::ChangePermissions)); block_guard.Cancel(); } - block_manager->Update(src_addr, num_pages, src_state, MemoryPermission::ReadAndWrite); - block_manager->Update(dst_addr, num_pages, MemoryState::Free); + block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::ReadAndWrite); + block_manager->Update(dst_addr, num_pages, KMemoryState::Free); return RESULT_SUCCESS; } ResultCode PageTable::MapPages(VAddr addr, const PageLinkedList& page_linked_list, - MemoryPermission perm) { + KMemoryPermission perm) { VAddr cur_addr{addr}; for (const auto& node : page_linked_list.Nodes()) { @@ -605,8 +607,8 @@ ResultCode PageTable::MapPages(VAddr addr, const PageLinkedList& page_linked_lis result.IsError()) { const std::size_t num_pages{(addr - cur_addr) / PageSize}; - ASSERT( - Operate(addr, num_pages, MemoryPermission::None, OperationType::Unmap).IsSuccess()); + ASSERT(Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap) + .IsSuccess()); return result; } @@ -617,8 +619,8 @@ ResultCode PageTable::MapPages(VAddr addr, const PageLinkedList& page_linked_lis return RESULT_SUCCESS; } -ResultCode PageTable::MapPages(VAddr addr, PageLinkedList& page_linked_list, MemoryState state, - MemoryPermission perm) { +ResultCode PageTable::MapPages(VAddr addr, PageLinkedList& page_linked_list, KMemoryState state, + KMemoryPermission perm) { std::lock_guard lock{page_table_lock}; const std::size_t num_pages{page_linked_list.GetNumPages()}; @@ -639,26 +641,27 @@ ResultCode PageTable::MapPages(VAddr addr, PageLinkedList& page_linked_list, Mem return RESULT_SUCCESS; } -ResultCode PageTable::SetCodeMemoryPermission(VAddr addr, std::size_t size, MemoryPermission perm) { +ResultCode PageTable::SetCodeMemoryPermission(VAddr addr, std::size_t size, + KMemoryPermission perm) { std::lock_guard lock{page_table_lock}; - MemoryState prev_state{}; - MemoryPermission prev_perm{}; + KMemoryState prev_state{}; + KMemoryPermission prev_perm{}; CASCADE_CODE(CheckMemoryState( - &prev_state, &prev_perm, nullptr, addr, size, MemoryState::FlagCode, MemoryState::FlagCode, - MemoryPermission::None, MemoryPermission::None, MemoryAttribute::Mask, - MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + &prev_state, &prev_perm, nullptr, addr, size, KMemoryState::FlagCode, + KMemoryState::FlagCode, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); - MemoryState state{prev_state}; + KMemoryState state{prev_state}; // Ensure state is mutable if permission allows write - if ((perm & MemoryPermission::Write) != MemoryPermission::None) { - if (prev_state == MemoryState::Code) { - state = MemoryState::CodeData; - } else if (prev_state == MemoryState::AliasCode) { - state = MemoryState::AliasCodeData; + if ((perm & KMemoryPermission::Write) != KMemoryPermission::None) { + if (prev_state == KMemoryState::Code) { + state = KMemoryState::CodeData; + } else if (prev_state == KMemoryState::AliasCode) { + state = KMemoryState::AliasCodeData; } else { UNREACHABLE(); } @@ -669,13 +672,13 @@ ResultCode PageTable::SetCodeMemoryPermission(VAddr addr, std::size_t size, Memo return RESULT_SUCCESS; } - if ((prev_perm & MemoryPermission::Execute) != (perm & MemoryPermission::Execute)) { + if ((prev_perm & KMemoryPermission::Execute) != (perm & KMemoryPermission::Execute)) { // Memory execution state is changing, invalidate CPU cache range system.InvalidateCpuInstructionCacheRange(addr, size); } const std::size_t num_pages{size / PageSize}; - const OperationType operation{(perm & MemoryPermission::Execute) != MemoryPermission::None + const OperationType operation{(perm & KMemoryPermission::Execute) != KMemoryPermission::None ? OperationType::ChangePermissionsAndRefresh : OperationType::ChangePermissions}; @@ -686,35 +689,35 @@ ResultCode PageTable::SetCodeMemoryPermission(VAddr addr, std::size_t size, Memo return RESULT_SUCCESS; } -MemoryInfo PageTable::QueryInfoImpl(VAddr addr) { +KMemoryInfo PageTable::QueryInfoImpl(VAddr addr) { std::lock_guard lock{page_table_lock}; return block_manager->FindBlock(addr).GetMemoryInfo(); } -MemoryInfo PageTable::QueryInfo(VAddr addr) { +KMemoryInfo PageTable::QueryInfo(VAddr addr) { if (!Contains(addr, 1)) { - return {address_space_end, 0 - address_space_end, MemoryState::Inaccessible, - MemoryPermission::None, MemoryAttribute::None, MemoryPermission::None}; + return {address_space_end, 0 - address_space_end, KMemoryState::Inaccessible, + KMemoryPermission::None, KMemoryAttribute::None, KMemoryPermission::None}; } return QueryInfoImpl(addr); } -ResultCode PageTable::ReserveTransferMemory(VAddr addr, std::size_t size, MemoryPermission perm) { +ResultCode PageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm) { std::lock_guard lock{page_table_lock}; - MemoryState state{}; - MemoryAttribute attribute{}; + KMemoryState state{}; + KMemoryAttribute attribute{}; - CASCADE_CODE(CheckMemoryState(&state, nullptr, &attribute, addr, size, - MemoryState::FlagCanTransfer | MemoryState::FlagReferenceCounted, - MemoryState::FlagCanTransfer | MemoryState::FlagReferenceCounted, - MemoryPermission::Mask, MemoryPermission::ReadAndWrite, - MemoryAttribute::Mask, MemoryAttribute::None, - MemoryAttribute::IpcAndDeviceMapped)); + CASCADE_CODE(CheckMemoryState( + &state, nullptr, &attribute, addr, size, + KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, + KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, KMemoryPermission::Mask, + KMemoryPermission::ReadAndWrite, KMemoryAttribute::Mask, KMemoryAttribute::None, + KMemoryAttribute::IpcAndDeviceMapped)); - block_manager->Update(addr, size / PageSize, state, perm, attribute | MemoryAttribute::Locked); + block_manager->Update(addr, size / PageSize, state, perm, attribute | KMemoryAttribute::Locked); return RESULT_SUCCESS; } @@ -722,33 +725,33 @@ ResultCode PageTable::ReserveTransferMemory(VAddr addr, std::size_t size, Memory ResultCode PageTable::ResetTransferMemory(VAddr addr, std::size_t size) { std::lock_guard lock{page_table_lock}; - MemoryState state{}; + KMemoryState state{}; - CASCADE_CODE(CheckMemoryState(&state, nullptr, nullptr, addr, size, - MemoryState::FlagCanTransfer | MemoryState::FlagReferenceCounted, - MemoryState::FlagCanTransfer | MemoryState::FlagReferenceCounted, - MemoryPermission::None, MemoryPermission::None, - MemoryAttribute::Mask, MemoryAttribute::Locked, - MemoryAttribute::IpcAndDeviceMapped)); + CASCADE_CODE( + CheckMemoryState(&state, nullptr, nullptr, addr, size, + KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, + KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, + KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::Mask, + KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); - block_manager->Update(addr, size / PageSize, state, MemoryPermission::ReadAndWrite); + block_manager->Update(addr, size / PageSize, state, KMemoryPermission::ReadAndWrite); return RESULT_SUCCESS; } -ResultCode PageTable::SetMemoryAttribute(VAddr addr, std::size_t size, MemoryAttribute mask, - MemoryAttribute value) { +ResultCode PageTable::SetMemoryAttribute(VAddr addr, std::size_t size, KMemoryAttribute mask, + KMemoryAttribute value) { std::lock_guard lock{page_table_lock}; - MemoryState state{}; - MemoryPermission perm{}; - MemoryAttribute attribute{}; + KMemoryState state{}; + KMemoryPermission perm{}; + KMemoryAttribute attribute{}; - CASCADE_CODE(CheckMemoryState(&state, &perm, &attribute, addr, size, - MemoryState::FlagCanChangeAttribute, - MemoryState::FlagCanChangeAttribute, MemoryPermission::None, - MemoryPermission::None, MemoryAttribute::LockedAndIpcLocked, - MemoryAttribute::None, MemoryAttribute::DeviceSharedAndUncached)); + CASCADE_CODE(CheckMemoryState( + &state, &perm, &attribute, addr, size, KMemoryState::FlagCanChangeAttribute, + KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, + KMemoryAttribute::DeviceSharedAndUncached)); attribute = attribute & ~mask; attribute = attribute | (mask & value); @@ -806,8 +809,8 @@ ResultVal PageTable::SetHeapSize(std::size_t size) { // Succeeded in allocation, commit the resource reservation memory_reservation.Commit(); - block_manager->Update(current_heap_addr, num_pages, MemoryState::Normal, - MemoryPermission::ReadAndWrite); + block_manager->Update(current_heap_addr, num_pages, KMemoryState::Normal, + KMemoryPermission::ReadAndWrite); current_heap_addr = heap_region_start + size; } @@ -817,8 +820,8 @@ ResultVal PageTable::SetHeapSize(std::size_t size) { ResultVal PageTable::AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, bool is_map_only, VAddr region_start, - std::size_t region_num_pages, MemoryState state, - MemoryPermission perm, PAddr map_addr) { + std::size_t region_num_pages, KMemoryState state, + KMemoryPermission perm, PAddr map_addr) { std::lock_guard lock{page_table_lock}; if (!CanContain(region_start, region_num_pages * PageSize, state)) { @@ -852,19 +855,19 @@ ResultVal PageTable::AllocateAndMapMemory(std::size_t needed_num_pages, s ResultCode PageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { std::lock_guard lock{page_table_lock}; - MemoryPermission perm{}; + KMemoryPermission perm{}; if (const ResultCode result{CheckMemoryState( - nullptr, &perm, nullptr, addr, size, MemoryState::FlagCanChangeAttribute, - MemoryState::FlagCanChangeAttribute, MemoryPermission::None, MemoryPermission::None, - MemoryAttribute::LockedAndIpcLocked, MemoryAttribute::None, - MemoryAttribute::DeviceSharedAndUncached)}; + nullptr, &perm, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, + KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, + KMemoryAttribute::DeviceSharedAndUncached)}; result.IsError()) { return result; } block_manager->UpdateLock( addr, size / PageSize, - [](MemoryBlockManager::iterator block, MemoryPermission perm) { + [](KMemoryBlockManager::iterator block, KMemoryPermission perm) { block->ShareToDevice(perm); }, perm); @@ -875,19 +878,19 @@ ResultCode PageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { ResultCode PageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) { std::lock_guard lock{page_table_lock}; - MemoryPermission perm{}; + KMemoryPermission perm{}; if (const ResultCode result{CheckMemoryState( - nullptr, &perm, nullptr, addr, size, MemoryState::FlagCanChangeAttribute, - MemoryState::FlagCanChangeAttribute, MemoryPermission::None, MemoryPermission::None, - MemoryAttribute::LockedAndIpcLocked, MemoryAttribute::None, - MemoryAttribute::DeviceSharedAndUncached)}; + nullptr, &perm, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, + KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, + KMemoryAttribute::DeviceSharedAndUncached)}; result.IsError()) { return result; } block_manager->UpdateLock( addr, size / PageSize, - [](MemoryBlockManager::iterator block, MemoryPermission perm) { + [](KMemoryBlockManager::iterator block, KMemoryPermission perm) { block->UnshareToDevice(perm); }, perm); @@ -896,15 +899,16 @@ ResultCode PageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) } ResultCode PageTable::InitializeMemoryLayout(VAddr start, VAddr end) { - block_manager = std::make_unique(start, end); + block_manager = std::make_unique(start, end); return RESULT_SUCCESS; } bool PageTable::IsRegionMapped(VAddr address, u64 size) { - return CheckMemoryState(address, size, MemoryState::All, MemoryState::Free, - MemoryPermission::Mask, MemoryPermission::None, MemoryAttribute::Mask, - MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped) + return CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, + KMemoryPermission::Mask, KMemoryPermission::None, + KMemoryAttribute::Mask, KMemoryAttribute::None, + KMemoryAttribute::IpcAndDeviceMapped) .IsError(); } @@ -966,7 +970,7 @@ ResultCode PageTable::Operate(VAddr addr, std::size_t num_pages, const PageLinke return RESULT_SUCCESS; } -ResultCode PageTable::Operate(VAddr addr, std::size_t num_pages, MemoryPermission perm, +ResultCode PageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, OperationType operation, PAddr map_addr) { std::lock_guard lock{page_table_lock}; @@ -993,34 +997,34 @@ ResultCode PageTable::Operate(VAddr addr, std::size_t num_pages, MemoryPermissio return RESULT_SUCCESS; } -constexpr VAddr PageTable::GetRegionAddress(MemoryState state) const { +constexpr VAddr PageTable::GetRegionAddress(KMemoryState state) const { switch (state) { - case MemoryState::Free: - case MemoryState::Kernel: + case KMemoryState::Free: + case KMemoryState::Kernel: return address_space_start; - case MemoryState::Normal: + case KMemoryState::Normal: return heap_region_start; - case MemoryState::Ipc: - case MemoryState::NonSecureIpc: - case MemoryState::NonDeviceIpc: + case KMemoryState::Ipc: + case KMemoryState::NonSecureIpc: + case KMemoryState::NonDeviceIpc: return alias_region_start; - case MemoryState::Stack: + case KMemoryState::Stack: return stack_region_start; - case MemoryState::Io: - case MemoryState::Static: - case MemoryState::ThreadLocal: + case KMemoryState::Io: + case KMemoryState::Static: + case KMemoryState::ThreadLocal: return kernel_map_region_start; - case MemoryState::Shared: - case MemoryState::AliasCode: - case MemoryState::AliasCodeData: - case MemoryState::Transferred: - case MemoryState::SharedTransferred: - case MemoryState::SharedCode: - case MemoryState::GeneratedCode: - case MemoryState::CodeOut: + case KMemoryState::Shared: + case KMemoryState::AliasCode: + case KMemoryState::AliasCodeData: + case KMemoryState::Transferred: + case KMemoryState::SharedTransferred: + case KMemoryState::SharedCode: + case KMemoryState::GeneratedCode: + case KMemoryState::CodeOut: return alias_code_region_start; - case MemoryState::Code: - case MemoryState::CodeData: + case KMemoryState::Code: + case KMemoryState::CodeData: return code_region_start; default: UNREACHABLE(); @@ -1028,34 +1032,34 @@ constexpr VAddr PageTable::GetRegionAddress(MemoryState state) const { } } -constexpr std::size_t PageTable::GetRegionSize(MemoryState state) const { +constexpr std::size_t PageTable::GetRegionSize(KMemoryState state) const { switch (state) { - case MemoryState::Free: - case MemoryState::Kernel: + case KMemoryState::Free: + case KMemoryState::Kernel: return address_space_end - address_space_start; - case MemoryState::Normal: + case KMemoryState::Normal: return heap_region_end - heap_region_start; - case MemoryState::Ipc: - case MemoryState::NonSecureIpc: - case MemoryState::NonDeviceIpc: + case KMemoryState::Ipc: + case KMemoryState::NonSecureIpc: + case KMemoryState::NonDeviceIpc: return alias_region_end - alias_region_start; - case MemoryState::Stack: + case KMemoryState::Stack: return stack_region_end - stack_region_start; - case MemoryState::Io: - case MemoryState::Static: - case MemoryState::ThreadLocal: + case KMemoryState::Io: + case KMemoryState::Static: + case KMemoryState::ThreadLocal: return kernel_map_region_end - kernel_map_region_start; - case MemoryState::Shared: - case MemoryState::AliasCode: - case MemoryState::AliasCodeData: - case MemoryState::Transferred: - case MemoryState::SharedTransferred: - case MemoryState::SharedCode: - case MemoryState::GeneratedCode: - case MemoryState::CodeOut: + case KMemoryState::Shared: + case KMemoryState::AliasCode: + case KMemoryState::AliasCodeData: + case KMemoryState::Transferred: + case KMemoryState::SharedTransferred: + case KMemoryState::SharedCode: + case KMemoryState::GeneratedCode: + case KMemoryState::CodeOut: return alias_code_region_end - alias_code_region_start; - case MemoryState::Code: - case MemoryState::CodeData: + case KMemoryState::Code: + case KMemoryState::CodeData: return code_region_end - code_region_start; default: UNREACHABLE(); @@ -1063,7 +1067,7 @@ constexpr std::size_t PageTable::GetRegionSize(MemoryState state) const { } } -constexpr bool PageTable::CanContain(VAddr addr, std::size_t size, MemoryState state) const { +constexpr bool PageTable::CanContain(VAddr addr, std::size_t size, KMemoryState state) const { const VAddr end{addr + size}; const VAddr last{end - 1}; const VAddr region_start{GetRegionAddress(state)}; @@ -1074,30 +1078,30 @@ constexpr bool PageTable::CanContain(VAddr addr, std::size_t size, MemoryState s const bool is_in_alias{!(end <= alias_region_start || alias_region_end <= addr)}; switch (state) { - case MemoryState::Free: - case MemoryState::Kernel: + case KMemoryState::Free: + case KMemoryState::Kernel: return is_in_region; - case MemoryState::Io: - case MemoryState::Static: - case MemoryState::Code: - case MemoryState::CodeData: - case MemoryState::Shared: - case MemoryState::AliasCode: - case MemoryState::AliasCodeData: - case MemoryState::Stack: - case MemoryState::ThreadLocal: - case MemoryState::Transferred: - case MemoryState::SharedTransferred: - case MemoryState::SharedCode: - case MemoryState::GeneratedCode: - case MemoryState::CodeOut: + case KMemoryState::Io: + case KMemoryState::Static: + case KMemoryState::Code: + case KMemoryState::CodeData: + case KMemoryState::Shared: + case KMemoryState::AliasCode: + case KMemoryState::AliasCodeData: + case KMemoryState::Stack: + case KMemoryState::ThreadLocal: + case KMemoryState::Transferred: + case KMemoryState::SharedTransferred: + case KMemoryState::SharedCode: + case KMemoryState::GeneratedCode: + case KMemoryState::CodeOut: return is_in_region && !is_in_heap && !is_in_alias; - case MemoryState::Normal: + case KMemoryState::Normal: ASSERT(is_in_heap); return is_in_region && !is_in_alias; - case MemoryState::Ipc: - case MemoryState::NonSecureIpc: - case MemoryState::NonDeviceIpc: + case KMemoryState::Ipc: + case KMemoryState::NonSecureIpc: + case KMemoryState::NonDeviceIpc: ASSERT(is_in_alias); return is_in_region && !is_in_heap; default: @@ -1105,10 +1109,10 @@ constexpr bool PageTable::CanContain(VAddr addr, std::size_t size, MemoryState s } } -constexpr ResultCode PageTable::CheckMemoryState(const MemoryInfo& info, MemoryState state_mask, - MemoryState state, MemoryPermission perm_mask, - MemoryPermission perm, MemoryAttribute attr_mask, - MemoryAttribute attr) const { +constexpr ResultCode PageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr) const { // Validate the states match expectation if ((info.state & state_mask) != state) { return ResultInvalidCurrentMemory; @@ -1123,23 +1127,23 @@ constexpr ResultCode PageTable::CheckMemoryState(const MemoryInfo& info, MemoryS return RESULT_SUCCESS; } -ResultCode PageTable::CheckMemoryState(MemoryState* out_state, MemoryPermission* out_perm, - MemoryAttribute* out_attr, VAddr addr, std::size_t size, - MemoryState state_mask, MemoryState state, - MemoryPermission perm_mask, MemoryPermission perm, - MemoryAttribute attr_mask, MemoryAttribute attr, - MemoryAttribute ignore_attr) { +ResultCode PageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, + KMemoryAttribute* out_attr, VAddr addr, std::size_t size, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr, + KMemoryAttribute ignore_attr) { std::lock_guard lock{page_table_lock}; // Get information about the first block const VAddr last_addr{addr + size - 1}; - MemoryBlockManager::const_iterator it{block_manager->FindIterator(addr)}; - MemoryInfo info{it->GetMemoryInfo()}; + KMemoryBlockManager::const_iterator it{block_manager->FindIterator(addr)}; + KMemoryInfo info{it->GetMemoryInfo()}; // Validate all blocks in the range have correct state - const MemoryState first_state{info.state}; - const MemoryPermission first_perm{info.perm}; - const MemoryAttribute first_attr{info.attribute}; + const KMemoryState first_state{info.state}; + const KMemoryPermission first_perm{info.perm}; + const KMemoryAttribute first_attr{info.attribute}; while (true) { // Validate the current block @@ -1149,8 +1153,8 @@ ResultCode PageTable::CheckMemoryState(MemoryState* out_state, MemoryPermission* if (!(info.perm == first_perm)) { return ResultInvalidCurrentMemory; } - if (!((info.attribute | static_cast(ignore_attr)) == - (first_attr | static_cast(ignore_attr)))) { + if (!((info.attribute | static_cast(ignore_attr)) == + (first_attr | static_cast(ignore_attr)))) { return ResultInvalidCurrentMemory; } @@ -1176,7 +1180,7 @@ ResultCode PageTable::CheckMemoryState(MemoryState* out_state, MemoryPermission* *out_perm = first_perm; } if (out_attr) { - *out_attr = first_attr & static_cast(~ignore_attr); + *out_attr = first_attr & static_cast(~ignore_attr); } return RESULT_SUCCESS; diff --git a/src/core/hle/kernel/memory/page_table.h b/src/core/hle/kernel/memory/page_table.h index ce0d38849..a4914d050 100644 --- a/src/core/hle/kernel/memory/page_table.h +++ b/src/core/hle/kernel/memory/page_table.h @@ -10,7 +10,7 @@ #include "common/common_types.h" #include "common/page_table.h" #include "core/file_sys/program_metadata.h" -#include "core/hle/kernel/memory/memory_block.h" +#include "core/hle/kernel/k_memory_block.h" #include "core/hle/kernel/memory/memory_manager.h" #include "core/hle/result.h" @@ -18,9 +18,11 @@ namespace Core { class System; } -namespace Kernel::Memory { +namespace Kernel { +class KMemoryBlockManager; +} -class MemoryBlockManager; +namespace Kernel::Memory { class PageTable final : NonCopyable { public: @@ -29,8 +31,8 @@ public: ResultCode InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, VAddr code_addr, std::size_t code_size, Memory::MemoryManager::Pool pool); - ResultCode MapProcessCode(VAddr addr, std::size_t pages_count, MemoryState state, - MemoryPermission perm); + ResultCode MapProcessCode(VAddr addr, std::size_t pages_count, KMemoryState state, + KMemoryPermission perm); ResultCode MapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); ResultCode UnmapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); ResultCode MapPhysicalMemory(VAddr addr, std::size_t size); @@ -38,20 +40,20 @@ public: ResultCode UnmapMemory(VAddr addr, std::size_t size); ResultCode Map(VAddr dst_addr, VAddr src_addr, std::size_t size); ResultCode Unmap(VAddr dst_addr, VAddr src_addr, std::size_t size); - ResultCode MapPages(VAddr addr, PageLinkedList& page_linked_list, MemoryState state, - MemoryPermission perm); - ResultCode SetCodeMemoryPermission(VAddr addr, std::size_t size, MemoryPermission perm); - MemoryInfo QueryInfo(VAddr addr); - ResultCode ReserveTransferMemory(VAddr addr, std::size_t size, MemoryPermission perm); + ResultCode MapPages(VAddr addr, PageLinkedList& page_linked_list, KMemoryState state, + KMemoryPermission perm); + ResultCode SetCodeMemoryPermission(VAddr addr, std::size_t size, KMemoryPermission perm); + KMemoryInfo QueryInfo(VAddr addr); + ResultCode ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm); ResultCode ResetTransferMemory(VAddr addr, std::size_t size); - ResultCode SetMemoryAttribute(VAddr addr, std::size_t size, MemoryAttribute mask, - MemoryAttribute value); + ResultCode SetMemoryAttribute(VAddr addr, std::size_t size, KMemoryAttribute mask, + KMemoryAttribute value); ResultCode SetHeapCapacity(std::size_t new_heap_capacity); ResultVal SetHeapSize(std::size_t size); ResultVal AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, bool is_map_only, VAddr region_start, - std::size_t region_num_pages, MemoryState state, - MemoryPermission perm, PAddr map_addr = 0); + std::size_t region_num_pages, KMemoryState state, + KMemoryPermission perm, PAddr map_addr = 0); ResultCode LockForDeviceAddressSpace(VAddr addr, std::size_t size); ResultCode UnlockForDeviceAddressSpace(VAddr addr, std::size_t size); @@ -72,47 +74,48 @@ private: ChangePermissionsAndRefresh, }; - static constexpr MemoryAttribute DefaultMemoryIgnoreAttr = - MemoryAttribute::DontCareMask | MemoryAttribute::IpcLocked | MemoryAttribute::DeviceShared; + static constexpr KMemoryAttribute DefaultMemoryIgnoreAttr = KMemoryAttribute::DontCareMask | + KMemoryAttribute::IpcLocked | + KMemoryAttribute::DeviceShared; ResultCode InitializeMemoryLayout(VAddr start, VAddr end); - ResultCode MapPages(VAddr addr, const PageLinkedList& page_linked_list, MemoryPermission perm); + ResultCode MapPages(VAddr addr, const PageLinkedList& page_linked_list, KMemoryPermission perm); void MapPhysicalMemory(PageLinkedList& page_linked_list, VAddr start, VAddr end); bool IsRegionMapped(VAddr address, u64 size); bool IsRegionContiguous(VAddr addr, u64 size) const; void AddRegionToPages(VAddr start, std::size_t num_pages, PageLinkedList& page_linked_list); - MemoryInfo QueryInfoImpl(VAddr addr); + KMemoryInfo QueryInfoImpl(VAddr addr); VAddr AllocateVirtualMemory(VAddr start, std::size_t region_num_pages, u64 needed_num_pages, std::size_t align); ResultCode Operate(VAddr addr, std::size_t num_pages, const PageLinkedList& page_group, OperationType operation); - ResultCode Operate(VAddr addr, std::size_t num_pages, MemoryPermission perm, + ResultCode Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, OperationType operation, PAddr map_addr = 0); - constexpr VAddr GetRegionAddress(MemoryState state) const; - constexpr std::size_t GetRegionSize(MemoryState state) const; - constexpr bool CanContain(VAddr addr, std::size_t size, MemoryState state) const; + constexpr VAddr GetRegionAddress(KMemoryState state) const; + constexpr std::size_t GetRegionSize(KMemoryState state) const; + constexpr bool CanContain(VAddr addr, std::size_t size, KMemoryState state) const; - constexpr ResultCode CheckMemoryState(const MemoryInfo& info, MemoryState state_mask, - MemoryState state, MemoryPermission perm_mask, - MemoryPermission perm, MemoryAttribute attr_mask, - MemoryAttribute attr) const; - ResultCode CheckMemoryState(MemoryState* out_state, MemoryPermission* out_perm, - MemoryAttribute* out_attr, VAddr addr, std::size_t size, - MemoryState state_mask, MemoryState state, - MemoryPermission perm_mask, MemoryPermission perm, - MemoryAttribute attr_mask, MemoryAttribute attr, - MemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr); - ResultCode CheckMemoryState(VAddr addr, std::size_t size, MemoryState state_mask, - MemoryState state, MemoryPermission perm_mask, - MemoryPermission perm, MemoryAttribute attr_mask, - MemoryAttribute attr, - MemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) { + constexpr ResultCode CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr) const; + ResultCode CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, + KMemoryAttribute* out_attr, VAddr addr, std::size_t size, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr, + KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr); + ResultCode CheckMemoryState(VAddr addr, std::size_t size, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr, + KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) { return CheckMemoryState(nullptr, nullptr, nullptr, addr, size, state_mask, state, perm_mask, perm, attr_mask, attr, ignore_attr); } std::recursive_mutex page_table_lock; - std::unique_ptr block_manager; + std::unique_ptr block_manager; public: constexpr VAddr GetAddressSpaceStart() const { @@ -212,7 +215,7 @@ public: return !IsOutsideASLRRegion(address, size); } constexpr PAddr GetPhysicalAddr(VAddr addr) { - return page_table_impl.backing_addr[addr >> Memory::PageBits] + addr; + return page_table_impl.backing_addr[addr >> PageBits] + addr; } private: diff --git a/src/core/hle/kernel/memory_types.h b/src/core/hle/kernel/memory_types.h new file mode 100644 index 000000000..d458f0eca --- /dev/null +++ b/src/core/hle/kernel/memory_types.h @@ -0,0 +1,18 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include + +#include "common/common_types.h" + +namespace Kernel { + +constexpr std::size_t PageBits{12}; +constexpr std::size_t PageSize{1 << PageBits}; + +using Page = std::array; + +} // namespace Kernel diff --git a/src/core/hle/kernel/process.cpp b/src/core/hle/kernel/process.cpp index 47b3ac57b..f83977a8e 100644 --- a/src/core/hle/kernel/process.cpp +++ b/src/core/hle/kernel/process.cpp @@ -14,14 +14,14 @@ #include "core/device_memory.h" #include "core/file_sys/program_metadata.h" #include "core/hle/kernel/code_set.h" +#include "core/hle/kernel/k_memory_block_manager.h" #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_scheduler.h" #include "core/hle/kernel/k_scoped_resource_reservation.h" +#include "core/hle/kernel/k_slab_heap.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/kernel.h" -#include "core/hle/kernel/memory/memory_block_manager.h" #include "core/hle/kernel/memory/page_table.h" -#include "core/hle/kernel/memory/slab_heap.h" #include "core/hle/kernel/process.h" #include "core/hle/kernel/svc_results.h" #include "core/hle/lock.h" @@ -291,9 +291,9 @@ ResultCode Process::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, } // Map process code region - if (const ResultCode result{page_table->MapProcessCode( - page_table->GetCodeRegionStart(), code_size / Memory::PageSize, - Memory::MemoryState::Code, Memory::MemoryPermission::None)}; + if (const ResultCode result{page_table->MapProcessCode(page_table->GetCodeRegionStart(), + code_size / PageSize, KMemoryState::Code, + KMemoryPermission::None)}; result.IsError()) { return result; } @@ -400,22 +400,22 @@ VAddr Process::CreateTLSRegion() { return *tls_page_iter->ReserveSlot(); } - Memory::Page* const tls_page_ptr{kernel.GetUserSlabHeapPages().Allocate()}; + Page* const tls_page_ptr{kernel.GetUserSlabHeapPages().Allocate()}; ASSERT(tls_page_ptr); const VAddr start{page_table->GetKernelMapRegionStart()}; const VAddr size{page_table->GetKernelMapRegionEnd() - start}; const PAddr tls_map_addr{system.DeviceMemory().GetPhysicalAddr(tls_page_ptr)}; - const VAddr tls_page_addr{ - page_table - ->AllocateAndMapMemory(1, Memory::PageSize, true, start, size / Memory::PageSize, - Memory::MemoryState::ThreadLocal, - Memory::MemoryPermission::ReadAndWrite, tls_map_addr) - .ValueOr(0)}; + const VAddr tls_page_addr{page_table + ->AllocateAndMapMemory(1, PageSize, true, start, size / PageSize, + KMemoryState::ThreadLocal, + KMemoryPermission::ReadAndWrite, + tls_map_addr) + .ValueOr(0)}; ASSERT(tls_page_addr); - std::memset(tls_page_ptr, 0, Memory::PageSize); + std::memset(tls_page_ptr, 0, PageSize); tls_pages.emplace_back(tls_page_addr); const auto reserve_result{tls_pages.back().ReserveSlot()}; @@ -442,15 +442,15 @@ void Process::FreeTLSRegion(VAddr tls_address) { void Process::LoadModule(CodeSet code_set, VAddr base_addr) { std::lock_guard lock{HLE::g_hle_lock}; const auto ReprotectSegment = [&](const CodeSet::Segment& segment, - Memory::MemoryPermission permission) { + KMemoryPermission permission) { page_table->SetCodeMemoryPermission(segment.addr + base_addr, segment.size, permission); }; system.Memory().WriteBlock(*this, base_addr, code_set.memory.data(), code_set.memory.size()); - ReprotectSegment(code_set.CodeSegment(), Memory::MemoryPermission::ReadAndExecute); - ReprotectSegment(code_set.RODataSegment(), Memory::MemoryPermission::Read); - ReprotectSegment(code_set.DataSegment(), Memory::MemoryPermission::ReadAndWrite); + ReprotectSegment(code_set.CodeSegment(), KMemoryPermission::ReadAndExecute); + ReprotectSegment(code_set.RODataSegment(), KMemoryPermission::Read); + ReprotectSegment(code_set.DataSegment(), KMemoryPermission::ReadAndWrite); } bool Process::IsSignaled() const { @@ -479,16 +479,15 @@ ResultCode Process::AllocateMainThreadStack(std::size_t stack_size) { ASSERT(stack_size); // The kernel always ensures that the given stack size is page aligned. - main_thread_stack_size = Common::AlignUp(stack_size, Memory::PageSize); + main_thread_stack_size = Common::AlignUp(stack_size, PageSize); const VAddr start{page_table->GetStackRegionStart()}; const std::size_t size{page_table->GetStackRegionEnd() - start}; CASCADE_RESULT(main_thread_stack_top, page_table->AllocateAndMapMemory( - main_thread_stack_size / Memory::PageSize, Memory::PageSize, false, start, - size / Memory::PageSize, Memory::MemoryState::Stack, - Memory::MemoryPermission::ReadAndWrite)); + main_thread_stack_size / PageSize, PageSize, false, start, size / PageSize, + KMemoryState::Stack, KMemoryPermission::ReadAndWrite)); main_thread_stack_top += main_thread_stack_size; diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 12cfdcf2c..4ffd65d33 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -27,6 +27,7 @@ #include "core/hle/kernel/k_address_arbiter.h" #include "core/hle/kernel/k_condition_variable.h" #include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_memory_block.h" #include "core/hle/kernel/k_memory_layout.h" #include "core/hle/kernel/k_readable_event.h" #include "core/hle/kernel/k_resource_limit.h" @@ -38,7 +39,6 @@ #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_writable_event.h" #include "core/hle/kernel/kernel.h" -#include "core/hle/kernel/memory/memory_block.h" #include "core/hle/kernel/memory/page_table.h" #include "core/hle/kernel/physical_core.h" #include "core/hle/kernel/process.h" @@ -230,9 +230,9 @@ static ResultCode SetMemoryAttribute(Core::System& system, VAddr address, u64 si return ResultInvalidCurrentMemory; } - const auto attributes{static_cast(mask | attribute)}; - if (attributes != static_cast(mask) || - (attributes | Memory::MemoryAttribute::Uncached) != Memory::MemoryAttribute::Uncached) { + const auto attributes{static_cast(mask | attribute)}; + if (attributes != static_cast(mask) || + (attributes | MemoryAttribute::Uncached) != MemoryAttribute::Uncached) { LOG_ERROR(Kernel_SVC, "Memory attribute doesn't match the given mask (Attribute: 0x{:X}, Mask: {:X}", attribute, mask); @@ -241,8 +241,8 @@ static ResultCode SetMemoryAttribute(Core::System& system, VAddr address, u64 si auto& page_table{system.Kernel().CurrentProcess()->PageTable()}; - return page_table.SetMemoryAttribute(address, size, static_cast(mask), - static_cast(attribute)); + return page_table.SetMemoryAttribute(address, size, static_cast(mask), + static_cast(attribute)); } static ResultCode SetMemoryAttribute32(Core::System& system, u32 address, u32 size, u32 mask, @@ -1231,9 +1231,8 @@ static ResultCode MapSharedMemory(Core::System& system, Handle shared_memory_han return ResultInvalidCurrentMemory; } - const auto permission_type = static_cast(permissions); - if ((permission_type | Memory::MemoryPermission::Write) != - Memory::MemoryPermission::ReadAndWrite) { + const auto permission_type = static_cast(permissions); + if ((permission_type | MemoryPermission::Write) != MemoryPermission::ReadWrite) { LOG_ERROR(Kernel_SVC, "Expected Read or ReadWrite permission but got permissions=0x{:08X}", permissions); return ResultInvalidMemoryPermissions; @@ -1273,7 +1272,8 @@ static ResultCode MapSharedMemory(Core::System& system, Handle shared_memory_han return ResultInvalidHandle; } - return shared_memory->Map(*current_process, addr, size, permission_type); + return shared_memory->Map(*current_process, addr, size, + static_cast(permission_type)); } static ResultCode MapSharedMemory32(Core::System& system, Handle shared_memory_handle, u32 addr, @@ -1886,9 +1886,8 @@ static ResultCode CreateTransferMemory(Core::System& system, Handle* handle, VAd return ResultInvalidCurrentMemory; } - const auto perms{static_cast(permissions)}; - if (perms > Memory::MemoryPermission::ReadAndWrite || - perms == Memory::MemoryPermission::Write) { + const auto perms{static_cast(permissions)}; + if (perms > MemoryPermission::ReadWrite || perms == MemoryPermission::Write) { LOG_ERROR(Kernel_SVC, "Invalid memory permissions for transfer memory! (perms={:08X})", permissions); return ResultInvalidMemoryPermissions; @@ -1902,7 +1901,8 @@ static ResultCode CreateTransferMemory(Core::System& system, Handle* handle, VAd LOG_ERROR(Kernel_SVC, "Could not reserve a new transfer memory"); return ResultResourceLimitedExceeded; } - auto transfer_mem_handle = TransferMemory::Create(kernel, system.Memory(), addr, size, perms); + auto transfer_mem_handle = TransferMemory::Create(kernel, system.Memory(), addr, size, + static_cast(perms)); if (const auto reserve_result{transfer_mem_handle->Reserve()}; reserve_result.IsError()) { return reserve_result; diff --git a/src/core/hle/kernel/transfer_memory.cpp b/src/core/hle/kernel/transfer_memory.cpp index 6b0fc1591..b92559616 100644 --- a/src/core/hle/kernel/transfer_memory.cpp +++ b/src/core/hle/kernel/transfer_memory.cpp @@ -24,7 +24,7 @@ TransferMemory::~TransferMemory() { std::shared_ptr TransferMemory::Create(KernelCore& kernel, Core::Memory::Memory& memory, VAddr base_address, std::size_t size, - Memory::MemoryPermission permissions) { + KMemoryPermission permissions) { std::shared_ptr transfer_memory{ std::make_shared(kernel, memory)}; diff --git a/src/core/hle/kernel/transfer_memory.h b/src/core/hle/kernel/transfer_memory.h index 777799d12..521951424 100644 --- a/src/core/hle/kernel/transfer_memory.h +++ b/src/core/hle/kernel/transfer_memory.h @@ -6,7 +6,7 @@ #include -#include "core/hle/kernel/memory/memory_block.h" +#include "core/hle/kernel/k_memory_block.h" #include "core/hle/kernel/object.h" #include "core/hle/kernel/physical_memory.h" @@ -36,7 +36,7 @@ public: static std::shared_ptr Create(KernelCore& kernel, Core::Memory::Memory& memory, VAddr base_address, std::size_t size, - Memory::MemoryPermission permissions); + KMemoryPermission permissions); TransferMemory(const TransferMemory&) = delete; TransferMemory& operator=(const TransferMemory&) = delete; @@ -82,7 +82,7 @@ private: std::size_t size{}; /// The memory permissions that are applied to this instance. - Memory::MemoryPermission owner_permissions{}; + KMemoryPermission owner_permissions{}; /// The process that this transfer memory instance was created under. Process* owner_process{}; -- cgit v1.2.3