From 2f0418c10134b4c8e5ae47ace623b5db57c0435c Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Thu, 21 Dec 2023 00:04:03 +0100 Subject: Core: Initial implementation of device memory mapping --- src/core/device_memory_manager.inc | 304 +++++++++++++++++++++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 src/core/device_memory_manager.inc (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc new file mode 100644 index 000000000..1f52b92d5 --- /dev/null +++ b/src/core/device_memory_manager.inc @@ -0,0 +1,304 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include + +#include "common/address_space.h" +#include "common/address_space.inc" +#include "common/alignment.h" +#include "common/scope_exit.h" +#include "core/device_memory.h" +#include "core/device_memory_manager.h" +#include "core/memory.h" + +namespace Core { + +struct EmptyAllocator { + EmptyAllocator([[maybe_unused]] DAddr address) {} +}; + +template +struct DeviceMemoryManagerAllocator { + static constexpr bool supports_pinning = DTraits::supports_pinning; + static constexpr size_t device_virtual_bits = DTraits::device_virtual_bits; + static constexpr size_t pin_bits = 32; + static constexpr DAddr first_address = 1ULL << Memory::YUZU_PAGEBITS; + static constexpr DAddr max_pin_area = supports_pinning ? 1ULL << pin_bits : first_address; + static constexpr DAddr max_device_area = 1ULL << device_virtual_bits; + + DeviceMemoryManagerAllocator() + : pin_allocator(first_address), + main_allocator(supports_pinning ? 1ULL << pin_bits : first_address) {} + + std::conditional_t, EmptyAllocator> + pin_allocator; + Common::FlatAllocator main_allocator; + + /// Returns true when vaddr -> vaddr+size is fully contained in the buffer + template + [[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept { + if constexpr (pin_area) { + return addr >= 0 && addr + size <= max_pin_area; + } else { + return addr >= max_pin_area && addr + size <= max_device_area; + } + } + + DAddr Allocate(size_t size) { + return main_allocator.Allocate(size); + } + + DAddr AllocatePinned(size_t size) { + return pin_allocator.Allocate(size); + } + + void DoInRange(DAddr address, size_t size, auto pin_func, auto main_func) { + if (IsInBounds(address, size)) { + pin_func(address, size); + return; + } + if (IsInBounds(address, size)) { + main_func(address, size); + return; + } + DAddr end_size = address + size - max_pin_area; + DAddr end_size2 = max_pin_area - address; + pin_func(address, end_size2); + main_func(max_pin_area, end_size); + } + + void AllocateFixed(DAddr b_address, size_t b_size) { + if constexpr (supports_pinning) { + DoInRange( + b_address, b_size, + [this](DAddr address, size_t size) { pin_allocator.AllocateFixed(address, size); }, + [this](DAddr address, size_t size) { + main_allocator.AllocateFixed(address, size); + }); + } else { + main_allocator.AllocateFixed(b_address, b_size); + } + } + + void Free(DAddr b_address, size_t b_size) { + if constexpr (supports_pinning) { + DoInRange( + b_address, b_size, + [this](DAddr address, size_t size) { pin_allocator.Free(address, size); }, + [this](DAddr address, size_t size) { main_allocator.Free(address, size); }); + } else { + main_allocator.Free(b_address, b_size); + } + } +}; + +template +DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) + : physical_base{reinterpret_cast(device_memory_.buffer.BackingBasePointer())}, + interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS), + compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)) { + impl = std::make_unique>(); +} + +template +DeviceMemoryManager::~DeviceMemoryManager() = default; + +template +void DeviceMemoryManager::BindInterface(DeviceInterface* interface_) { + interface = interface_; +} + +template +DAddr DeviceMemoryManager::Allocate(size_t size) { + return impl->Allocate(size); +} + +template +void DeviceMemoryManager::AllocateFixed(DAddr start, size_t size) { + return impl->AllocateFixed(start, size); +} + +template +DAddr DeviceMemoryManager::AllocatePinned(size_t size) { + return impl->AllocatePinned(size); +} + +template +void DeviceMemoryManager::Free(DAddr start, size_t size) { + impl->Free(start, size); +} + +template +void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size_t size, + size_t p_id) { + Core::Memory::Memory* process_memory = registered_processes[p_id]; + size_t start_page_d = address >> Memory::YUZU_PAGEBITS; + size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; + std::atomic_thread_fence(std::memory_order_acquire); + for (size_t i = 0; i < num_pages; i++) { + auto* ptr = process_memory->GetPointer( + Common::ProcessAddress(virtual_address + i * Memory::YUZU_PAGESIZE)); + if (ptr == nullptr) [[unlikely]] { + compressed_physical_ptr[start_page_d + i] = 0; + continue; + } + auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; + compressed_physical_ptr[start_page_d + i] = phys_addr; + compressed_device_addr[phys_addr - 1U] = static_cast(start_page_d + i); + } + std::atomic_thread_fence(std::memory_order_release); +} + +template +void DeviceMemoryManager::Unmap(DAddr address, size_t size) { + size_t start_page_d = address >> Memory::YUZU_PAGEBITS; + size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; + std::atomic_thread_fence(std::memory_order_acquire); + for (size_t i = 0; i < num_pages; i++) { + auto phys_addr = compressed_physical_ptr[start_page_d + i]; + compressed_physical_ptr[start_page_d + i] = 0; + if (phys_addr != 0) { + compressed_device_addr[phys_addr - 1] = 0; + } + } + std::atomic_thread_fence(std::memory_order_release); +} + +template +template +T* DeviceMemoryManager::GetPointer(DAddr address) { + const size_t index = address >> Memory::YUZU_PAGEBITS; + const size_t offset = address & Memory::YUZU_PAGEMASK; + auto phys_addr = compressed_physical_ptr[index]; + if (phys_addr == 0) [[unlikely]] { + return nullptr; + } + return GetPointerFromRaw( + static_cast(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + offset)); +} + +template +template +const T* DeviceMemoryManager::GetPointer(DAddr address) const { + const size_t index = address >> Memory::YUZU_PAGEBITS; + const size_t offset = address & Memory::YUZU_PAGEMASK; + auto phys_addr = compressed_physical_ptr[index]; + if (phys_addr == 0) [[unlikely]] { + return nullptr; + } + return GetPointerFromRaw( + static_cast(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + offset)); +} + +template +template +void DeviceMemoryManager::Write(DAddr address, T value) { + T* ptr = GetPointer(address); + if (!ptr) [[unlikely]] { + return; + } + std::memcpy(ptr, &value, sizeof(T)); +} + +template +template +T DeviceMemoryManager::Read(DAddr address) const { + const T* ptr = GetPointer(address); + T result{}; + if (!ptr) [[unlikely]] { + return result; + } + std::memcpy(&result, ptr, sizeof(T)); + return result; +} + +template +void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto on_unmapped, + auto on_memory, auto increment) { + std::size_t remaining_size = size; + std::size_t page_index = addr >> Memory::YUZU_PAGEBITS; + std::size_t page_offset = addr & Memory::YUZU_PAGEMASK; + + while (remaining_size) { + const std::size_t copy_amount = + std::min(static_cast(Memory::YUZU_PAGESIZE) - page_offset, remaining_size); + const auto current_vaddr = + static_cast((page_index << Memory::YUZU_PAGEBITS) + page_offset); + SCOPE_EXIT({ + page_index++; + page_offset = 0; + increment(copy_amount); + remaining_size -= copy_amount; + }); + + auto phys_addr = compressed_physical_ptr[page_index]; + if (phys_addr == 0) { + on_unmapped(copy_amount, current_vaddr); + continue; + } + auto* mem_ptr = GetPointerFromRaw( + static_cast(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + page_offset)); + on_memory(copy_amount, mem_ptr); + } +} + +template +void DeviceMemoryManager::ReadBlock(DAddr address, void* dest_pointer, size_t size) { + WalkBlock( + address, size, + [&](size_t copy_amount, DAddr current_vaddr) { + LOG_ERROR( + HW_Memory, + "Unmapped Device ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, address, size); + std::memset(dest_pointer, 0, copy_amount); + }, + [&](size_t copy_amount, const u8* const src_ptr) { + std::memcpy(dest_pointer, src_ptr, copy_amount); + }, + [&](const std::size_t copy_amount) { + dest_pointer = static_cast(dest_pointer) + copy_amount; + }); +} + +template +void DeviceMemoryManager::WriteBlock(DAddr address, void* src_pointer, size_t size) { + WalkBlock( + address, size, + [&](size_t copy_amount, DAddr current_vaddr) { + LOG_ERROR( + HW_Memory, + "Unmapped Device WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, address, size); + }, + [&](size_t copy_amount, u8* const dst_ptr) { + std::memcpy(dst_ptr, src_pointer, copy_amount); + }, + [&](const std::size_t copy_amount) { + src_pointer = static_cast(src_pointer) + copy_amount; + }); +} + +template +size_t DeviceMemoryManager::RegisterProcess(Memory::Memory* memory_interface) { + size_t new_id; + if (!id_pool.empty()) { + new_id = id_pool.front(); + id_pool.pop_front(); + registered_processes[new_id] = memory_interface; + } else { + registered_processes.emplace_back(memory_interface); + new_id = registered_processes.size() - 1U; + } + return new_id; +} + +template +void DeviceMemoryManager::UnregisterProcess(size_t id) { + registered_processes[id] = nullptr; + id_pool.push_front(id); +} + +} // namespace Core \ No newline at end of file -- cgit v1.2.3 From c85d7ccd79fb69bc096cd19bb8f95ac9534ffc23 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 24 Dec 2023 21:49:54 +0100 Subject: SMMU: Implement backing CPU page protect/unprotect --- src/core/device_memory_manager.inc | 82 +++++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 5 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 1f52b92d5..77410f72f 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -2,12 +2,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include #include "common/address_space.h" #include "common/address_space.inc" #include "common/alignment.h" +#include "common/assert.h" +#include "common/div_ceil.h" #include "common/scope_exit.h" #include "core/device_memory.h" #include "core/device_memory_manager.h" @@ -51,7 +54,11 @@ struct DeviceMemoryManagerAllocator { } DAddr AllocatePinned(size_t size) { - return pin_allocator.Allocate(size); + if constexpr (supports_pinning) { + return pin_allocator.Allocate(size); + } else { + return DAddr{}; + } } void DoInRange(DAddr address, size_t size, auto pin_func, auto main_func) { @@ -100,6 +107,7 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS), compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)) { impl = std::make_unique>(); + cached_pages = std::make_unique(); } template @@ -132,14 +140,14 @@ void DeviceMemoryManager::Free(DAddr start, size_t size) { template void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size_t size, - size_t p_id) { - Core::Memory::Memory* process_memory = registered_processes[p_id]; + size_t process_id) { + Core::Memory::Memory* process_memory = registered_processes[process_id]; size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; std::atomic_thread_fence(std::memory_order_acquire); for (size_t i = 0; i < num_pages; i++) { - auto* ptr = process_memory->GetPointer( - Common::ProcessAddress(virtual_address + i * Memory::YUZU_PAGESIZE)); + const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE; + auto* ptr = process_memory->GetPointer(Common::ProcessAddress(new_vaddress)); if (ptr == nullptr) [[unlikely]] { compressed_physical_ptr[start_page_d + i] = 0; continue; @@ -147,6 +155,7 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; compressed_physical_ptr[start_page_d + i] = phys_addr; compressed_device_addr[phys_addr - 1U] = static_cast(start_page_d + i); + InsertCPUBacking(start_page_d + i, new_vaddress, process_id); } std::atomic_thread_fence(std::memory_order_release); } @@ -159,6 +168,7 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { for (size_t i = 0; i < num_pages; i++) { auto phys_addr = compressed_physical_ptr[start_page_d + i]; compressed_physical_ptr[start_page_d + i] = 0; + cpu_backing_address[start_page_d + i] = 0; if (phys_addr != 0) { compressed_device_addr[phys_addr - 1] = 0; } @@ -301,4 +311,66 @@ void DeviceMemoryManager::UnregisterProcess(size_t id) { id_pool.push_front(id); } +template +void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size, s32 delta) { + u64 uncache_begin = 0; + u64 cache_begin = 0; + u64 uncache_bytes = 0; + u64 cache_bytes = 0; + const auto* MarkRegionCaching = &DeviceMemoryManager::DeviceMethods::MarkRegionCaching; + + std::atomic_thread_fence(std::memory_order_acquire); + const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); + size_t page = addr >> Memory::YUZU_PAGEBITS; + auto [process_id, base_vaddress] = ExtractCPUBacking(page); + size_t vpage = base_vaddress >> Memory::YUZU_PAGEBITS; + auto* memory_interface = registered_processes[process_id]; + for (; page != page_end; ++page) { + std::atomic_uint16_t& count = cached_pages->at(page >> 2).Count(page); + + if (delta > 0) { + ASSERT_MSG(count.load(std::memory_order::relaxed) < std::numeric_limits::max(), + "Count may overflow!"); + } else if (delta < 0) { + ASSERT_MSG(count.load(std::memory_order::relaxed) > 0, "Count may underflow!"); + } else { + ASSERT_MSG(false, "Delta must be non-zero!"); + } + + // Adds or subtracts 1, as count is a unsigned 8-bit value + count.fetch_add(static_cast(delta), std::memory_order_release); + + // Assume delta is either -1 or 1 + if (count.load(std::memory_order::relaxed) == 0) { + if (uncache_bytes == 0) { + uncache_begin = vpage; + } + uncache_bytes += Memory::YUZU_PAGESIZE; + } else if (uncache_bytes > 0) { + MarkRegionCaching(memory_interface, uncache_begin << Memory::YUZU_PAGEBITS, + uncache_bytes, false); + uncache_bytes = 0; + } + if (count.load(std::memory_order::relaxed) == 1 && delta > 0) { + if (cache_bytes == 0) { + cache_begin = vpage; + } + cache_bytes += Memory::YUZU_PAGESIZE; + } else if (cache_bytes > 0) { + MarkRegionCaching(memory_interface, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, + true); + cache_bytes = 0; + } + vpage++; + } + if (uncache_bytes > 0) { + MarkRegionCaching(memory_interface, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, + false); + } + if (cache_bytes > 0) { + MarkRegionCaching(memory_interface, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, + true); + } +} + } // namespace Core \ No newline at end of file -- cgit v1.2.3 From 0a2536a0df1f4aea406f2132d3edda0430acc9d1 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Mon, 25 Dec 2023 07:32:16 +0100 Subject: SMMU: Initial adaptation to video_core. --- src/core/device_memory_manager.inc | 72 +++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 8 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 77410f72f..8c5f82d31 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -105,7 +105,8 @@ template DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{reinterpret_cast(device_memory_.buffer.BackingBasePointer())}, interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS), - compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)) { + compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)), + cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); } @@ -144,10 +145,10 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size Core::Memory::Memory* process_memory = registered_processes[process_id]; size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; - std::atomic_thread_fence(std::memory_order_acquire); + std::scoped_lock lk(mapping_guard); for (size_t i = 0; i < num_pages; i++) { const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE; - auto* ptr = process_memory->GetPointer(Common::ProcessAddress(new_vaddress)); + auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress)); if (ptr == nullptr) [[unlikely]] { compressed_physical_ptr[start_page_d + i] = 0; continue; @@ -157,14 +158,14 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size compressed_device_addr[phys_addr - 1U] = static_cast(start_page_d + i); InsertCPUBacking(start_page_d + i, new_vaddress, process_id); } - std::atomic_thread_fence(std::memory_order_release); } template void DeviceMemoryManager::Unmap(DAddr address, size_t size) { size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; - std::atomic_thread_fence(std::memory_order_acquire); + interface->InvalidateRegion(address, size); + std::scoped_lock lk(mapping_guard); for (size_t i = 0; i < num_pages; i++) { auto phys_addr = compressed_physical_ptr[start_page_d + i]; compressed_physical_ptr[start_page_d + i] = 0; @@ -173,7 +174,6 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { compressed_device_addr[phys_addr - 1] = 0; } } - std::atomic_thread_fence(std::memory_order_release); } template @@ -256,6 +256,45 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o template void DeviceMemoryManager::ReadBlock(DAddr address, void* dest_pointer, size_t size) { + interface->FlushRegion(address, size); + WalkBlock( + address, size, + [&](size_t copy_amount, DAddr current_vaddr) { + LOG_ERROR( + HW_Memory, + "Unmapped Device ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, address, size); + std::memset(dest_pointer, 0, copy_amount); + }, + [&](size_t copy_amount, const u8* const src_ptr) { + std::memcpy(dest_pointer, src_ptr, copy_amount); + }, + [&](const std::size_t copy_amount) { + dest_pointer = static_cast(dest_pointer) + copy_amount; + }); +} + +template +void DeviceMemoryManager::WriteBlock(DAddr address, const void* src_pointer, size_t size) { + WalkBlock( + address, size, + [&](size_t copy_amount, DAddr current_vaddr) { + LOG_ERROR( + HW_Memory, + "Unmapped Device WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, address, size); + }, + [&](size_t copy_amount, u8* const dst_ptr) { + std::memcpy(dst_ptr, src_pointer, copy_amount); + }, + [&](const std::size_t copy_amount) { + src_pointer = static_cast(src_pointer) + copy_amount; + }); + interface->InvalidateRegion(address, size); +} + +template +void DeviceMemoryManager::ReadBlockUnsafe(DAddr address, void* dest_pointer, size_t size) { WalkBlock( address, size, [&](size_t copy_amount, DAddr current_vaddr) { @@ -274,7 +313,8 @@ void DeviceMemoryManager::ReadBlock(DAddr address, void* dest_pointer, s } template -void DeviceMemoryManager::WriteBlock(DAddr address, void* src_pointer, size_t size) { +void DeviceMemoryManager::WriteBlockUnsafe(DAddr address, const void* src_pointer, + size_t size) { WalkBlock( address, size, [&](size_t copy_amount, DAddr current_vaddr) { @@ -287,7 +327,7 @@ void DeviceMemoryManager::WriteBlock(DAddr address, void* src_pointer, s std::memcpy(dst_ptr, src_pointer, copy_amount); }, [&](const std::size_t copy_amount) { - src_pointer = static_cast(src_pointer) + copy_amount; + src_pointer = static_cast(src_pointer) + copy_amount; }); } @@ -313,6 +353,18 @@ void DeviceMemoryManager::UnregisterProcess(size_t id) { template void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size, s32 delta) { + bool locked = false; + auto lock = [&] { + if (!locked) { + counter_guard.lock(); + locked = true; + } + }; + SCOPE_EXIT({ + if (locked) { + counter_guard.unlock(); + } + }); u64 uncache_begin = 0; u64 cache_begin = 0; u64 uncache_bytes = 0; @@ -347,6 +399,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } uncache_bytes += Memory::YUZU_PAGESIZE; } else if (uncache_bytes > 0) { + lock(); MarkRegionCaching(memory_interface, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, false); uncache_bytes = 0; @@ -357,6 +410,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } cache_bytes += Memory::YUZU_PAGESIZE; } else if (cache_bytes > 0) { + lock(); MarkRegionCaching(memory_interface, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, true); cache_bytes = 0; @@ -364,10 +418,12 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size vpage++; } if (uncache_bytes > 0) { + lock(); MarkRegionCaching(memory_interface, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, false); } if (cache_bytes > 0) { + lock(); MarkRegionCaching(memory_interface, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, true); } -- cgit v1.2.3 From 34a8d0cc8e04b4b9d8e5a75e552f0adb31b5d718 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Fri, 29 Dec 2023 07:53:52 +0100 Subject: SMMU: Implement physical memory mirroring --- src/core/device_memory_manager.inc | 154 ++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 3 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 8c5f82d31..4fb3ad3ab 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -18,10 +18,117 @@ namespace Core { +namespace { + +class PhysicalAddressContainer { +public: + PhysicalAddressContainer() = default; + ~PhysicalAddressContainer() = default; + + void GatherValues(u32 start_entry, Common::ScratchBuffer& buffer) { + buffer.resize(8); + buffer.resize(0); + size_t index = 0; + const auto add_value = [&](u32 value) { + buffer[index] = value; + index++; + buffer.resize(index); + }; + + u32 iter_entry = start_entry; + Entry* current = &storage[iter_entry - 1]; + add_value(current->value); + while (current->next_entry != 0) { + iter_entry = current->next_entry; + current = &storage[iter_entry - 1]; + add_value(current->value); + } + } + + u32 Register(u32 value) { + return RegisterImplementation(value); + } + + void Register(u32 value, u32 start_entry) { + auto entry_id = RegisterImplementation(value); + u32 iter_entry = start_entry; + Entry* current = &storage[iter_entry - 1]; + while (current->next_entry != 0) { + iter_entry = current->next_entry; + current = &storage[iter_entry - 1]; + } + current->next_entry = entry_id; + } + + std::pair Unregister(u32 value, u32 start_entry) { + u32 iter_entry = start_entry; + Entry* previous{}; + Entry* current = &storage[iter_entry - 1]; + Entry* next{}; + bool more_than_one_remaining = false; + u32 result_start{start_entry}; + size_t count = 0; + while (current->value != value) { + count++; + previous = current; + iter_entry = current->next_entry; + current = &storage[iter_entry - 1]; + } + // Find next + u32 next_entry = current->next_entry; + if (next_entry != 0) { + next = &storage[next_entry - 1]; + more_than_one_remaining = next->next_entry != 0; + } + if (previous) { + previous->next_entry = next_entry; + } else { + result_start = next_entry; + } + free_entries.emplace_back(iter_entry); + return std::make_pair(more_than_one_remaining || count > 1, result_start); + } + + u32 ReleaseEntry(u32 start_entry) { + Entry* current = &storage[start_entry - 1]; + free_entries.emplace_back(start_entry); + return current->value; + } + +private: + u32 RegisterImplementation(u32 value) { + auto entry_id = GetNewEntry(); + auto& entry = storage[entry_id - 1]; + entry.next_entry = 0; + entry.value = value; + return entry_id; + } + u32 GetNewEntry() { + if (!free_entries.empty()) { + u32 result = free_entries.front(); + free_entries.pop_front(); + return result; + } + storage.emplace_back(); + u32 new_entry = static_cast(storage.size()); + return new_entry; + } + + struct Entry { + u32 next_entry{}; + u32 value{}; + }; + + std::deque storage; + std::deque free_entries; +}; + struct EmptyAllocator { EmptyAllocator([[maybe_unused]] DAddr address) {} }; +} // namespace + template struct DeviceMemoryManagerAllocator { static constexpr bool supports_pinning = DTraits::supports_pinning; @@ -38,6 +145,7 @@ struct DeviceMemoryManagerAllocator { std::conditional_t, EmptyAllocator> pin_allocator; Common::FlatAllocator main_allocator; + PhysicalAddressContainer multi_dev_address; /// Returns true when vaddr -> vaddr+size is fully contained in the buffer template @@ -109,6 +217,9 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); + for (size_t i = 0; i < 1ULL << (33 - 12); i++) { + compressed_device_addr[i] = 0; + } } template @@ -155,8 +266,19 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size } auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; compressed_physical_ptr[start_page_d + i] = phys_addr; - compressed_device_addr[phys_addr - 1U] = static_cast(start_page_d + i); InsertCPUBacking(start_page_d + i, new_vaddress, process_id); + const u32 base_dev = compressed_device_addr[phys_addr - 1U]; + const u32 new_dev = static_cast(start_page_d + i); + if (base_dev == 0) [[likely]] { + compressed_device_addr[phys_addr - 1U] = new_dev; + continue; + } + u32 start_id = base_dev & MULTI_MASK; + if ((base_dev >> MULTI_FLAG_BITS) == 0) { + start_id = impl->multi_dev_address.Register(base_dev); + compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id; + } + impl->multi_dev_address.Register(new_dev, start_id); } } @@ -170,12 +292,38 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { auto phys_addr = compressed_physical_ptr[start_page_d + i]; compressed_physical_ptr[start_page_d + i] = 0; cpu_backing_address[start_page_d + i] = 0; - if (phys_addr != 0) { - compressed_device_addr[phys_addr - 1] = 0; + if (phys_addr != 0) [[likely]] { + const u32 base_dev = compressed_device_addr[phys_addr - 1U]; + if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] { + compressed_device_addr[phys_addr - 1] = 0; + continue; + } + const auto [more_entries, new_start] = impl->multi_dev_address.Unregister( + static_cast(start_page_d + i), base_dev & MULTI_MASK); + if (!more_entries) { + compressed_device_addr[phys_addr - 1] = + impl->multi_dev_address.ReleaseEntry(new_start); + continue; + } + compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG; } } } +template +void DeviceMemoryManager::InnerGatherDeviceAddresses(Common::ScratchBuffer& buffer, + PAddr address) { + size_t phys_addr = address >> page_bits; + std::scoped_lock lk(mapping_guard); + u32 backing = compressed_device_addr[phys_addr]; + if ((backing >> MULTI_FLAG_BITS) != 0) { + impl->multi_dev_address.GatherValues(backing & MULTI_MASK, buffer); + return; + } + buffer.resize(1); + buffer[0] = backing; +} + template template T* DeviceMemoryManager::GetPointer(DAddr address) { -- cgit v1.2.3 From bad705f245f1591d8a92cd57bfbc87dd2cdb0fea Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Fri, 29 Dec 2023 09:27:37 +0100 Subject: SMMU: Fix Unregister on MultiAddress --- src/core/device_memory_manager.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 4fb3ad3ab..b3a5f3d8b 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -78,7 +78,7 @@ public: u32 next_entry = current->next_entry; if (next_entry != 0) { next = &storage[next_entry - 1]; - more_than_one_remaining = next->next_entry != 0; + more_than_one_remaining = next->next_entry != 0 || previous != nullptr; } if (previous) { previous->next_entry = next_entry; -- cgit v1.2.3 From 0adc09e0afcde345a5303efd73b3b7737245a7d9 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 30 Dec 2023 03:36:24 +0100 Subject: GPU-SMMU: Estimate game leak and preallocate device region. --- src/core/device_memory_manager.inc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index b3a5f3d8b..138eb5017 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -20,10 +20,10 @@ namespace Core { namespace { -class PhysicalAddressContainer { +class MultiAddressContainer { public: - PhysicalAddressContainer() = default; - ~PhysicalAddressContainer() = default; + MultiAddressContainer() = default; + ~MultiAddressContainer() = default; void GatherValues(u32 start_entry, Common::ScratchBuffer& buffer) { buffer.resize(8); @@ -145,7 +145,7 @@ struct DeviceMemoryManagerAllocator { std::conditional_t, EmptyAllocator> pin_allocator; Common::FlatAllocator main_allocator; - PhysicalAddressContainer multi_dev_address; + MultiAddressContainer multi_dev_address; /// Returns true when vaddr -> vaddr+size is fully contained in the buffer template -- cgit v1.2.3 From 303cd311621b25fbb8d55e0ed2cc4c3248de44ad Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 30 Dec 2023 04:37:25 +0100 Subject: SMMU: Add Android compatibility --- src/core/device_memory_manager.inc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 138eb5017..4f883cece 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -217,9 +217,6 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); - for (size_t i = 0; i < 1ULL << (33 - 12); i++) { - compressed_device_addr[i] = 0; - } } template @@ -517,7 +514,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size u64 cache_begin = 0; u64 uncache_bytes = 0; u64 cache_bytes = 0; - const auto* MarkRegionCaching = &DeviceMemoryManager::DeviceMethods::MarkRegionCaching; + const auto MarkRegionCaching = &DeviceMemoryManager::DeviceMethods::MarkRegionCaching; std::atomic_thread_fence(std::memory_order_acquire); const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); @@ -577,4 +574,4 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } } -} // namespace Core \ No newline at end of file +} // namespace Core -- cgit v1.2.3 From 9b11b9dce58b144d7f4407677a5e2dd52ca1888c Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 30 Dec 2023 08:20:29 +0100 Subject: SMMU: Simplify and remove old code. --- src/core/device_memory_manager.inc | 69 ++++---------------------------------- 1 file changed, 7 insertions(+), 62 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 4f883cece..e9d0efe19 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -131,81 +131,31 @@ struct EmptyAllocator { template struct DeviceMemoryManagerAllocator { - static constexpr bool supports_pinning = DTraits::supports_pinning; static constexpr size_t device_virtual_bits = DTraits::device_virtual_bits; - static constexpr size_t pin_bits = 32; static constexpr DAddr first_address = 1ULL << Memory::YUZU_PAGEBITS; - static constexpr DAddr max_pin_area = supports_pinning ? 1ULL << pin_bits : first_address; static constexpr DAddr max_device_area = 1ULL << device_virtual_bits; - DeviceMemoryManagerAllocator() - : pin_allocator(first_address), - main_allocator(supports_pinning ? 1ULL << pin_bits : first_address) {} + DeviceMemoryManagerAllocator() : main_allocator(first_address) {} - std::conditional_t, EmptyAllocator> - pin_allocator; Common::FlatAllocator main_allocator; MultiAddressContainer multi_dev_address; /// Returns true when vaddr -> vaddr+size is fully contained in the buffer template [[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept { - if constexpr (pin_area) { - return addr >= 0 && addr + size <= max_pin_area; - } else { - return addr >= max_pin_area && addr + size <= max_device_area; - } + return addr >= 0 && addr + size <= max_device_area; } DAddr Allocate(size_t size) { return main_allocator.Allocate(size); } - DAddr AllocatePinned(size_t size) { - if constexpr (supports_pinning) { - return pin_allocator.Allocate(size); - } else { - return DAddr{}; - } - } - - void DoInRange(DAddr address, size_t size, auto pin_func, auto main_func) { - if (IsInBounds(address, size)) { - pin_func(address, size); - return; - } - if (IsInBounds(address, size)) { - main_func(address, size); - return; - } - DAddr end_size = address + size - max_pin_area; - DAddr end_size2 = max_pin_area - address; - pin_func(address, end_size2); - main_func(max_pin_area, end_size); - } - void AllocateFixed(DAddr b_address, size_t b_size) { - if constexpr (supports_pinning) { - DoInRange( - b_address, b_size, - [this](DAddr address, size_t size) { pin_allocator.AllocateFixed(address, size); }, - [this](DAddr address, size_t size) { - main_allocator.AllocateFixed(address, size); - }); - } else { - main_allocator.AllocateFixed(b_address, b_size); - } + main_allocator.AllocateFixed(b_address, b_size); } void Free(DAddr b_address, size_t b_size) { - if constexpr (supports_pinning) { - DoInRange( - b_address, b_size, - [this](DAddr address, size_t size) { pin_allocator.Free(address, size); }, - [this](DAddr address, size_t size) { main_allocator.Free(address, size); }); - } else { - main_allocator.Free(b_address, b_size); - } + main_allocator.Free(b_address, b_size); } }; @@ -237,11 +187,6 @@ void DeviceMemoryManager::AllocateFixed(DAddr start, size_t size) { return impl->AllocateFixed(start, size); } -template -DAddr DeviceMemoryManager::AllocatePinned(size_t size) { - return impl->AllocatePinned(size); -} - template void DeviceMemoryManager::Free(DAddr start, size_t size) { impl->Free(start, size); @@ -523,10 +468,10 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size size_t vpage = base_vaddress >> Memory::YUZU_PAGEBITS; auto* memory_interface = registered_processes[process_id]; for (; page != page_end; ++page) { - std::atomic_uint16_t& count = cached_pages->at(page >> 2).Count(page); + std::atomic_uint8_t& count = cached_pages->at(page >> 3).Count(page); if (delta > 0) { - ASSERT_MSG(count.load(std::memory_order::relaxed) < std::numeric_limits::max(), + ASSERT_MSG(count.load(std::memory_order::relaxed) < std::numeric_limits::max(), "Count may overflow!"); } else if (delta < 0) { ASSERT_MSG(count.load(std::memory_order::relaxed) > 0, "Count may underflow!"); @@ -535,7 +480,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } // Adds or subtracts 1, as count is a unsigned 8-bit value - count.fetch_add(static_cast(delta), std::memory_order_release); + count.fetch_add(static_cast(delta), std::memory_order_release); // Assume delta is either -1 or 1 if (count.load(std::memory_order::relaxed) == 0) { -- cgit v1.2.3 From d8f1ce2f7640200d92a12698c42029316ac1a611 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 30 Dec 2023 09:37:16 +0100 Subject: SMMU: Add continuity tracking optimization. --- src/core/device_memory_manager.inc | 52 +++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index e9d0efe19..175f0cd5f 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -164,6 +164,7 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo : physical_base{reinterpret_cast(device_memory_.buffer.BackingBasePointer())}, interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS), compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)), + continuity_tracker(device_as_size >> Memory::YUZU_PAGEBITS), cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); @@ -194,7 +195,7 @@ void DeviceMemoryManager::Free(DAddr start, size_t size) { template void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size_t size, - size_t process_id) { + size_t process_id, bool track) { Core::Memory::Memory* process_memory = registered_processes[process_id]; size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; @@ -222,6 +223,9 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size } impl->multi_dev_address.Register(new_dev, start_id); } + if (track) { + TrackContinuityImpl(address, virtual_address, size, process_id); + } } template @@ -251,6 +255,47 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { } } } +template +void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtual_address, + size_t size, size_t process_id) { + Core::Memory::Memory* process_memory = registered_processes[process_id]; + size_t start_page_d = address >> Memory::YUZU_PAGEBITS; + size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; + uintptr_t last_ptr = 0; + size_t page_count = 1; + for (size_t i = num_pages; i > 0; i--) { + size_t index = i - 1; + const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE; + const uintptr_t new_ptr = reinterpret_cast( + process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress))); + if (new_ptr + page_size == last_ptr) { + page_count++; + } else { + page_count = 1; + } + last_ptr = new_ptr; + continuity_tracker[start_page_d + index] = static_cast(page_count); + } +} +template +u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) { + size_t page_index = src_addr >> page_bits; + size_t subbits = src_addr & page_mask; + if ((continuity_tracker[page_index] << page_bits) >= size + subbits) { + return GetPointer(src_addr); + } + return nullptr; +} + +template +const u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) const { + size_t page_index = src_addr >> page_bits; + size_t subbits = src_addr & page_mask; + if ((continuity_tracker[page_index] << page_bits) >= size + subbits) { + return GetPointer(src_addr); + } + return nullptr; +} template void DeviceMemoryManager::InnerGatherDeviceAddresses(Common::ScratchBuffer& buffer, @@ -322,12 +367,13 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o std::size_t page_offset = addr & Memory::YUZU_PAGEMASK; while (remaining_size) { + const size_t next_pages = static_cast(continuity_tracker[page_index]); const std::size_t copy_amount = - std::min(static_cast(Memory::YUZU_PAGESIZE) - page_offset, remaining_size); + std::min((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size); const auto current_vaddr = static_cast((page_index << Memory::YUZU_PAGEBITS) + page_offset); SCOPE_EXIT({ - page_index++; + page_index += next_pages; page_offset = 0; increment(copy_amount); remaining_size -= copy_amount; -- cgit v1.2.3 From 590d9b7e1d875e0403fb87cfcd4a8d52c50e2b81 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 31 Dec 2023 20:55:15 +0100 Subject: Core: Clang format and other small issues. --- src/core/device_memory_manager.inc | 50 ++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 15 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 175f0cd5f..a0eb4214e 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -12,6 +12,7 @@ #include "common/assert.h" #include "common/div_ceil.h" #include "common/scope_exit.h" +#include "common/settings.h" #include "core/device_memory.h" #include "core/device_memory_manager.h" #include "core/memory.h" @@ -162,20 +163,39 @@ struct DeviceMemoryManagerAllocator { template DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{reinterpret_cast(device_memory_.buffer.BackingBasePointer())}, - interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS), - compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)), + device_inter{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS), + compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == + Settings::MemoryLayout::Memory_4Gb + ? physical_min_bits + : physical_max_bits) - + Memory::YUZU_PAGEBITS)), continuity_tracker(device_as_size >> Memory::YUZU_PAGEBITS), cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); + + const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; + for (size_t i = 0; i < total_virtual; i++) { + compressed_physical_ptr[i] = 0; + continuity_tracker[i] = 1; + cpu_backing_address[i] = 0; + } + const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == + Settings::MemoryLayout::Memory_4Gb + ? physical_min_bits + : physical_max_bits) - + Memory::YUZU_PAGEBITS); + for (size_t i = 0; i < total_phys; i++) { + compressed_device_addr[i] = 0; + } } template DeviceMemoryManager::~DeviceMemoryManager() = default; template -void DeviceMemoryManager::BindInterface(DeviceInterface* interface_) { - interface = interface_; +void DeviceMemoryManager::BindInterface(DeviceInterface* device_inter_) { + device_inter = device_inter_; } template @@ -232,7 +252,7 @@ template void DeviceMemoryManager::Unmap(DAddr address, size_t size) { size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; - interface->InvalidateRegion(address, size); + device_inter->InvalidateRegion(address, size); std::scoped_lock lk(mapping_guard); for (size_t i = 0; i < num_pages; i++) { auto phys_addr = compressed_physical_ptr[start_page_d + i]; @@ -392,7 +412,7 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o template void DeviceMemoryManager::ReadBlock(DAddr address, void* dest_pointer, size_t size) { - interface->FlushRegion(address, size); + device_inter->FlushRegion(address, size); WalkBlock( address, size, [&](size_t copy_amount, DAddr current_vaddr) { @@ -426,7 +446,7 @@ void DeviceMemoryManager::WriteBlock(DAddr address, const void* src_poin [&](const std::size_t copy_amount) { src_pointer = static_cast(src_pointer) + copy_amount; }); - interface->InvalidateRegion(address, size); + device_inter->InvalidateRegion(address, size); } template @@ -468,14 +488,14 @@ void DeviceMemoryManager::WriteBlockUnsafe(DAddr address, const void* sr } template -size_t DeviceMemoryManager::RegisterProcess(Memory::Memory* memory_interface) { +size_t DeviceMemoryManager::RegisterProcess(Memory::Memory* memory_device_inter) { size_t new_id; if (!id_pool.empty()) { new_id = id_pool.front(); id_pool.pop_front(); - registered_processes[new_id] = memory_interface; + registered_processes[new_id] = memory_device_inter; } else { - registered_processes.emplace_back(memory_interface); + registered_processes.emplace_back(memory_device_inter); new_id = registered_processes.size() - 1U; } return new_id; @@ -512,7 +532,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size size_t page = addr >> Memory::YUZU_PAGEBITS; auto [process_id, base_vaddress] = ExtractCPUBacking(page); size_t vpage = base_vaddress >> Memory::YUZU_PAGEBITS; - auto* memory_interface = registered_processes[process_id]; + auto* memory_device_inter = registered_processes[process_id]; for (; page != page_end; ++page) { std::atomic_uint8_t& count = cached_pages->at(page >> 3).Count(page); @@ -536,7 +556,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size uncache_bytes += Memory::YUZU_PAGESIZE; } else if (uncache_bytes > 0) { lock(); - MarkRegionCaching(memory_interface, uncache_begin << Memory::YUZU_PAGEBITS, + MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, false); uncache_bytes = 0; } @@ -547,7 +567,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size cache_bytes += Memory::YUZU_PAGESIZE; } else if (cache_bytes > 0) { lock(); - MarkRegionCaching(memory_interface, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, + MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, true); cache_bytes = 0; } @@ -555,12 +575,12 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } if (uncache_bytes > 0) { lock(); - MarkRegionCaching(memory_interface, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, + MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, false); } if (cache_bytes > 0) { lock(); - MarkRegionCaching(memory_interface, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, + MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, true); } } -- cgit v1.2.3 From a874ab0133459b713205a87738234fae03dc715b Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Fri, 5 Jan 2024 03:47:48 +0100 Subject: SMMU: Fix 8Gb layout. --- src/core/device_memory_manager.inc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index a0eb4214e..5241293b6 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -340,8 +340,8 @@ T* DeviceMemoryManager::GetPointer(DAddr address) { if (phys_addr == 0) [[unlikely]] { return nullptr; } - return GetPointerFromRaw( - static_cast(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + offset)); + return GetPointerFromRaw((static_cast(phys_addr - 1) << Memory::YUZU_PAGEBITS) + + offset); } template @@ -353,8 +353,8 @@ const T* DeviceMemoryManager::GetPointer(DAddr address) const { if (phys_addr == 0) [[unlikely]] { return nullptr; } - return GetPointerFromRaw( - static_cast(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + offset)); + return GetPointerFromRaw((static_cast(phys_addr - 1) << Memory::YUZU_PAGEBITS) + + offset); } template @@ -405,7 +405,7 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o continue; } auto* mem_ptr = GetPointerFromRaw( - static_cast(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + page_offset)); + (static_cast(phys_addr - 1) << Memory::YUZU_PAGEBITS) + page_offset); on_memory(copy_amount, mem_ptr); } } -- cgit v1.2.3 From 067284733075fb0604dbcdc6238d23cfa27c5355 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 7 Jan 2024 04:50:06 +0100 Subject: SMMU: Fix Right Shift UB. --- src/core/device_memory_manager.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 5241293b6..d7b4abacc 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -301,7 +301,7 @@ template u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) { size_t page_index = src_addr >> page_bits; size_t subbits = src_addr & page_mask; - if ((continuity_tracker[page_index] << page_bits) >= size + subbits) { + if ((static_cast(continuity_tracker[page_index]) << page_bits) >= size + subbits) { return GetPointer(src_addr); } return nullptr; @@ -311,7 +311,7 @@ template const u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) const { size_t page_index = src_addr >> page_bits; size_t subbits = src_addr & page_mask; - if ((continuity_tracker[page_index] << page_bits) >= size + subbits) { + if ((static_cast(continuity_tracker[page_index]) << page_bits) >= size + subbits) { return GetPointer(src_addr); } return nullptr; -- cgit v1.2.3 From beb438bb0bede8b8906a41f7a1ad7b010ec3ec60 Mon Sep 17 00:00:00 2001 From: Liam Date: Mon, 15 Jan 2024 21:47:59 -0500 Subject: nvdrv: use static typing for SessionId, smmu Asid types --- src/core/device_memory_manager.inc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index d7b4abacc..f6e4ad874 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -215,8 +215,8 @@ void DeviceMemoryManager::Free(DAddr start, size_t size) { template void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size_t size, - size_t process_id, bool track) { - Core::Memory::Memory* process_memory = registered_processes[process_id]; + Asid asid, bool track) { + Core::Memory::Memory* process_memory = registered_processes[asid.id]; size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; std::scoped_lock lk(mapping_guard); @@ -229,7 +229,7 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size } auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; compressed_physical_ptr[start_page_d + i] = phys_addr; - InsertCPUBacking(start_page_d + i, new_vaddress, process_id); + InsertCPUBacking(start_page_d + i, new_vaddress, asid); const u32 base_dev = compressed_device_addr[phys_addr - 1U]; const u32 new_dev = static_cast(start_page_d + i); if (base_dev == 0) [[likely]] { @@ -244,7 +244,7 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size impl->multi_dev_address.Register(new_dev, start_id); } if (track) { - TrackContinuityImpl(address, virtual_address, size, process_id); + TrackContinuityImpl(address, virtual_address, size, asid); } } @@ -277,8 +277,8 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { } template void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtual_address, - size_t size, size_t process_id) { - Core::Memory::Memory* process_memory = registered_processes[process_id]; + size_t size, Asid asid) { + Core::Memory::Memory* process_memory = registered_processes[asid.id]; size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; uintptr_t last_ptr = 0; @@ -488,8 +488,8 @@ void DeviceMemoryManager::WriteBlockUnsafe(DAddr address, const void* sr } template -size_t DeviceMemoryManager::RegisterProcess(Memory::Memory* memory_device_inter) { - size_t new_id; +Asid DeviceMemoryManager::RegisterProcess(Memory::Memory* memory_device_inter) { + size_t new_id{}; if (!id_pool.empty()) { new_id = id_pool.front(); id_pool.pop_front(); @@ -498,13 +498,13 @@ size_t DeviceMemoryManager::RegisterProcess(Memory::Memory* memory_devic registered_processes.emplace_back(memory_device_inter); new_id = registered_processes.size() - 1U; } - return new_id; + return Asid{new_id}; } template -void DeviceMemoryManager::UnregisterProcess(size_t id) { - registered_processes[id] = nullptr; - id_pool.push_front(id); +void DeviceMemoryManager::UnregisterProcess(Asid asid) { + registered_processes[asid.id] = nullptr; + id_pool.push_front(asid.id); } template @@ -530,9 +530,9 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size std::atomic_thread_fence(std::memory_order_acquire); const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); size_t page = addr >> Memory::YUZU_PAGEBITS; - auto [process_id, base_vaddress] = ExtractCPUBacking(page); + auto [asid, base_vaddress] = ExtractCPUBacking(page); size_t vpage = base_vaddress >> Memory::YUZU_PAGEBITS; - auto* memory_device_inter = registered_processes[process_id]; + auto* memory_device_inter = registered_processes[asid.id]; for (; page != page_end; ++page) { std::atomic_uint8_t& count = cached_pages->at(page >> 3).Count(page); -- cgit v1.2.3 From 748465f5a578fcd99f91e0591ac773940172a72e Mon Sep 17 00:00:00 2001 From: Liam Date: Mon, 15 Jan 2024 21:56:38 -0500 Subject: device_memory_manager: use unique_lock for update --- src/core/device_memory_manager.inc | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) (limited to 'src/core/device_memory_manager.inc') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index f6e4ad874..8ce122872 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -509,18 +509,12 @@ void DeviceMemoryManager::UnregisterProcess(Asid asid) { template void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size, s32 delta) { - bool locked = false; - auto lock = [&] { - if (!locked) { - counter_guard.lock(); - locked = true; + std::unique_lock lk(counter_guard, std::defer_lock); + const auto Lock = [&] { + if (!lk) { + lk.lock(); } }; - SCOPE_EXIT({ - if (locked) { - counter_guard.unlock(); - } - }); u64 uncache_begin = 0; u64 cache_begin = 0; u64 uncache_bytes = 0; @@ -555,7 +549,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } uncache_bytes += Memory::YUZU_PAGESIZE; } else if (uncache_bytes > 0) { - lock(); + Lock(); MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, false); uncache_bytes = 0; @@ -566,7 +560,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } cache_bytes += Memory::YUZU_PAGESIZE; } else if (cache_bytes > 0) { - lock(); + Lock(); MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, true); cache_bytes = 0; @@ -574,12 +568,12 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size vpage++; } if (uncache_bytes > 0) { - lock(); + Lock(); MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, false); } if (cache_bytes > 0) { - lock(); + Lock(); MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, true); } -- cgit v1.2.3