diff options
Diffstat (limited to 'src/video_core/memory_manager.cpp')
| -rw-r--r-- | src/video_core/memory_manager.cpp | 147 |
1 files changed, 55 insertions, 92 deletions
diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index a3efd365e..1e090279f 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -16,36 +16,63 @@ namespace Tegra { -MemoryManager::MemoryManager(Core::System& system_) - : system{system_}, page_table(page_table_size) {} +MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 page_bits_) + : system{system_}, address_space_bits{address_space_bits_}, page_bits{page_bits_}, entries{}, + page_table{address_space_bits, address_space_bits + page_bits - 38, page_bits} { + address_space_size = 1ULL << address_space_bits; + allocate_start = address_space_bits > 32 ? 1ULL << 32 : 0; + page_size = 1ULL << page_bits; + page_mask = page_size - 1ULL; + const u64 page_table_bits = address_space_bits - cpu_page_bits; + const u64 page_table_size = 1ULL << page_table_bits; + page_table_mask = page_table_size - 1; + + entries.resize(page_table_size / 32, 0); +} MemoryManager::~MemoryManager() = default; -void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { - rasterizer = rasterizer_; +MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const { + position = position >> page_bits; + const u64 entry_mask = entries[position / 32]; + const size_t sub_index = position % 32; + return static_cast<EntryType>((entry_mask >> (2 * sub_index)) & 0x03ULL); +} + +void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) { + position = position >> page_bits; + const u64 entry_mask = entries[position / 32]; + const size_t sub_index = position % 32; + entries[position / 32] = + (~(3ULL << sub_index * 2) & entry_mask) | (static_cast<u64>(entry) << sub_index * 2); } -GPUVAddr MemoryManager::UpdateRange(GPUVAddr gpu_addr, PageEntry page_entry, std::size_t size) { +template <MemoryManager::EntryType entry_type> +GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, + size_t size) { u64 remaining_size{size}; + if constexpr (entry_type == EntryType::Mapped) { + page_table.ReserveRange(gpu_addr, size); + } for (u64 offset{}; offset < size; offset += page_size) { - if (remaining_size < page_size) { - SetPageEntry(gpu_addr + offset, page_entry + offset, remaining_size); - } else { - SetPageEntry(gpu_addr + offset, page_entry + offset); + const GPUVAddr current_gpu_addr = gpu_addr + offset; + SetEntry(current_gpu_addr, entry_type); + if constexpr (entry_type == EntryType::Mapped) { + const VAddr current_cpu_addr = cpu_addr + offset; + const auto index = PageEntryIndex(current_gpu_addr); + page_table[index] = static_cast<u32>(current_cpu_addr >> 12ULL); } remaining_size -= page_size; } return gpu_addr; } +void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { + rasterizer = rasterizer_; +} + GPUVAddr MemoryManager::Map(VAddr cpu_addr, GPUVAddr gpu_addr, std::size_t size) { - const auto it = std::ranges::lower_bound(map_ranges, gpu_addr, {}, &MapRange::first); - if (it != map_ranges.end() && it->first == gpu_addr) { - it->second = size; - } else { - map_ranges.insert(it, MapRange{gpu_addr, size}); - } - return UpdateRange(gpu_addr, cpu_addr, size); + return PageTableOp<EntryType::Mapped>(gpu_addr, cpu_addr, size); } GPUVAddr MemoryManager::MapAllocate(VAddr cpu_addr, std::size_t size, std::size_t align) { @@ -62,13 +89,6 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { if (size == 0) { return; } - const auto it = std::ranges::lower_bound(map_ranges, gpu_addr, {}, &MapRange::first); - if (it != map_ranges.end()) { - ASSERT(it->first == gpu_addr); - map_ranges.erase(it); - } else { - ASSERT_MSG(false, "Unmapping non-existent GPU address=0x{:x}", gpu_addr); - } const auto submapped_ranges = GetSubmappedRange(gpu_addr, size); for (const auto& [map_addr, map_size] : submapped_ranges) { @@ -79,63 +99,23 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { rasterizer->UnmapMemory(*cpu_addr, map_size); } - UpdateRange(gpu_addr, PageEntry::State::Unmapped, size); + PageTableOp<EntryType::Free>(gpu_addr, 0, size); } std::optional<GPUVAddr> MemoryManager::AllocateFixed(GPUVAddr gpu_addr, std::size_t size) { for (u64 offset{}; offset < size; offset += page_size) { - if (!GetPageEntry(gpu_addr + offset).IsUnmapped()) { + if (GetEntry(gpu_addr + offset) != EntryType::Free) { return std::nullopt; } } - return UpdateRange(gpu_addr, PageEntry::State::Allocated, size); + return PageTableOp<EntryType::Reserved>(gpu_addr, 0, size); } GPUVAddr MemoryManager::Allocate(std::size_t size, std::size_t align) { return *AllocateFixed(*FindFreeRange(size, align), size); } -void MemoryManager::TryLockPage(PageEntry page_entry, std::size_t size) { - if (!page_entry.IsValid()) { - return; - } - - ASSERT(system.CurrentProcess() - ->PageTable() - .LockForDeviceAddressSpace(page_entry.ToAddress(), size) - .IsSuccess()); -} - -void MemoryManager::TryUnlockPage(PageEntry page_entry, std::size_t size) { - if (!page_entry.IsValid()) { - return; - } - - ASSERT(system.CurrentProcess() - ->PageTable() - .UnlockForDeviceAddressSpace(page_entry.ToAddress(), size) - .IsSuccess()); -} - -PageEntry MemoryManager::GetPageEntry(GPUVAddr gpu_addr) const { - return page_table[PageEntryIndex(gpu_addr)]; -} - -void MemoryManager::SetPageEntry(GPUVAddr gpu_addr, PageEntry page_entry, std::size_t size) { - // TODO(bunnei): We should lock/unlock device regions. This currently causes issues due to - // improper tracking, but should be fixed in the future. - - //// Unlock the old page - // TryUnlockPage(page_table[PageEntryIndex(gpu_addr)], size); - - //// Lock the new page - // TryLockPage(page_entry, size); - auto& current_page = page_table[PageEntryIndex(gpu_addr)]; - - current_page = page_entry; -} - std::optional<GPUVAddr> MemoryManager::FindFreeRange(std::size_t size, std::size_t align, bool start_32bit_address) const { if (!align) { @@ -145,9 +125,9 @@ std::optional<GPUVAddr> MemoryManager::FindFreeRange(std::size_t size, std::size } u64 available_size{}; - GPUVAddr gpu_addr{start_32bit_address ? address_space_start_low : address_space_start}; + GPUVAddr gpu_addr{allocate_start}; while (gpu_addr + available_size < address_space_size) { - if (GetPageEntry(gpu_addr + available_size).IsUnmapped()) { + if (GetEntry(gpu_addr + available_size) == EntryType::Free) { available_size += page_size; if (available_size >= size) { @@ -168,15 +148,12 @@ std::optional<GPUVAddr> MemoryManager::FindFreeRange(std::size_t size, std::size } std::optional<VAddr> MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const { - if (gpu_addr == 0) { - return std::nullopt; - } - const auto page_entry{GetPageEntry(gpu_addr)}; - if (!page_entry.IsValid()) { + if (GetEntry(gpu_addr) != EntryType::Mapped) { return std::nullopt; } - return page_entry.ToAddress() + (gpu_addr & page_mask); + const VAddr cpu_addr_base = static_cast<VAddr>(page_table[PageEntryIndex(gpu_addr)]) << 12ULL; + return cpu_addr_base + (gpu_addr & page_mask); } std::optional<VAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t size) const { @@ -227,10 +204,6 @@ template void MemoryManager::Write<u32>(GPUVAddr addr, u32 data); template void MemoryManager::Write<u64>(GPUVAddr addr, u64 data); u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) { - if (!GetPageEntry(gpu_addr).IsValid()) { - return {}; - } - const auto address{GpuToCpuAddress(gpu_addr)}; if (!address) { return {}; @@ -240,10 +213,6 @@ u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) { } const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const { - if (!GetPageEntry(gpu_addr).IsValid()) { - return {}; - } - const auto address{GpuToCpuAddress(gpu_addr)}; if (!address) { return {}; @@ -252,12 +221,6 @@ const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const { return system.Memory().GetPointer(*address); } -size_t MemoryManager::BytesToMapEnd(GPUVAddr gpu_addr) const noexcept { - auto it = std::ranges::upper_bound(map_ranges, gpu_addr, {}, &MapRange::first); - --it; - return it->second - (gpu_addr - it->first); -} - void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, bool is_safe) const { std::size_t remaining_size{size}; @@ -268,7 +231,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: const std::size_t copy_amount{ std::min(static_cast<std::size_t>(page_size) - page_offset, remaining_size)}; const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (page_addr && *page_addr != 0) { + if (page_addr) { const auto src_addr{*page_addr + page_offset}; if (is_safe) { // Flush must happen on the rasterizer interface, such that memory is always @@ -307,7 +270,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe const std::size_t copy_amount{ std::min(static_cast<std::size_t>(page_size) - page_offset, remaining_size)}; const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (page_addr && *page_addr != 0) { + if (page_addr) { const auto dest_addr{*page_addr + page_offset}; if (is_safe) { @@ -392,7 +355,7 @@ bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) cons size_t page_index{gpu_addr >> page_bits}; const size_t page_last{(gpu_addr + size + page_size - 1) >> page_bits}; while (page_index < page_last) { - if (!page_table[page_index].IsValid() || page_table[page_index].ToAddress() == 0) { + if (GetEntry(page_index << page_bits) == EntryType::Free) { return false; } ++page_index; @@ -408,7 +371,7 @@ std::vector<std::pair<GPUVAddr, std::size_t>> MemoryManager::GetSubmappedRange( size_t page_offset{gpu_addr & page_mask}; std::optional<std::pair<GPUVAddr, std::size_t>> last_segment{}; std::optional<VAddr> old_page_addr{}; - const auto extend_size = [&last_segment, &page_index, &page_offset](std::size_t bytes) { + const auto extend_size = [this, &last_segment, &page_index, &page_offset](std::size_t bytes) { if (!last_segment) { const GPUVAddr new_base_addr = (page_index << page_bits) + page_offset; last_segment = {new_base_addr, bytes}; |