diff --git a/src/common/error.cpp b/src/common/error.cpp index a1125b6e..972a3428 100644 --- a/src/common/error.cpp +++ b/src/common/error.cpp @@ -20,7 +20,7 @@ std::string NativeErrorToString(int e) { DWORD res = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_IGNORE_INSERTS, - nullptr, e, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + nullptr, e, MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), reinterpret_cast(&err_str), 1, nullptr); if (!res) { return "(FormatMessageA failed to format error)"; diff --git a/src/core/address_space.cpp b/src/core/address_space.cpp index fe9780f5..420ed59a 100644 --- a/src/core/address_space.cpp +++ b/src/core/address_space.cpp @@ -6,7 +6,6 @@ #include "common/error.h" #include "core/address_space.h" #include "core/libraries/kernel/memory_management.h" -#include "core/virtual_memory.h" #ifdef _WIN32 #include @@ -17,11 +16,40 @@ namespace Core { static constexpr size_t BackingSize = SCE_KERNEL_MAIN_DMEM_SIZE; -static constexpr size_t VirtualSize = USER_MAX - USER_MIN + 1; #ifdef _WIN32 struct AddressSpace::Impl { Impl() : process{GetCurrentProcess()} { + // Allocate virtual address placeholder for our address space. + MEM_ADDRESS_REQUIREMENTS req{}; + MEM_EXTENDED_PARAMETER param{}; + req.LowestStartingAddress = reinterpret_cast(SYSTEM_MANAGED_MIN); + // The ending address must align to page boundary - 1 + // https://stackoverflow.com/questions/54223343/virtualalloc2-with-memextendedparameteraddressrequirements-always-produces-error + req.HighestEndingAddress = reinterpret_cast(USER_MIN + UserSize - 1); + req.Alignment = 0; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = &req; + + // Typically, lower parts of system managed area is already reserved in windows. + // If reservation fails attempt again by reducing the area size a little bit. + // System managed is about 31GB in size so also cap the number of times we can reduce it + // to a reasonable amount. + static constexpr size_t ReductionOnFail = 1_GB; + static constexpr size_t MaxReductions = 10; + virtual_size = SystemSize + UserSize + ReductionOnFail; + for (u32 i = 0; i < MaxReductions && !virtual_base; i++) { + virtual_size -= ReductionOnFail; + virtual_base = static_cast(VirtualAlloc2(process, NULL, virtual_size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, + PAGE_NOACCESS, ¶m, 1)); + } + ASSERT_MSG(virtual_base, "Unable to reserve virtual address space!"); + + // Initializer placeholder tracker + const uintptr_t virtual_addr = reinterpret_cast(virtual_base); + placeholders.insert({virtual_addr, virtual_addr + virtual_size}); + // Allocate backing file that represents the total physical memory. backing_handle = CreateFileMapping2(INVALID_HANDLE_VALUE, nullptr, FILE_MAP_WRITE | FILE_MAP_READ, @@ -35,21 +63,6 @@ struct AddressSpace::Impl { void* const ret = MapViewOfFile3(backing_handle, process, backing_base, 0, BackingSize, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0); ASSERT(ret == backing_base); - // Allocate virtual address placeholder for our address space. - MEM_ADDRESS_REQUIREMENTS req{}; - MEM_EXTENDED_PARAMETER param{}; - req.LowestStartingAddress = reinterpret_cast(USER_MIN); - req.HighestEndingAddress = reinterpret_cast(USER_MAX); - req.Alignment = 0; - param.Type = MemExtendedParameterAddressRequirements; - param.Pointer = &req; - virtual_base = static_cast(VirtualAlloc2(process, nullptr, VirtualSize, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, - PAGE_NOACCESS, ¶m, 1)); - ASSERT(virtual_base); - - const uintptr_t virtual_addr = reinterpret_cast(virtual_base); - placeholders.insert({virtual_addr, virtual_addr + VirtualSize}); } ~Impl() { @@ -71,7 +84,7 @@ struct AddressSpace::Impl { } } - void* MapUser(VAddr virtual_addr, PAddr phys_addr, size_t size, ULONG prot) { + void* Map(VAddr virtual_addr, PAddr phys_addr, size_t size, ULONG prot) { const auto it = placeholders.find(virtual_addr); ASSERT_MSG(it != placeholders.end(), "Cannot map already mapped region"); ASSERT_MSG(virtual_addr >= it->lower() && virtual_addr + size <= it->upper(), @@ -106,54 +119,25 @@ struct AddressSpace::Impl { ptr = MapViewOfFile3(backing_handle, process, reinterpret_cast(virtual_addr), phys_addr, size, MEM_REPLACE_PLACEHOLDER, prot, nullptr, 0); } else { - ptr = VirtualAlloc2(process, reinterpret_cast(virtual_addr), size, - MEM_REPLACE_PLACEHOLDER, prot, nullptr, 0); + ptr = + VirtualAlloc2(process, reinterpret_cast(virtual_addr), size, + MEM_RESERVE | MEM_COMMIT | MEM_REPLACE_PLACEHOLDER, prot, nullptr, 0); } ASSERT_MSG(ptr, "{}", Common::GetLastErrorMsg()); return ptr; } - void* MapPrivate(VAddr virtual_addr, size_t size, u64 alignment, ULONG prot, - bool no_commit = false) { - // Map a private allocation - PVOID addr = reinterpret_cast(virtual_addr); - MEM_ADDRESS_REQUIREMENTS req{}; - MEM_EXTENDED_PARAMETER param{}; - // req.LowestStartingAddress = - // (virtual_addr == 0 ? reinterpret_cast(SYSTEM_MANAGED_MIN) - // : reinterpret_cast(virtual_addr)); - req.HighestEndingAddress = reinterpret_cast(SYSTEM_MANAGED_MAX); - req.Alignment = alignment < 64_KB ? 0 : alignment; - param.Type = MemExtendedParameterAddressRequirements; - param.Pointer = &req; - ULONG alloc_type = MEM_RESERVE | (alignment > 2_MB ? MEM_LARGE_PAGES : 0); - if (!no_commit) { - alloc_type |= MEM_COMMIT; - } - // Check if the area has been reserved beforehand (typically for tesselation buffer) - // and in that case don't reserve it again as Windows complains. - if (virtual_addr) { - MEMORY_BASIC_INFORMATION info; - VirtualQuery(addr, &info, sizeof(info)); - if (info.State == MEM_RESERVE) { - alloc_type &= ~MEM_RESERVE; - } - } - void* ptr{}; - if (virtual_addr) { - ptr = VirtualAlloc2(process, addr, size, alloc_type, prot, NULL, 0); - ASSERT_MSG(ptr && VAddr(ptr) == virtual_addr, "{}", Common::GetLastErrorMsg()); + void Unmap(VAddr virtual_addr, PAddr phys_addr, size_t size) { + bool ret; + if (phys_addr != -1) { + ret = UnmapViewOfFile2(process, reinterpret_cast(virtual_addr), + MEM_PRESERVE_PLACEHOLDER); } else { - ptr = VirtualAlloc2(process, nullptr, size, alloc_type, prot, ¶m, 1); - ASSERT_MSG(ptr, "{}", Common::GetLastErrorMsg()); + ret = VirtualFreeEx(process, reinterpret_cast(virtual_addr), size, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER); } - return ptr; - } - - void UnmapUser(VAddr virtual_addr, size_t size) { - const bool ret = UnmapViewOfFile2(process, reinterpret_cast(virtual_addr), - MEM_PRESERVE_PLACEHOLDER); - ASSERT_MSG(ret, "Unmap operation on virtual_addr={:#X} failed", virtual_addr); + ASSERT_MSG(ret, "Unmap operation on virtual_addr={:#X} failed: {}", virtual_addr, + Common::GetLastErrorMsg()); // The unmap call will create a new placeholder region. We need to see if we can coalesce it // with neighbors. @@ -186,12 +170,6 @@ struct AddressSpace::Impl { placeholders.insert({placeholder_start, placeholder_end}); } - void UnmapPrivate(VAddr virtual_addr, size_t size) { - const bool ret = - VirtualFreeEx(process, reinterpret_cast(virtual_addr), 0, MEM_RELEASE); - ASSERT_MSG(ret, "{}", Common::GetLastErrorMsg()); - } - void Protect(VAddr virtual_addr, size_t size, bool read, bool write, bool execute) { DWORD new_flags{}; if (read && write) { @@ -221,6 +199,7 @@ struct AddressSpace::Impl { HANDLE backing_handle{}; u8* backing_base{}; u8* virtual_base{}; + size_t virtual_size{}; boost::icl::separate_interval_set placeholders; }; #else @@ -239,22 +218,12 @@ struct AddressSpace::Impl { UNREACHABLE(); } - void* MapUser(VAddr virtual_addr, PAddr phys_addr, size_t size, PosixPageProtection prot) { + void* Map(VAddr virtual_addr, PAddr phys_addr, size_t size, PosixPageProtection prot) { UNREACHABLE(); return nullptr; } - void* MapPrivate(VAddr virtual_addr, size_t size, u64 alignment, PosixPageProtection prot, - bool no_commit = false) { - UNREACHABLE(); - return nullptr; - } - - void UnmapUser(VAddr virtual_addr, size_t size) { - UNREACHABLE(); - } - - void UnmapPrivate(VAddr virtual_addr, size_t size) { + void Unmap(VAddr virtual_addr, PAddr phys_addr, size_t size) { UNREACHABLE(); } @@ -264,36 +233,30 @@ struct AddressSpace::Impl { u8* backing_base{}; u8* virtual_base{}; + size_t virtual_size{}; }; #endif AddressSpace::AddressSpace() : impl{std::make_unique()} { virtual_base = impl->virtual_base; backing_base = impl->backing_base; + virtual_size = impl->virtual_size; } AddressSpace::~AddressSpace() = default; -void* AddressSpace::Map(VAddr virtual_addr, size_t size, u64 alignment, PAddr phys_addr) { - if (virtual_addr >= USER_MIN) { - return impl->MapUser(virtual_addr, phys_addr, size, PAGE_READWRITE); - } - return impl->MapPrivate(virtual_addr, size, alignment, PAGE_READWRITE); +void* AddressSpace::Map(VAddr virtual_addr, size_t size, u64 alignment, PAddr phys_addr, + bool is_exec) { + return impl->Map(virtual_addr, phys_addr, size, + is_exec ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE); } -void AddressSpace::Unmap(VAddr virtual_addr, size_t size) { - if (virtual_addr >= USER_MIN) { - return impl->UnmapUser(virtual_addr, size); - } - return impl->UnmapPrivate(virtual_addr, size); +void AddressSpace::Unmap(VAddr virtual_addr, size_t size, PAddr phys_addr) { + return impl->Unmap(virtual_addr, phys_addr, size); } void AddressSpace::Protect(VAddr virtual_addr, size_t size, MemoryPermission perms) { return impl->Protect(virtual_addr, size, true, true, true); } -void* AddressSpace::Reserve(size_t size, u64 alignment) { - return impl->MapPrivate(0, size, alignment, PAGE_READWRITE, true); -} - } // namespace Core diff --git a/src/core/address_space.h b/src/core/address_space.h index 0e344358..ccaeb199 100644 --- a/src/core/address_space.h +++ b/src/core/address_space.h @@ -14,9 +14,23 @@ enum class MemoryPermission : u32 { Write = 1 << 1, ReadWrite = Read | Write, Execute = 1 << 2, + ReadWriteExecute = Read | Write | Execute, }; DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission) +constexpr VAddr SYSTEM_RESERVED = 0x800000000ULL; +constexpr VAddr CODE_BASE_OFFSET = 0x100000000ULL; +constexpr VAddr SYSTEM_MANAGED_MIN = 0x0000040000ULL; +constexpr VAddr SYSTEM_MANAGED_MAX = 0x07FFFFBFFFULL; +constexpr VAddr USER_MIN = 0x1000000000ULL; +constexpr VAddr USER_MAX = 0xFBFFFFFFFFULL; + +// User area size is normally larger than this. However games are unlikely to map to high +// regions of that area, so by default we allocate a smaller virtual address space (about 1/4th). +// to save space on page tables. +static constexpr size_t UserSize = 1ULL << 38; +static constexpr size_t SystemSize = USER_MIN - SYSTEM_MANAGED_MIN; + /** * Represents the user virtual address space backed by a dmem memory block */ @@ -25,12 +39,15 @@ public: explicit AddressSpace(); ~AddressSpace(); - [[nodiscard]] u8* VirtualBase() noexcept { - return virtual_base; + [[nodiscard]] VAddr VirtualBase() noexcept { + return reinterpret_cast(virtual_base); } [[nodiscard]] const u8* VirtualBase() const noexcept { return virtual_base; } + [[nodiscard]] size_t VirtualSize() const noexcept { + return virtual_size; + } /** * @brief Maps memory to the specified virtual address. @@ -42,20 +59,20 @@ public: * If zero is provided the mapping is considered as private. * @return A pointer to the mapped memory. */ - void* Map(VAddr virtual_addr, size_t size, u64 alignment = 0, PAddr phys_addr = -1); + void* Map(VAddr virtual_addr, size_t size, u64 alignment = 0, PAddr phys_addr = -1, + bool exec = false); /// Unmaps specified virtual memory area. - void Unmap(VAddr virtual_addr, size_t size); + void Unmap(VAddr virtual_addr, size_t size, PAddr phys_addr); void Protect(VAddr virtual_addr, size_t size, MemoryPermission perms); - void* Reserve(size_t size, u64 alignment); - private: struct Impl; std::unique_ptr impl; u8* backing_base{}; u8* virtual_base{}; + size_t virtual_size{}; }; } // namespace Core diff --git a/src/core/libraries/gnmdriver/gnmdriver.cpp b/src/core/libraries/gnmdriver/gnmdriver.cpp index c33a0546..71040f32 100644 --- a/src/core/libraries/gnmdriver/gnmdriver.cpp +++ b/src/core/libraries/gnmdriver/gnmdriver.cpp @@ -10,7 +10,6 @@ #include "core/libraries/gnmdriver/gnmdriver.h" #include "core/libraries/libs.h" #include "core/libraries/videoout/video_out.h" -#include "core/memory.h" #include "core/platform.h" #include "video_core/amdgpu/liverpool.h" #include "video_core/amdgpu/pm4_cmds.h" @@ -40,10 +39,7 @@ struct AscQueueInfo { u32 ring_size_dw; }; static VideoCore::SlotVector asc_queues{}; - -static constexpr u32 TessellationFactorRingSize = 128_KB; -static constexpr u32 TessellationFactorRingAlignment = 64_KB; // toolkit is using this alignment -VAddr tessellation_factors_ring_addr{0}; +static constexpr VAddr tessellation_factors_ring_addr = 0xFF0000000ULL; static void DumpCommandList(std::span cmd_list, const std::string& postfix) { using namespace Common::FS; @@ -624,11 +620,6 @@ int PS4_SYSV_ABI sceGnmGetShaderStatus() { VAddr PS4_SYSV_ABI sceGnmGetTheTessellationFactorRingBufferBaseAddress() { LOG_TRACE(Lib_GnmDriver, "called"); // Actual virtual buffer address is hardcoded in the driver to 0xff00'000 - if (tessellation_factors_ring_addr == 0) { - auto* memory = Core::Memory::Instance(); - tessellation_factors_ring_addr = - memory->Reserve(TessellationFactorRingSize, TessellationFactorRingAlignment); - } return tessellation_factors_ring_addr; } diff --git a/src/core/libraries/kernel/memory_management.cpp b/src/core/libraries/kernel/memory_management.cpp index 11711f57..05ff359b 100644 --- a/src/core/libraries/kernel/memory_management.cpp +++ b/src/core/libraries/kernel/memory_management.cpp @@ -55,6 +55,20 @@ s32 PS4_SYSV_ABI sceKernelAllocateMainDirectMemory(size_t len, size_t alignment, physAddrOut); } +s32 PS4_SYSV_ABI sceKernelCheckedReleaseDirectMemory(u64 start, size_t len) { + LOG_INFO(Kernel_Vmm, "called start = {:#x}, len = {:#x}", start, len); + auto* memory = Core::Memory::Instance(); + memory->Free(start, len); + return ORBIS_OK; +} + +s32 PS4_SYSV_ABI sceKernelVirtualQuery(const void* addr, int flags, OrbisVirtualQueryInfo* info, + size_t infoSize) { + LOG_INFO(Kernel_Vmm, "called addr = {}, flags = {:#x}", fmt::ptr(addr), flags); + auto* memory = Core::Memory::Instance(); + return memory->VirtualQuery(std::bit_cast(addr), flags, info); +} + int PS4_SYSV_ABI sceKernelMapNamedDirectMemory(void** addr, u64 len, int prot, int flags, s64 directMemoryStart, u64 alignment, const char* name) { @@ -83,7 +97,7 @@ int PS4_SYSV_ABI sceKernelMapNamedDirectMemory(void** addr, u64 len, int prot, i const auto map_flags = static_cast(flags); auto* memory = Core::Memory::Instance(); return memory->MapMemory(addr, in_addr, len, mem_prot, map_flags, Core::VMAType::Direct, "", - directMemoryStart, alignment); + false, directMemoryStart, alignment); } int PS4_SYSV_ABI sceKernelMapDirectMemory(void** addr, u64 len, int prot, int flags, diff --git a/src/core/libraries/kernel/memory_management.h b/src/core/libraries/kernel/memory_management.h index c0085d31..65139532 100644 --- a/src/core/libraries/kernel/memory_management.h +++ b/src/core/libraries/kernel/memory_management.h @@ -3,6 +3,7 @@ #pragma once +#include "common/bit_field.h" #include "common/types.h" constexpr u64 SCE_KERNEL_MAIN_DMEM_SIZE = 5376_MB; // ~ 6GB @@ -36,6 +37,22 @@ struct OrbisQueryInfo { int memoryType; }; +struct OrbisVirtualQueryInfo { + uintptr_t start; + uintptr_t end; + size_t offset; + s32 protection; + s32 memory_type; + union { + BitField<0, 1, u32> is_flexible; + BitField<1, 1, u32> is_direct; + BitField<2, 1, u32> is_stack; + BitField<3, 1, u32> is_pooled; + BitField<4, 1, u32> is_commited; + }; + std::array name; +}; + u64 PS4_SYSV_ABI sceKernelGetDirectMemorySize(); int PS4_SYSV_ABI sceKernelAllocateDirectMemory(s64 searchStart, s64 searchEnd, u64 len, u64 alignment, int memoryType, s64* physAddrOut); @@ -46,6 +63,9 @@ int PS4_SYSV_ABI sceKernelMapDirectMemory(void** addr, u64 len, int prot, int fl s64 directMemoryStart, u64 alignment); s32 PS4_SYSV_ABI sceKernelAllocateMainDirectMemory(size_t len, size_t alignment, int memoryType, s64* physAddrOut); +s32 PS4_SYSV_ABI sceKernelCheckedReleaseDirectMemory(u64 start, size_t len); +s32 PS4_SYSV_ABI sceKernelVirtualQuery(const void* addr, int flags, OrbisVirtualQueryInfo* info, + size_t infoSize); s32 PS4_SYSV_ABI sceKernelMapNamedFlexibleMemory(void** addrInOut, std::size_t len, int prot, int flags, const char* name); s32 PS4_SYSV_ABI sceKernelMapFlexibleMemory(void** addr_in_out, std::size_t len, int prot, diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 5180a633..0c5d127f 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later -#include #include "common/alignment.h" #include "common/assert.h" #include "common/scope_exit.h" @@ -13,49 +12,82 @@ namespace Core { MemoryManager::MemoryManager() { - // Insert a virtual memory area that covers the user area. - const size_t user_size = USER_MAX - USER_MIN - 1; - vma_map.emplace(USER_MIN, VirtualMemoryArea{USER_MIN, user_size}); + // Insert an area that covers direct memory physical block. + dmem_map.emplace(0, DirectMemoryArea{0, SCE_KERNEL_MAIN_DMEM_SIZE}); - // Insert a virtual memory area that covers the system managed area. - const size_t sys_size = SYSTEM_MANAGED_MAX - SYSTEM_MANAGED_MIN - 1; - vma_map.emplace(SYSTEM_MANAGED_MIN, VirtualMemoryArea{SYSTEM_MANAGED_MIN, sys_size}); + // Insert a virtual memory area that covers the entire area we manage. + const VAddr virtual_base = impl.VirtualBase(); + const size_t virtual_size = impl.VirtualSize(); + vma_map.emplace(virtual_base, VirtualMemoryArea{virtual_base, virtual_size}); } MemoryManager::~MemoryManager() = default; PAddr MemoryManager::Allocate(PAddr search_start, PAddr search_end, size_t size, u64 alignment, int memory_type) { - PAddr free_addr = search_start; + std::scoped_lock lk{mutex}; - // Iterate through allocated blocked and find the next free position - for (const auto& block : allocations) { - const PAddr end = block.base + block.size; - free_addr = std::max(end, free_addr); + auto dmem_area = FindDmemArea(search_start); + + const auto is_suitable = [&] { + return dmem_area->second.is_free && dmem_area->second.size >= size; + }; + while (!is_suitable() && dmem_area->second.GetEnd() <= search_end) { + dmem_area++; } + ASSERT_MSG(is_suitable(), "Unable to find free direct memory area"); // Align free position + PAddr free_addr = dmem_area->second.base; free_addr = alignment > 0 ? Common::AlignUp(free_addr, alignment) : free_addr; - ASSERT(free_addr >= search_start && free_addr + size <= search_end); // Add the allocated region to the list and commit its pages. - allocations.emplace_back(free_addr, size, memory_type); + auto& area = AddDmemAllocation(free_addr, size); + area.memory_type = memory_type; + area.is_free = false; return free_addr; } void MemoryManager::Free(PAddr phys_addr, size_t size) { - const auto it = std::ranges::find_if(allocations, [&](const auto& alloc) { - return alloc.base == phys_addr && alloc.size == size; - }); - ASSERT(it != allocations.end()); + std::scoped_lock lk{mutex}; - // Free the ranges. - allocations.erase(it); + const auto dmem_area = FindDmemArea(phys_addr); + ASSERT(dmem_area != dmem_map.end() && dmem_area->second.base == phys_addr && + dmem_area->second.size == size); + + // Release any dmem mappings that reference this physical block. + std::vector> remove_list; + for (const auto& [addr, mapping] : vma_map) { + if (mapping.type != VMAType::Direct) { + continue; + } + if (mapping.phys_base <= phys_addr && phys_addr < mapping.phys_base + mapping.size) { + LOG_INFO(Kernel_Vmm, "Unmaping direct mapping {:#x} with size {:#x}", addr, + mapping.size); + // Unmaping might erase from vma_map. We can't do it here. + remove_list.emplace_back(addr, mapping.size); + } + } + for (const auto& [addr, size] : remove_list) { + UnmapMemory(addr, size); + } + + // Mark region as free and attempt to coalesce it with neighbours. + auto& area = dmem_area->second; + area.is_free = true; + area.memory_type = 0; + MergeAdjacent(dmem_map, dmem_area); } int MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, size_t size, MemoryProt prot, MemoryMapFlags flags, VMAType type, std::string_view name, - PAddr phys_addr, u64 alignment) { + bool is_exec, PAddr phys_addr, u64 alignment) { + std::scoped_lock lk{mutex}; + + // When virtual addr is zero, force it to virtual_base. The guest cannot pass Fixed + // flag so we will take the branch that searches for free (or reserved) mappings. + virtual_addr = (virtual_addr == 0) ? impl.VirtualBase() : virtual_addr; + VAddr mapped_addr = alignment > 0 ? Common::AlignUp(virtual_addr, alignment) : virtual_addr; SCOPE_EXIT { auto& new_vma = AddMapping(mapped_addr, size); @@ -65,18 +97,11 @@ int MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, size_t size, M new_vma.type = type; if (type == VMAType::Direct) { + new_vma.phys_base = phys_addr; MapVulkanMemory(mapped_addr, size); } }; - // When virtual addr is zero let the address space manager pick the address. - // Alignment matters here as we let the OS pick the address. - if (virtual_addr == 0) { - *out_addr = impl.Map(virtual_addr, size, alignment); - mapped_addr = std::bit_cast(*out_addr); - return ORBIS_OK; - } - // Fixed mapping means the virtual address must exactly match the provided one. if (True(flags & MemoryMapFlags::Fixed) && True(flags & MemoryMapFlags::NoOverwrite)) { // This should return SCE_KERNEL_ERROR_ENOMEM but shouldn't normally happen. @@ -92,21 +117,28 @@ int MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, size_t size, M it++; } ASSERT(it != vma_map.end()); - mapped_addr = alignment > 0 ? Common::AlignUp(it->second.base, alignment) : it->second.base; + const VAddr base = it->second.base; + mapped_addr = alignment > 0 ? Common::AlignUp(base, alignment) : base; } // Perform the mapping. - *out_addr = impl.Map(mapped_addr, size, alignment, phys_addr); + *out_addr = impl.Map(mapped_addr, size, alignment, phys_addr, is_exec); return ORBIS_OK; } void MemoryManager::UnmapMemory(VAddr virtual_addr, size_t size) { + std::scoped_lock lk{mutex}; + // TODO: Partial unmaps are technically supported by the guest. const auto it = vma_map.find(virtual_addr); ASSERT_MSG(it != vma_map.end() && it->first == virtual_addr, "Attempting to unmap partially mapped range"); - if (it->second.type == VMAType::Direct) { + const auto type = it->second.type; + fmt::print("{}\n", u32(type)); + std::fflush(stdout); + const PAddr phys_addr = type == VMAType::Direct ? it->second.phys_base : -1; + if (type == VMAType::Direct) { UnmapVulkanMemory(virtual_addr, size); } @@ -115,13 +147,15 @@ void MemoryManager::UnmapMemory(VAddr virtual_addr, size_t size) { vma.type = VMAType::Free; vma.prot = MemoryProt::NoAccess; vma.phys_base = 0; - MergeAdjacent(it); + MergeAdjacent(vma_map, it); // Unmap the memory region. - impl.Unmap(virtual_addr, size); + impl.Unmap(virtual_addr, size, phys_addr); } int MemoryManager::QueryProtection(VAddr addr, void** start, void** end, u32* prot) { + std::scoped_lock lk{mutex}; + const auto it = FindVMA(addr); const auto& vma = it->second; ASSERT_MSG(vma.type != VMAType::Free, "Provided address is not mapped"); @@ -132,18 +166,70 @@ int MemoryManager::QueryProtection(VAddr addr, void** start, void** end, u32* pr return ORBIS_OK; } -int MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next, - Libraries::Kernel::OrbisQueryInfo* out_info) { - const auto it = std::ranges::find_if(allocations, [&](const DirectMemoryArea& alloc) { - return alloc.base <= addr && addr < alloc.base + alloc.size; - }); - if (it == allocations.end()) { - return SCE_KERNEL_ERROR_EACCES; +int MemoryManager::VirtualQuery(VAddr addr, int flags, + Libraries::Kernel::OrbisVirtualQueryInfo* info) { + auto it = FindVMA(addr); + if (it->second.type == VMAType::Free && flags == 1) { + it++; + } + if (it->second.type == VMAType::Free) { + LOG_WARNING(Kernel_Vmm, "VirtualQuery on free memory region"); + return ORBIS_KERNEL_ERROR_EACCES; } - out_info->start = it->base; - out_info->end = it->base + it->size; - out_info->memoryType = it->memory_type; + const auto& vma = it->second; + info->start = vma.base; + info->end = vma.base + vma.size; + info->is_flexible.Assign(vma.type == VMAType::Flexible); + info->is_direct.Assign(vma.type == VMAType::Direct); + info->is_commited.Assign(vma.type != VMAType::Free); + if (vma.type == VMAType::Direct) { + const auto dmem_it = FindDmemArea(vma.phys_base); + ASSERT(dmem_it != dmem_map.end()); + info->memory_type = dmem_it->second.memory_type; + } + + return ORBIS_OK; +} + +int MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next, + Libraries::Kernel::OrbisQueryInfo* out_info) { + std::scoped_lock lk{mutex}; + + auto dmem_area = FindDmemArea(addr); + if (dmem_area->second.is_free && find_next) { + dmem_area++; + } + + if (dmem_area == dmem_map.end() || dmem_area->second.is_free) { + LOG_ERROR(Core, "Unable to find allocated direct memory region to query!"); + return ORBIS_KERNEL_ERROR_EACCES; + } + + const auto& area = dmem_area->second; + out_info->start = area.base; + out_info->end = area.GetEnd(); + out_info->memoryType = area.memory_type; + return ORBIS_OK; +} + +int MemoryManager::DirectQueryAvailable(PAddr search_start, PAddr search_end, size_t alignment, + PAddr* phys_addr_out, size_t* size_out) { + std::scoped_lock lk{mutex}; + + auto dmem_area = FindDmemArea(search_start); + PAddr paddr{}; + size_t max_size{}; + while (dmem_area != dmem_map.end() && dmem_area->second.GetEnd() <= search_end) { + if (dmem_area->second.size > max_size) { + paddr = dmem_area->second.base; + max_size = dmem_area->second.size; + } + dmem_area++; + } + + *phys_addr_out = alignment > 0 ? Common::AlignUp(paddr, alignment) : paddr; + *size_out = max_size; return ORBIS_OK; } @@ -178,6 +264,30 @@ VirtualMemoryArea& MemoryManager::AddMapping(VAddr virtual_addr, size_t size) { return vma_handle->second; } +DirectMemoryArea& MemoryManager::AddDmemAllocation(PAddr addr, size_t size) { + auto dmem_handle = FindDmemArea(addr); + ASSERT_MSG(dmem_handle != dmem_map.end(), "Physical address not in dmem_map"); + + const DirectMemoryArea& area = dmem_handle->second; + ASSERT_MSG(area.is_free && area.base <= addr, + "Adding an allocation to already allocated region"); + + const PAddr start_in_area = addr - area.base; + const PAddr end_in_vma = start_in_area + size; + ASSERT_MSG(end_in_vma <= area.size, "Mapping cannot fit inside free region"); + + if (end_in_vma != area.size) { + // Split VMA at the end of the allocated region + Split(dmem_handle, end_in_vma); + } + if (start_in_area != 0) { + // Split VMA at the start of the allocated region + dmem_handle = Split(dmem_handle, start_in_area); + } + + return dmem_handle->second; +} + MemoryManager::VMAHandle MemoryManager::Split(VMAHandle vma_handle, size_t offset_in_vma) { auto& old_vma = vma_handle->second; ASSERT(offset_in_vma < old_vma.size && offset_in_vma > 0); @@ -193,24 +303,17 @@ MemoryManager::VMAHandle MemoryManager::Split(VMAHandle vma_handle, size_t offse return vma_map.emplace_hint(std::next(vma_handle), new_vma.base, new_vma); } -MemoryManager::VMAHandle MemoryManager::MergeAdjacent(VMAHandle iter) { - const auto next_vma = std::next(iter); - if (next_vma != vma_map.end() && iter->second.CanMergeWith(next_vma->second)) { - iter->second.size += next_vma->second.size; - vma_map.erase(next_vma); - } +MemoryManager::DMemHandle MemoryManager::Split(DMemHandle dmem_handle, size_t offset_in_area) { + auto& old_area = dmem_handle->second; + ASSERT(offset_in_area < old_area.size && offset_in_area > 0); - if (iter != vma_map.begin()) { - auto prev_vma = std::prev(iter); - if (prev_vma->second.CanMergeWith(iter->second)) { - prev_vma->second.size += iter->second.size; - vma_map.erase(iter); - iter = prev_vma; - } - } + auto new_area = old_area; + old_area.size = offset_in_area; + new_area.base += offset_in_area; + new_area.size -= offset_in_area; - return iter; -} + return dmem_map.emplace_hint(std::next(dmem_handle), new_area.base, new_area); +}; void MemoryManager::MapVulkanMemory(VAddr addr, size_t size) { return; diff --git a/src/core/memory.h b/src/core/memory.h index 87cd9188..c5d130c0 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include #include @@ -11,6 +12,7 @@ #include "common/singleton.h" #include "common/types.h" #include "core/address_space.h" +#include "core/libraries/kernel/memory_management.h" #include "video_core/renderer_vulkan/vk_common.h" namespace Vulkan { @@ -47,12 +49,28 @@ enum class VMAType : u32 { Flexible = 3, Pooled = 4, Stack = 5, + Code = 6, }; struct DirectMemoryArea { PAddr base = 0; size_t size = 0; int memory_type = 0; + bool is_free = true; + + PAddr GetEnd() const { + return base + size; + } + + bool CanMergeWith(const DirectMemoryArea& next) const { + if (base + size != next.base) { + return false; + } + if (is_free != next.is_free) { + return false; + } + return true; + } }; struct VirtualMemoryArea { @@ -81,14 +99,10 @@ struct VirtualMemoryArea { } }; -constexpr VAddr SYSTEM_RESERVED = 0x800000000ULL; -constexpr VAddr CODE_BASE_OFFSET = 0x100000000ULL; -constexpr VAddr SYSTEM_MANAGED_MIN = 0x0000040000ULL; -constexpr VAddr SYSTEM_MANAGED_MAX = 0x07FFFFBFFFULL; -constexpr VAddr USER_MIN = 0x1000000000ULL; -constexpr VAddr USER_MAX = 0xFBFFFFFFFFULL; - class MemoryManager { + using DMemMap = std::map; + using DMemHandle = DMemMap::iterator; + using VMAMap = std::map; using VMAHandle = VMAMap::iterator; @@ -107,35 +121,57 @@ public: int MapMemory(void** out_addr, VAddr virtual_addr, size_t size, MemoryProt prot, MemoryMapFlags flags, VMAType type, std::string_view name = "", - PAddr phys_addr = -1, u64 alignment = 0); + bool is_exec = false, PAddr phys_addr = -1, u64 alignment = 0); void UnmapMemory(VAddr virtual_addr, size_t size); int QueryProtection(VAddr addr, void** start, void** end, u32* prot); + int VirtualQuery(VAddr addr, int flags, Libraries::Kernel::OrbisVirtualQueryInfo* info); + int DirectMemoryQuery(PAddr addr, bool find_next, Libraries::Kernel::OrbisQueryInfo* out_info); - VAddr Reserve(size_t size, u64 alignment) { - return reinterpret_cast(impl.Reserve(size, alignment)); - } + int DirectQueryAvailable(PAddr search_start, PAddr search_end, size_t alignment, + PAddr* phys_addr_out, size_t* size_out); std::pair GetVulkanBuffer(VAddr addr); private: VMAHandle FindVMA(VAddr target) { - // Return first the VMA with base >= target. - const auto it = vma_map.lower_bound(target); - if (it != vma_map.end() && it->first == target) { - return it; + return std::prev(vma_map.upper_bound(target)); + } + + DMemHandle FindDmemArea(PAddr target) { + return std::prev(dmem_map.upper_bound(target)); + } + + template + Handle MergeAdjacent(auto& handle_map, Handle iter) { + const auto next_vma = std::next(iter); + if (next_vma != handle_map.end() && iter->second.CanMergeWith(next_vma->second)) { + iter->second.size += next_vma->second.size; + handle_map.erase(next_vma); } - return std::prev(it); + + if (iter != handle_map.begin()) { + auto prev_vma = std::prev(iter); + if (prev_vma->second.CanMergeWith(iter->second)) { + prev_vma->second.size += iter->second.size; + handle_map.erase(iter); + iter = prev_vma; + } + } + + return iter; } VirtualMemoryArea& AddMapping(VAddr virtual_addr, size_t size); + DirectMemoryArea& AddDmemAllocation(PAddr addr, size_t size); + VMAHandle Split(VMAHandle vma_handle, size_t offset_in_vma); - VMAHandle MergeAdjacent(VMAHandle iter); + DMemHandle Split(DMemHandle dmem_handle, size_t offset_in_area); void MapVulkanMemory(VAddr addr, size_t size); @@ -143,8 +179,9 @@ private: private: AddressSpace impl; - std::vector allocations; + DMemMap dmem_map; VMAMap vma_map; + std::recursive_mutex mutex; struct MappedMemory { vk::UniqueBuffer buffer; diff --git a/src/core/module.cpp b/src/core/module.cpp index 102fbe1b..661e4cbd 100644 --- a/src/core/module.cpp +++ b/src/core/module.cpp @@ -7,6 +7,7 @@ #include "common/logging/log.h" #include "common/string_util.h" #include "core/aerolib/aerolib.h" +#include "core/memory.h" #include "core/module.h" #include "core/tls.h" #include "core/virtual_memory.h" @@ -81,8 +82,11 @@ void Module::LoadModuleToMemory() { aligned_base_size = Common::AlignUp(base_size, BlockAlign); // Map module segments (and possible TLS trampolines) - base_virtual_addr = VirtualMemory::memory_alloc(LoadAddress, aligned_base_size + TrampolineSize, - VirtualMemory::MemoryMode::ExecuteReadWrite); + auto* memory = Core::Memory::Instance(); + void** out_addr = reinterpret_cast(&base_virtual_addr); + const auto name = file.filename().string(); + memory->MapMemory(out_addr, LoadAddress, aligned_base_size + TrampolineSize, + MemoryProt::CpuReadWrite, MemoryMapFlags::Fixed, VMAType::Code, name, true); LoadAddress += CODE_BASE_INCR * (1 + aligned_base_size / CODE_BASE_INCR); // Initialize trampoline generator. diff --git a/src/main.cpp b/src/main.cpp index e72cb136..2a97780d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -20,6 +20,7 @@ #include "core/libraries/libs.h" #include "core/libraries/videoout/video_out.h" #include "core/linker.h" +#include "core/memory.h" #include "input/controller.h" #include "sdl_window.h" @@ -30,6 +31,8 @@ int main(int argc, char* argv[]) { fmt::print("Usage: {} \n", argv[0]); return -1; } + // Initialize memory system as early as possible to reserve mappings. + [[maybe_unused]] const auto* memory = Core::Memory::Instance(); const auto config_dir = Common::FS::GetUserPath(Common::FS::PathType::UserDir); Config::load(config_dir / "config.toml"); Common::Log::Initialize();