video_core: Implement guest buffer manager (#373)

* video_core: Introduce buffer cache

* video_core: Use multi level page table for caches

* renderer_vulkan: Remove unused stream buffer

* fix build

* oops forgot optimize off
This commit is contained in:
TheTurtle 2024-08-08 15:02:10 +03:00 committed by GitHub
parent 159be2c7f4
commit 381ba8c7a5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
55 changed files with 2697 additions and 1039 deletions

View File

@ -283,6 +283,7 @@ set(COMMON src/common/logging/backend.cpp
src/common/native_clock.h src/common/native_clock.h
src/common/path_util.cpp src/common/path_util.cpp
src/common/path_util.h src/common/path_util.h
src/common/object_pool.h
src/common/polyfill_thread.h src/common/polyfill_thread.h
src/common/rdtsc.cpp src/common/rdtsc.cpp
src/common/rdtsc.h src/common/rdtsc.h
@ -294,6 +295,7 @@ set(COMMON src/common/logging/backend.cpp
src/common/thread.h src/common/thread.h
src/common/types.h src/common/types.h
src/common/uint128.h src/common/uint128.h
src/common/unique_function.h
src/common/version.h src/common/version.h
src/common/ntapi.h src/common/ntapi.h
src/common/ntapi.cpp src/common/ntapi.cpp
@ -367,7 +369,6 @@ set(CORE src/core/aerolib/stubs.cpp
) )
set(SHADER_RECOMPILER src/shader_recompiler/exception.h set(SHADER_RECOMPILER src/shader_recompiler/exception.h
src/shader_recompiler/object_pool.h
src/shader_recompiler/profile.h src/shader_recompiler/profile.h
src/shader_recompiler/recompiler.cpp src/shader_recompiler/recompiler.cpp
src/shader_recompiler/recompiler.h src/shader_recompiler/recompiler.h
@ -451,6 +452,13 @@ set(VIDEO_CORE src/video_core/amdgpu/liverpool.cpp
src/video_core/amdgpu/pm4_cmds.h src/video_core/amdgpu/pm4_cmds.h
src/video_core/amdgpu/pm4_opcodes.h src/video_core/amdgpu/pm4_opcodes.h
src/video_core/amdgpu/resource.h src/video_core/amdgpu/resource.h
src/video_core/buffer_cache/buffer.cpp
src/video_core/buffer_cache/buffer.h
src/video_core/buffer_cache/buffer_cache.cpp
src/video_core/buffer_cache/buffer_cache.h
src/video_core/buffer_cache/memory_tracker_base.h
src/video_core/buffer_cache/range_set.h
src/video_core/buffer_cache/word_manager.h
src/video_core/renderer_vulkan/liverpool_to_vk.cpp src/video_core/renderer_vulkan/liverpool_to_vk.cpp
src/video_core/renderer_vulkan/liverpool_to_vk.h src/video_core/renderer_vulkan/liverpool_to_vk.h
src/video_core/renderer_vulkan/renderer_vulkan.cpp src/video_core/renderer_vulkan/renderer_vulkan.cpp
@ -479,8 +487,6 @@ set(VIDEO_CORE src/video_core/amdgpu/liverpool.cpp
src/video_core/renderer_vulkan/vk_scheduler.h src/video_core/renderer_vulkan/vk_scheduler.h
src/video_core/renderer_vulkan/vk_shader_util.cpp src/video_core/renderer_vulkan/vk_shader_util.cpp
src/video_core/renderer_vulkan/vk_shader_util.h src/video_core/renderer_vulkan/vk_shader_util.h
src/video_core/renderer_vulkan/vk_stream_buffer.cpp
src/video_core/renderer_vulkan/vk_stream_buffer.h
src/video_core/renderer_vulkan/vk_swapchain.cpp src/video_core/renderer_vulkan/vk_swapchain.cpp
src/video_core/renderer_vulkan/vk_swapchain.h src/video_core/renderer_vulkan/vk_swapchain.h
src/video_core/texture_cache/image.cpp src/video_core/texture_cache/image.cpp
@ -496,6 +502,9 @@ set(VIDEO_CORE src/video_core/amdgpu/liverpool.cpp
src/video_core/texture_cache/tile_manager.cpp src/video_core/texture_cache/tile_manager.cpp
src/video_core/texture_cache/tile_manager.h src/video_core/texture_cache/tile_manager.h
src/video_core/texture_cache/types.h src/video_core/texture_cache/types.h
src/video_core/page_manager.cpp
src/video_core/page_manager.h
src/video_core/multi_level_page_table.h
src/video_core/renderdoc.cpp src/video_core/renderdoc.cpp
src/video_core/renderdoc.h src/video_core/renderdoc.h
) )

View File

@ -8,7 +8,7 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
namespace Shader { namespace Common {
template <typename T> template <typename T>
requires std::is_destructible_v<T> requires std::is_destructible_v<T>
@ -104,4 +104,4 @@ private:
size_t new_chunk_size{}; size_t new_chunk_size{};
}; };
} // namespace Shader } // namespace Common

61
src/common/unique_function.h Executable file
View File

@ -0,0 +1,61 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <memory>
#include <utility>
namespace Common {
/// General purpose function wrapper similar to std::function.
/// Unlike std::function, the captured values don't have to be copyable.
/// This class can be moved but not copied.
template <typename ResultType, typename... Args>
class UniqueFunction {
class CallableBase {
public:
virtual ~CallableBase() = default;
virtual ResultType operator()(Args&&...) = 0;
};
template <typename Functor>
class Callable final : public CallableBase {
public:
Callable(Functor&& functor_) : functor{std::move(functor_)} {}
~Callable() override = default;
ResultType operator()(Args&&... args) override {
return functor(std::forward<Args>(args)...);
}
private:
Functor functor;
};
public:
UniqueFunction() = default;
template <typename Functor>
UniqueFunction(Functor&& functor)
: callable{std::make_unique<Callable<Functor>>(std::move(functor))} {}
UniqueFunction& operator=(UniqueFunction&& rhs) noexcept = default;
UniqueFunction(UniqueFunction&& rhs) noexcept = default;
UniqueFunction& operator=(const UniqueFunction&) = delete;
UniqueFunction(const UniqueFunction&) = delete;
ResultType operator()(Args&&... args) const {
return (*callable)(std::forward<Args>(args)...);
}
explicit operator bool() const noexcept {
return static_cast<bool>(callable);
}
private:
std::unique_ptr<CallableBase> callable;
};
} // namespace Common

View File

@ -465,7 +465,7 @@ int PS4_SYSV_ABI scePthreadMutexDestroy(ScePthreadMutex* mutex) {
int result = pthread_mutex_destroy(&(*mutex)->pth_mutex); int result = pthread_mutex_destroy(&(*mutex)->pth_mutex);
LOG_INFO(Kernel_Pthread, "name={}, result={}", (*mutex)->name, result); LOG_DEBUG(Kernel_Pthread, "name={}, result={}", (*mutex)->name, result);
delete *mutex; delete *mutex;
*mutex = nullptr; *mutex = nullptr;
@ -725,7 +725,7 @@ int PS4_SYSV_ABI scePthreadCondDestroy(ScePthreadCond* cond) {
} }
int result = pthread_cond_destroy(&(*cond)->cond); int result = pthread_cond_destroy(&(*cond)->cond);
LOG_INFO(Kernel_Pthread, "scePthreadCondDestroy, result={}", result); LOG_DEBUG(Kernel_Pthread, "scePthreadCondDestroy, result={}", result);
delete *cond; delete *cond;
*cond = nullptr; *cond = nullptr;
@ -811,8 +811,6 @@ int PS4_SYSV_ABI posix_pthread_cond_timedwait(ScePthreadCond* cond, ScePthreadMu
} }
int PS4_SYSV_ABI posix_pthread_cond_broadcast(ScePthreadCond* cond) { int PS4_SYSV_ABI posix_pthread_cond_broadcast(ScePthreadCond* cond) {
LOG_INFO(Kernel_Pthread,
"posix posix_pthread_cond_broadcast redirect to scePthreadCondBroadcast");
int result = scePthreadCondBroadcast(cond); int result = scePthreadCondBroadcast(cond);
if (result != 0) { if (result != 0) {
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
@ -824,7 +822,6 @@ int PS4_SYSV_ABI posix_pthread_cond_broadcast(ScePthreadCond* cond) {
} }
int PS4_SYSV_ABI posix_pthread_mutexattr_init(ScePthreadMutexattr* attr) { int PS4_SYSV_ABI posix_pthread_mutexattr_init(ScePthreadMutexattr* attr) {
// LOG_INFO(Kernel_Pthread, "posix pthread_mutexattr_init redirect to scePthreadMutexattrInit");
int result = scePthreadMutexattrInit(attr); int result = scePthreadMutexattrInit(attr);
if (result < 0) { if (result < 0) {
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
@ -836,7 +833,6 @@ int PS4_SYSV_ABI posix_pthread_mutexattr_init(ScePthreadMutexattr* attr) {
} }
int PS4_SYSV_ABI posix_pthread_mutexattr_settype(ScePthreadMutexattr* attr, int type) { int PS4_SYSV_ABI posix_pthread_mutexattr_settype(ScePthreadMutexattr* attr, int type) {
// LOG_INFO(Kernel_Pthread, "posix pthread_mutex_init redirect to scePthreadMutexInit");
int result = scePthreadMutexattrSettype(attr, type); int result = scePthreadMutexattrSettype(attr, type);
if (result < 0) { if (result < 0) {
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
@ -861,7 +857,6 @@ int PS4_SYSV_ABI posix_pthread_once(pthread_once_t* once_control, void (*init_ro
int PS4_SYSV_ABI posix_pthread_mutexattr_setprotocol(ScePthreadMutexattr* attr, int protocol) { int PS4_SYSV_ABI posix_pthread_mutexattr_setprotocol(ScePthreadMutexattr* attr, int protocol) {
int result = scePthreadMutexattrSetprotocol(attr, protocol); int result = scePthreadMutexattrSetprotocol(attr, protocol);
LOG_INFO(Kernel_Pthread, "redirect to scePthreadMutexattrSetprotocol: result = {}", result);
if (result < 0) { if (result < 0) {
UNREACHABLE(); UNREACHABLE();
} }
@ -1295,8 +1290,6 @@ int PS4_SYSV_ABI posix_pthread_attr_setdetachstate(ScePthreadAttr* attr, int det
int PS4_SYSV_ABI posix_pthread_create_name_np(ScePthread* thread, const ScePthreadAttr* attr, int PS4_SYSV_ABI posix_pthread_create_name_np(ScePthread* thread, const ScePthreadAttr* attr,
PthreadEntryFunc start_routine, void* arg, PthreadEntryFunc start_routine, void* arg,
const char* name) { const char* name) {
LOG_INFO(Kernel_Pthread, "posix pthread_create redirect to scePthreadCreate: name = {}", name);
int result = scePthreadCreate(thread, attr, start_routine, arg, name); int result = scePthreadCreate(thread, attr, start_routine, arg, name);
if (result != 0) { if (result != 0) {
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
@ -1343,17 +1336,11 @@ int PS4_SYSV_ABI posix_pthread_cond_init(ScePthreadCond* cond, const ScePthreadC
int PS4_SYSV_ABI posix_pthread_cond_signal(ScePthreadCond* cond) { int PS4_SYSV_ABI posix_pthread_cond_signal(ScePthreadCond* cond) {
int result = scePthreadCondSignal(cond); int result = scePthreadCondSignal(cond);
LOG_INFO(Kernel_Pthread,
"posix posix_pthread_cond_signal redirect to scePthreadCondSignal, result = {}",
result);
return result; return result;
} }
int PS4_SYSV_ABI posix_pthread_cond_destroy(ScePthreadCond* cond) { int PS4_SYSV_ABI posix_pthread_cond_destroy(ScePthreadCond* cond) {
int result = scePthreadCondDestroy(cond); int result = scePthreadCondDestroy(cond);
LOG_INFO(Kernel_Pthread,
"posix posix_pthread_cond_destroy redirect to scePthreadCondDestroy, result = {}",
result);
return result; return result;
} }

View File

@ -470,7 +470,7 @@ int PS4_SYSV_ABI scePadSetUserColor() {
} }
int PS4_SYSV_ABI scePadSetVibration(s32 handle, const OrbisPadVibrationParam* pParam) { int PS4_SYSV_ABI scePadSetVibration(s32 handle, const OrbisPadVibrationParam* pParam) {
LOG_ERROR(Lib_Pad, "(STUBBED) called"); LOG_DEBUG(Lib_Pad, "(STUBBED) called");
return ORBIS_OK; return ORBIS_OK;
} }

View File

@ -7,7 +7,7 @@
#include "core/libraries/error_codes.h" #include "core/libraries/error_codes.h"
#include "core/libraries/kernel/memory_management.h" #include "core/libraries/kernel/memory_management.h"
#include "core/memory.h" #include "core/memory.h"
#include "video_core/renderer_vulkan/vk_instance.h" #include "video_core/renderer_vulkan/vk_rasterizer.h"
namespace Core { namespace Core {
@ -172,7 +172,7 @@ int MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, size_t size, M
if (type == VMAType::Direct) { if (type == VMAType::Direct) {
new_vma.phys_base = phys_addr; new_vma.phys_base = phys_addr;
MapVulkanMemory(mapped_addr, size); rasterizer->MapMemory(mapped_addr, size);
} }
if (type == VMAType::Flexible) { if (type == VMAType::Flexible) {
flexible_usage += size; flexible_usage += size;
@ -222,7 +222,7 @@ void MemoryManager::UnmapMemory(VAddr virtual_addr, size_t size) {
const auto type = it->second.type; const auto type = it->second.type;
const bool has_backing = type == VMAType::Direct || type == VMAType::File; const bool has_backing = type == VMAType::Direct || type == VMAType::File;
if (type == VMAType::Direct) { if (type == VMAType::Direct) {
UnmapVulkanMemory(virtual_addr, size); rasterizer->UnmapMemory(virtual_addr, size);
} }
if (type == VMAType::Flexible) { if (type == VMAType::Flexible) {
flexible_usage -= size; flexible_usage -= size;
@ -263,7 +263,7 @@ int MemoryManager::QueryProtection(VAddr addr, void** start, void** end, u32* pr
} }
int MemoryManager::VirtualQuery(VAddr addr, int flags, int MemoryManager::VirtualQuery(VAddr addr, int flags,
Libraries::Kernel::OrbisVirtualQueryInfo* info) { ::Libraries::Kernel::OrbisVirtualQueryInfo* info) {
std::scoped_lock lk{mutex}; std::scoped_lock lk{mutex};
auto it = FindVMA(addr); auto it = FindVMA(addr);
@ -293,7 +293,7 @@ int MemoryManager::VirtualQuery(VAddr addr, int flags,
} }
int MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next, int MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next,
Libraries::Kernel::OrbisQueryInfo* out_info) { ::Libraries::Kernel::OrbisQueryInfo* out_info) {
std::scoped_lock lk{mutex}; std::scoped_lock lk{mutex};
auto dmem_area = FindDmemArea(addr); auto dmem_area = FindDmemArea(addr);
@ -333,13 +333,6 @@ int MemoryManager::DirectQueryAvailable(PAddr search_start, PAddr search_end, si
return ORBIS_OK; return ORBIS_OK;
} }
std::pair<vk::Buffer, size_t> MemoryManager::GetVulkanBuffer(VAddr addr) {
auto it = mapped_memories.upper_bound(addr);
it = std::prev(it);
ASSERT(it != mapped_memories.end() && it->first <= addr);
return std::make_pair(*it->second.buffer, addr - it->first);
}
void MemoryManager::NameVirtualRange(VAddr virtual_addr, size_t size, std::string_view name) { void MemoryManager::NameVirtualRange(VAddr virtual_addr, size_t size, std::string_view name) {
auto it = FindVMA(virtual_addr); auto it = FindVMA(virtual_addr);
@ -455,85 +448,6 @@ MemoryManager::DMemHandle MemoryManager::Split(DMemHandle dmem_handle, size_t of
return dmem_map.emplace_hint(std::next(dmem_handle), new_area.base, new_area); return dmem_map.emplace_hint(std::next(dmem_handle), new_area.base, new_area);
}; };
void MemoryManager::MapVulkanMemory(VAddr addr, size_t size) {
return;
const vk::Device device = instance->GetDevice();
const auto memory_props = instance->GetPhysicalDevice().getMemoryProperties();
void* host_pointer = reinterpret_cast<void*>(addr);
const auto host_mem_props = device.getMemoryHostPointerPropertiesEXT(
vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, host_pointer);
ASSERT(host_mem_props.memoryTypeBits != 0);
int mapped_memory_type = -1;
auto find_mem_type_with_flag = [&](const vk::MemoryPropertyFlags flags) {
u32 host_mem_types = host_mem_props.memoryTypeBits;
while (host_mem_types != 0) {
// Try to find a cached memory type
mapped_memory_type = std::countr_zero(host_mem_types);
host_mem_types -= (1 << mapped_memory_type);
if ((memory_props.memoryTypes[mapped_memory_type].propertyFlags & flags) == flags) {
return;
}
}
mapped_memory_type = -1;
};
// First try to find a memory that is both coherent and cached
find_mem_type_with_flag(vk::MemoryPropertyFlagBits::eHostCoherent |
vk::MemoryPropertyFlagBits::eHostCached);
if (mapped_memory_type == -1)
// Then only coherent (lower performance)
find_mem_type_with_flag(vk::MemoryPropertyFlagBits::eHostCoherent);
if (mapped_memory_type == -1) {
LOG_CRITICAL(Render_Vulkan, "No coherent memory available for memory mapping");
mapped_memory_type = std::countr_zero(host_mem_props.memoryTypeBits);
}
const vk::StructureChain alloc_info = {
vk::MemoryAllocateInfo{
.allocationSize = size,
.memoryTypeIndex = static_cast<uint32_t>(mapped_memory_type),
},
vk::ImportMemoryHostPointerInfoEXT{
.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT,
.pHostPointer = host_pointer,
},
};
const auto [it, new_memory] = mapped_memories.try_emplace(addr);
ASSERT_MSG(new_memory, "Attempting to remap already mapped vulkan memory");
auto& memory = it->second;
memory.backing = device.allocateMemoryUnique(alloc_info.get());
constexpr vk::BufferUsageFlags MapFlags =
vk::BufferUsageFlagBits::eIndexBuffer | vk::BufferUsageFlagBits::eVertexBuffer |
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst |
vk::BufferUsageFlagBits::eUniformBuffer | vk::BufferUsageFlagBits::eStorageBuffer;
const vk::StructureChain buffer_info = {
vk::BufferCreateInfo{
.size = size,
.usage = MapFlags,
.sharingMode = vk::SharingMode::eExclusive,
},
vk::ExternalMemoryBufferCreateInfoKHR{
.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT,
}};
memory.buffer = device.createBufferUnique(buffer_info.get());
device.bindBufferMemory(*memory.buffer, *memory.backing, 0);
}
void MemoryManager::UnmapVulkanMemory(VAddr addr, size_t size) {
return;
const auto it = mapped_memories.find(addr);
ASSERT(it != mapped_memories.end() && it->second.buffer_size == size);
mapped_memories.erase(it);
}
int MemoryManager::GetDirectMemoryType(PAddr addr, int* directMemoryTypeOut, int MemoryManager::GetDirectMemoryType(PAddr addr, int* directMemoryTypeOut,
void** directMemoryStartOut, void** directMemoryEndOut) { void** directMemoryStartOut, void** directMemoryEndOut) {
std::scoped_lock lk{mutex}; std::scoped_lock lk{mutex};

View File

@ -3,20 +3,17 @@
#pragma once #pragma once
#include <functional> #include <map>
#include <mutex> #include <mutex>
#include <string_view> #include <string_view>
#include <vector>
#include <boost/icl/split_interval_map.hpp>
#include "common/enum.h" #include "common/enum.h"
#include "common/singleton.h" #include "common/singleton.h"
#include "common/types.h" #include "common/types.h"
#include "core/address_space.h" #include "core/address_space.h"
#include "core/libraries/kernel/memory_management.h" #include "core/libraries/kernel/memory_management.h"
#include "video_core/renderer_vulkan/vk_common.h"
namespace Vulkan { namespace Vulkan {
class Instance; class Rasterizer;
} }
namespace Libraries::Kernel { namespace Libraries::Kernel {
@ -128,8 +125,8 @@ public:
explicit MemoryManager(); explicit MemoryManager();
~MemoryManager(); ~MemoryManager();
void SetInstance(const Vulkan::Instance* instance_) { void SetRasterizer(Vulkan::Rasterizer* rasterizer_) {
instance = instance_; rasterizer = rasterizer_;
} }
void SetTotalFlexibleSize(u64 size) { void SetTotalFlexibleSize(u64 size) {
@ -140,9 +137,7 @@ public:
return total_flexible_size - flexible_usage; return total_flexible_size - flexible_usage;
} }
/// Returns the offset of the mapped virtual system managed memory base from where it usually VAddr SystemReservedVirtualBase() noexcept {
/// would be mapped.
[[nodiscard]] VAddr SystemReservedVirtualBase() noexcept {
return impl.SystemReservedVirtualBase(); return impl.SystemReservedVirtualBase();
} }
@ -172,8 +167,6 @@ public:
int DirectQueryAvailable(PAddr search_start, PAddr search_end, size_t alignment, int DirectQueryAvailable(PAddr search_start, PAddr search_end, size_t alignment,
PAddr* phys_addr_out, size_t* size_out); PAddr* phys_addr_out, size_t* size_out);
std::pair<vk::Buffer, size_t> GetVulkanBuffer(VAddr addr);
int GetDirectMemoryType(PAddr addr, int* directMemoryTypeOut, void** directMemoryStartOut, int GetDirectMemoryType(PAddr addr, int* directMemoryTypeOut, void** directMemoryStartOut,
void** directMemoryEndOut); void** directMemoryEndOut);
@ -218,10 +211,6 @@ private:
DMemHandle Split(DMemHandle dmem_handle, size_t offset_in_area); DMemHandle Split(DMemHandle dmem_handle, size_t offset_in_area);
void MapVulkanMemory(VAddr addr, size_t size);
void UnmapVulkanMemory(VAddr addr, size_t size);
private: private:
AddressSpace impl; AddressSpace impl;
DMemMap dmem_map; DMemMap dmem_map;
@ -229,14 +218,7 @@ private:
std::recursive_mutex mutex; std::recursive_mutex mutex;
size_t total_flexible_size = 448_MB; size_t total_flexible_size = 448_MB;
size_t flexible_usage{}; size_t flexible_usage{};
Vulkan::Rasterizer* rasterizer{};
struct MappedMemory {
vk::UniqueBuffer buffer;
vk::UniqueDeviceMemory backing;
size_t buffer_size;
};
std::map<VAddr, MappedMemory> mapped_memories;
const Vulkan::Instance* instance{};
}; };
using Memory = Common::Singleton<MemoryManager>; using Memory = Common::Singleton<MemoryManager>;

View File

@ -88,6 +88,7 @@ void Module::LoadModuleToMemory(u32& max_tls_index) {
aligned_base_size + TrampolineSize, MemoryProt::CpuReadWrite, aligned_base_size + TrampolineSize, MemoryProt::CpuReadWrite,
MemoryMapFlags::Fixed, VMAType::Code, name, true); MemoryMapFlags::Fixed, VMAType::Code, name, true);
LoadOffset += CODE_BASE_INCR * (1 + aligned_base_size / CODE_BASE_INCR); LoadOffset += CODE_BASE_INCR * (1 + aligned_base_size / CODE_BASE_INCR);
LOG_INFO(Core_Linker, "Loading module {} to {}", name, fmt::ptr(*out_addr));
// Initialize trampoline generator. // Initialize trampoline generator.
void* trampoline_addr = std::bit_cast<void*>(base_virtual_addr + aligned_base_size); void* trampoline_addr = std::bit_cast<void*>(base_virtual_addr + aligned_base_size);

View File

@ -21,6 +21,7 @@ Id VsOutputAttrPointer(EmitContext& ctx, VsOutput output) {
case VsOutput::ClipDist7: { case VsOutput::ClipDist7: {
const u32 index = u32(output) - u32(VsOutput::ClipDist0); const u32 index = u32(output) - u32(VsOutput::ClipDist0);
const Id clip_num{ctx.ConstU32(index)}; const Id clip_num{ctx.ConstU32(index)};
ASSERT_MSG(Sirit::ValidId(ctx.clip_distances), "Clip distance used but not defined");
return ctx.OpAccessChain(ctx.output_f32, ctx.clip_distances, clip_num); return ctx.OpAccessChain(ctx.output_f32, ctx.clip_distances, clip_num);
} }
case VsOutput::CullDist0: case VsOutput::CullDist0:
@ -33,6 +34,7 @@ Id VsOutputAttrPointer(EmitContext& ctx, VsOutput output) {
case VsOutput::CullDist7: { case VsOutput::CullDist7: {
const u32 index = u32(output) - u32(VsOutput::CullDist0); const u32 index = u32(output) - u32(VsOutput::CullDist0);
const Id cull_num{ctx.ConstU32(index)}; const Id cull_num{ctx.ConstU32(index)};
ASSERT_MSG(Sirit::ValidId(ctx.cull_distances), "Cull distance used but not defined");
return ctx.OpAccessChain(ctx.output_f32, ctx.cull_distances, cull_num); return ctx.OpAccessChain(ctx.output_f32, ctx.cull_distances, cull_num);
} }
default: default:
@ -125,7 +127,12 @@ Id EmitReadConst(EmitContext& ctx) {
} }
Id EmitReadConstBuffer(EmitContext& ctx, u32 handle, Id index) { Id EmitReadConstBuffer(EmitContext& ctx, u32 handle, Id index) {
const auto& buffer = ctx.buffers[handle]; auto& buffer = ctx.buffers[handle];
if (!Sirit::ValidId(buffer.offset)) {
buffer.offset = ctx.GetBufferOffset(handle);
}
const Id offset_dwords{ctx.OpShiftRightLogical(ctx.U32[1], buffer.offset, ctx.ConstU32(2U))};
index = ctx.OpIAdd(ctx.U32[1], index, offset_dwords);
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)}; const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};
return ctx.OpLoad(buffer.data_types->Get(1), ptr); return ctx.OpLoad(buffer.data_types->Get(1), ptr);
} }
@ -137,7 +144,7 @@ Id EmitReadConstBufferU32(EmitContext& ctx, u32 handle, Id index) {
Id EmitReadStepRate(EmitContext& ctx, int rate_idx) { Id EmitReadStepRate(EmitContext& ctx, int rate_idx) {
return ctx.OpLoad( return ctx.OpLoad(
ctx.U32[1], ctx.OpAccessChain(ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U32[1]), ctx.U32[1], ctx.OpAccessChain(ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U32[1]),
ctx.instance_step_rates, ctx.push_data_block,
rate_idx == 0 ? ctx.u32_zero_value : ctx.u32_one_value)); rate_idx == 0 ? ctx.u32_zero_value : ctx.u32_one_value));
} }
@ -221,7 +228,11 @@ Id EmitLoadBufferU32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address) {
template <u32 N> template <u32 N>
static Id EmitLoadBufferF32xN(EmitContext& ctx, u32 handle, Id address) { static Id EmitLoadBufferF32xN(EmitContext& ctx, u32 handle, Id address) {
const auto& buffer = ctx.buffers[handle]; auto& buffer = ctx.buffers[handle];
if (!Sirit::ValidId(buffer.offset)) {
buffer.offset = ctx.GetBufferOffset(handle);
}
address = ctx.OpIAdd(ctx.U32[1], address, buffer.offset);
const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u)); const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u));
if constexpr (N == 1) { if constexpr (N == 1) {
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)}; const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};
@ -314,7 +325,7 @@ static Id ComponentOffset(EmitContext& ctx, Id address, u32 stride, u32 bit_offs
} }
static Id GetBufferFormatValue(EmitContext& ctx, u32 handle, Id address, u32 comp) { static Id GetBufferFormatValue(EmitContext& ctx, u32 handle, Id address, u32 comp) {
const auto& buffer = ctx.buffers[handle]; auto& buffer = ctx.buffers[handle];
const auto format = buffer.buffer.GetDataFmt(); const auto format = buffer.buffer.GetDataFmt();
switch (format) { switch (format) {
case AmdGpu::DataFormat::FormatInvalid: case AmdGpu::DataFormat::FormatInvalid:
@ -399,6 +410,11 @@ static Id GetBufferFormatValue(EmitContext& ctx, u32 handle, Id address, u32 com
template <u32 N> template <u32 N>
static Id EmitLoadBufferFormatF32xN(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address) { static Id EmitLoadBufferFormatF32xN(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address) {
auto& buffer = ctx.buffers[handle];
if (!Sirit::ValidId(buffer.offset)) {
buffer.offset = ctx.GetBufferOffset(handle);
}
address = ctx.OpIAdd(ctx.U32[1], address, buffer.offset);
if constexpr (N == 1) { if constexpr (N == 1) {
return GetBufferFormatValue(ctx, handle, address, 0); return GetBufferFormatValue(ctx, handle, address, 0);
} else { } else {
@ -428,7 +444,11 @@ Id EmitLoadBufferFormatF32x4(EmitContext& ctx, IR::Inst* inst, u32 handle, Id ad
template <u32 N> template <u32 N>
static void EmitStoreBufferF32xN(EmitContext& ctx, u32 handle, Id address, Id value) { static void EmitStoreBufferF32xN(EmitContext& ctx, u32 handle, Id address, Id value) {
const auto& buffer = ctx.buffers[handle]; auto& buffer = ctx.buffers[handle];
if (!Sirit::ValidId(buffer.offset)) {
buffer.offset = ctx.GetBufferOffset(handle);
}
address = ctx.OpIAdd(ctx.U32[1], address, buffer.offset);
const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u)); const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u));
if constexpr (N == 1) { if constexpr (N == 1) {
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)}; const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};

View File

@ -46,9 +46,9 @@ EmitContext::EmitContext(const Profile& profile_, IR::Program& program, u32& bin
stage{program.info.stage}, binding{binding_} { stage{program.info.stage}, binding{binding_} {
AddCapability(spv::Capability::Shader); AddCapability(spv::Capability::Shader);
DefineArithmeticTypes(); DefineArithmeticTypes();
DefineInterfaces(program); DefineInterfaces();
DefineBuffers(info); DefineBuffers();
DefineImagesAndSamplers(info); DefineImagesAndSamplers();
DefineSharedMemory(); DefineSharedMemory();
} }
@ -117,9 +117,10 @@ void EmitContext::DefineArithmeticTypes() {
full_result_u32x2 = Name(TypeStruct(U32[1], U32[1]), "full_result_u32x2"); full_result_u32x2 = Name(TypeStruct(U32[1], U32[1]), "full_result_u32x2");
} }
void EmitContext::DefineInterfaces(const IR::Program& program) { void EmitContext::DefineInterfaces() {
DefineInputs(program.info); DefinePushDataBlock();
DefineOutputs(program.info); DefineInputs();
DefineOutputs();
} }
Id GetAttributeType(EmitContext& ctx, AmdGpu::NumberFormat fmt) { Id GetAttributeType(EmitContext& ctx, AmdGpu::NumberFormat fmt) {
@ -164,6 +165,16 @@ EmitContext::SpirvAttribute EmitContext::GetAttributeInfo(AmdGpu::NumberFormat f
throw InvalidArgument("Invalid attribute type {}", fmt); throw InvalidArgument("Invalid attribute type {}", fmt);
} }
Id EmitContext::GetBufferOffset(u32 binding) {
const u32 half = Shader::PushData::BufOffsetIndex + (binding >> 4);
const u32 comp = (binding & 0xf) >> 2;
const u32 offset = (binding & 0x3) << 3;
const Id ptr{OpAccessChain(TypePointer(spv::StorageClass::PushConstant, U32[1]),
push_data_block, ConstU32(half), ConstU32(comp))};
const Id value{OpLoad(U32[1], ptr)};
return OpBitFieldUExtract(U32[1], value, ConstU32(offset), ConstU32(8U));
}
Id MakeDefaultValue(EmitContext& ctx, u32 default_value) { Id MakeDefaultValue(EmitContext& ctx, u32 default_value) {
switch (default_value) { switch (default_value) {
case 0: case 0:
@ -179,24 +190,13 @@ Id MakeDefaultValue(EmitContext& ctx, u32 default_value) {
} }
} }
void EmitContext::DefineInputs(const Info& info) { void EmitContext::DefineInputs() {
switch (stage) { switch (stage) {
case Stage::Vertex: { case Stage::Vertex: {
vertex_index = DefineVariable(U32[1], spv::BuiltIn::VertexIndex, spv::StorageClass::Input); vertex_index = DefineVariable(U32[1], spv::BuiltIn::VertexIndex, spv::StorageClass::Input);
base_vertex = DefineVariable(U32[1], spv::BuiltIn::BaseVertex, spv::StorageClass::Input); base_vertex = DefineVariable(U32[1], spv::BuiltIn::BaseVertex, spv::StorageClass::Input);
instance_id = DefineVariable(U32[1], spv::BuiltIn::InstanceIndex, spv::StorageClass::Input); instance_id = DefineVariable(U32[1], spv::BuiltIn::InstanceIndex, spv::StorageClass::Input);
// Create push constants block for instance steps rates
const Id struct_type{Name(TypeStruct(U32[1], U32[1]), "instance_step_rates")};
Decorate(struct_type, spv::Decoration::Block);
MemberName(struct_type, 0, "sr0");
MemberName(struct_type, 1, "sr1");
MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U);
MemberDecorate(struct_type, 1, spv::Decoration::Offset, 4U);
instance_step_rates = DefineVar(struct_type, spv::StorageClass::PushConstant);
Name(instance_step_rates, "step_rates");
interfaces.push_back(instance_step_rates);
for (const auto& input : info.vs_inputs) { for (const auto& input : info.vs_inputs) {
const Id type{GetAttributeType(*this, input.fmt)}; const Id type{GetAttributeType(*this, input.fmt)};
if (input.instance_step_rate == Info::VsInput::InstanceIdType::OverStepRate0 || if (input.instance_step_rate == Info::VsInput::InstanceIdType::OverStepRate0 ||
@ -260,19 +260,20 @@ void EmitContext::DefineInputs(const Info& info) {
} }
} }
void EmitContext::DefineOutputs(const Info& info) { void EmitContext::DefineOutputs() {
switch (stage) { switch (stage) {
case Stage::Vertex: { case Stage::Vertex: {
output_position = DefineVariable(F32[4], spv::BuiltIn::Position, spv::StorageClass::Output); output_position = DefineVariable(F32[4], spv::BuiltIn::Position, spv::StorageClass::Output);
const std::array<Id, 8> zero{f32_zero_value, f32_zero_value, f32_zero_value, const bool has_extra_pos_stores = info.stores.Get(IR::Attribute::Position1) ||
f32_zero_value, f32_zero_value, f32_zero_value, info.stores.Get(IR::Attribute::Position2) ||
f32_zero_value, f32_zero_value}; info.stores.Get(IR::Attribute::Position3);
if (has_extra_pos_stores) {
const Id type{TypeArray(F32[1], ConstU32(8U))}; const Id type{TypeArray(F32[1], ConstU32(8U))};
const Id initializer{ConstantComposite(type, zero)}; clip_distances =
clip_distances = DefineVariable(type, spv::BuiltIn::ClipDistance, spv::StorageClass::Output, DefineVariable(type, spv::BuiltIn::ClipDistance, spv::StorageClass::Output);
initializer); cull_distances =
cull_distances = DefineVariable(type, spv::BuiltIn::CullDistance, spv::StorageClass::Output, DefineVariable(type, spv::BuiltIn::CullDistance, spv::StorageClass::Output);
initializer); }
for (u32 i = 0; i < IR::NumParams; i++) { for (u32 i = 0; i < IR::NumParams; i++) {
const IR::Attribute param{IR::Attribute::Param0 + i}; const IR::Attribute param{IR::Attribute::Param0 + i};
if (!info.stores.GetAny(param)) { if (!info.stores.GetAny(param)) {
@ -304,7 +305,24 @@ void EmitContext::DefineOutputs(const Info& info) {
} }
} }
void EmitContext::DefineBuffers(const Info& info) { void EmitContext::DefinePushDataBlock() {
// Create push constants block for instance steps rates
const Id struct_type{Name(TypeStruct(U32[1], U32[1], U32[4], U32[4]), "AuxData")};
Decorate(struct_type, spv::Decoration::Block);
MemberName(struct_type, 0, "sr0");
MemberName(struct_type, 1, "sr1");
MemberName(struct_type, 2, "buf_offsets0");
MemberName(struct_type, 3, "buf_offsets1");
MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U);
MemberDecorate(struct_type, 1, spv::Decoration::Offset, 4U);
MemberDecorate(struct_type, 2, spv::Decoration::Offset, 8U);
MemberDecorate(struct_type, 3, spv::Decoration::Offset, 24U);
push_data_block = DefineVar(struct_type, spv::StorageClass::PushConstant);
Name(push_data_block, "push_data");
interfaces.push_back(push_data_block);
}
void EmitContext::DefineBuffers() {
boost::container::small_vector<Id, 8> type_ids; boost::container::small_vector<Id, 8> type_ids;
for (u32 i = 0; const auto& buffer : info.buffers) { for (u32 i = 0; const auto& buffer : info.buffers) {
const auto* data_types = True(buffer.used_types & IR::Type::F32) ? &F32 : &U32; const auto* data_types = True(buffer.used_types & IR::Type::F32) ? &F32 : &U32;
@ -322,8 +340,8 @@ void EmitContext::DefineBuffers(const Info& info) {
Decorate(struct_type, spv::Decoration::Block); Decorate(struct_type, spv::Decoration::Block);
MemberName(struct_type, 0, "data"); MemberName(struct_type, 0, "data");
MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U); MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U);
}
type_ids.push_back(record_array_type); type_ids.push_back(record_array_type);
}
const auto storage_class = const auto storage_class =
buffer.is_storage ? spv::StorageClass::StorageBuffer : spv::StorageClass::Uniform; buffer.is_storage ? spv::StorageClass::StorageBuffer : spv::StorageClass::Uniform;
@ -430,7 +448,7 @@ Id ImageType(EmitContext& ctx, const ImageResource& desc, Id sampled_type) {
throw InvalidArgument("Invalid texture type {}", desc.type); throw InvalidArgument("Invalid texture type {}", desc.type);
} }
void EmitContext::DefineImagesAndSamplers(const Info& info) { void EmitContext::DefineImagesAndSamplers() {
for (const auto& image_desc : info.images) { for (const auto& image_desc : info.images) {
const VectorIds* data_types = [&] { const VectorIds* data_types = [&] {
switch (image_desc.nfmt) { switch (image_desc.nfmt) {

View File

@ -40,6 +40,7 @@ public:
~EmitContext(); ~EmitContext();
Id Def(const IR::Value& value); Id Def(const IR::Value& value);
Id GetBufferOffset(u32 binding);
[[nodiscard]] Id DefineInput(Id type, u32 location) { [[nodiscard]] Id DefineInput(Id type, u32 location) {
const Id input_id{DefineVar(type, spv::StorageClass::Input)}; const Id input_id{DefineVar(type, spv::StorageClass::Input)};
@ -168,7 +169,7 @@ public:
Id output_position{}; Id output_position{};
Id vertex_index{}; Id vertex_index{};
Id instance_id{}; Id instance_id{};
Id instance_step_rates{}; Id push_data_block{};
Id base_vertex{}; Id base_vertex{};
Id frag_coord{}; Id frag_coord{};
Id front_facing{}; Id front_facing{};
@ -201,14 +202,15 @@ public:
struct BufferDefinition { struct BufferDefinition {
Id id; Id id;
Id offset;
const VectorIds* data_types; const VectorIds* data_types;
Id pointer_type; Id pointer_type;
AmdGpu::Buffer buffer; AmdGpu::Buffer buffer;
}; };
u32& binding; u32& binding;
boost::container::small_vector<BufferDefinition, 4> buffers; boost::container::small_vector<BufferDefinition, 16> buffers;
boost::container::small_vector<TextureDefinition, 4> images; boost::container::small_vector<TextureDefinition, 8> images;
boost::container::small_vector<Id, 4> samplers; boost::container::small_vector<Id, 4> samplers;
Id sampler_type{}; Id sampler_type{};
@ -227,11 +229,12 @@ public:
private: private:
void DefineArithmeticTypes(); void DefineArithmeticTypes();
void DefineInterfaces(const IR::Program& program); void DefineInterfaces();
void DefineInputs(const Info& info); void DefineInputs();
void DefineOutputs(const Info& info); void DefineOutputs();
void DefineBuffers(const Info& info); void DefinePushDataBlock();
void DefineImagesAndSamplers(const Info& info); void DefineBuffers();
void DefineImagesAndSamplers();
void DefineSharedMemory(); void DefineSharedMemory();
SpirvAttribute GetAttributeInfo(AmdGpu::NumberFormat fmt, Id id); SpirvAttribute GetAttributeInfo(AmdGpu::NumberFormat fmt, Id id);

View File

@ -40,7 +40,7 @@ static IR::Condition MakeCondition(Opcode opcode) {
} }
} }
CFG::CFG(ObjectPool<Block>& block_pool_, std::span<const GcnInst> inst_list_) CFG::CFG(Common::ObjectPool<Block>& block_pool_, std::span<const GcnInst> inst_list_)
: block_pool{block_pool_}, inst_list{inst_list_} { : block_pool{block_pool_}, inst_list{inst_list_} {
index_to_pc.resize(inst_list.size() + 1); index_to_pc.resize(inst_list.size() + 1);
EmitLabels(); EmitLabels();

View File

@ -8,10 +8,10 @@
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
#include <boost/intrusive/set.hpp> #include <boost/intrusive/set.hpp>
#include "common/object_pool.h"
#include "common/types.h" #include "common/types.h"
#include "shader_recompiler/frontend/instruction.h" #include "shader_recompiler/frontend/instruction.h"
#include "shader_recompiler/ir/condition.h" #include "shader_recompiler/ir/condition.h"
#include "shader_recompiler/object_pool.h"
namespace Shader::Gcn { namespace Shader::Gcn {
@ -49,7 +49,7 @@ class CFG {
using Label = u32; using Label = u32;
public: public:
explicit CFG(ObjectPool<Block>& block_pool, std::span<const GcnInst> inst_list); explicit CFG(Common::ObjectPool<Block>& block_pool, std::span<const GcnInst> inst_list);
[[nodiscard]] std::string Dot() const; [[nodiscard]] std::string Dot() const;
@ -59,7 +59,7 @@ private:
void LinkBlocks(); void LinkBlocks();
public: public:
ObjectPool<Block>& block_pool; Common::ObjectPool<Block>& block_pool;
std::span<const GcnInst> inst_list; std::span<const GcnInst> inst_list;
std::vector<u32> index_to_pc; std::vector<u32> index_to_pc;
boost::container::small_vector<Label, 16> labels; boost::container::small_vector<Label, 16> labels;

View File

@ -1,10 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
namespace Shader::Gcn {
void Translate();
} // namespace Shader::Gcn

View File

@ -287,7 +287,7 @@ bool NeedsLift(Node goto_stmt, Node label_stmt) noexcept {
*/ */
class GotoPass { class GotoPass {
public: public:
explicit GotoPass(CFG& cfg, ObjectPool<Statement>& stmt_pool) : pool{stmt_pool} { explicit GotoPass(CFG& cfg, Common::ObjectPool<Statement>& stmt_pool) : pool{stmt_pool} {
std::vector gotos{BuildTree(cfg)}; std::vector gotos{BuildTree(cfg)};
const auto end{gotos.rend()}; const auto end{gotos.rend()};
for (auto goto_stmt = gotos.rbegin(); goto_stmt != end; ++goto_stmt) { for (auto goto_stmt = gotos.rbegin(); goto_stmt != end; ++goto_stmt) {
@ -563,7 +563,7 @@ private:
return parent_tree.insert(std::next(loop), *new_goto); return parent_tree.insert(std::next(loop), *new_goto);
} }
ObjectPool<Statement>& pool; Common::ObjectPool<Statement>& pool;
Statement root_stmt{FunctionTag{}}; Statement root_stmt{FunctionTag{}};
}; };
@ -597,8 +597,9 @@ private:
class TranslatePass { class TranslatePass {
public: public:
TranslatePass(ObjectPool<IR::Inst>& inst_pool_, ObjectPool<IR::Block>& block_pool_, TranslatePass(Common::ObjectPool<IR::Inst>& inst_pool_,
ObjectPool<Statement>& stmt_pool_, Statement& root_stmt, Common::ObjectPool<IR::Block>& block_pool_,
Common::ObjectPool<Statement>& stmt_pool_, Statement& root_stmt,
IR::AbstractSyntaxList& syntax_list_, std::span<const GcnInst> inst_list_, IR::AbstractSyntaxList& syntax_list_, std::span<const GcnInst> inst_list_,
Info& info_, const Profile& profile_) Info& info_, const Profile& profile_)
: stmt_pool{stmt_pool_}, inst_pool{inst_pool_}, block_pool{block_pool_}, : stmt_pool{stmt_pool_}, inst_pool{inst_pool_}, block_pool{block_pool_},
@ -808,9 +809,9 @@ private:
return block_pool.Create(inst_pool); return block_pool.Create(inst_pool);
} }
ObjectPool<Statement>& stmt_pool; Common::ObjectPool<Statement>& stmt_pool;
ObjectPool<IR::Inst>& inst_pool; Common::ObjectPool<IR::Inst>& inst_pool;
ObjectPool<IR::Block>& block_pool; Common::ObjectPool<IR::Block>& block_pool;
IR::AbstractSyntaxList& syntax_list; IR::AbstractSyntaxList& syntax_list;
const Block dummy_flow_block{.is_dummy = true}; const Block dummy_flow_block{.is_dummy = true};
std::span<const GcnInst> inst_list; std::span<const GcnInst> inst_list;
@ -819,9 +820,10 @@ private:
}; };
} // Anonymous namespace } // Anonymous namespace
IR::AbstractSyntaxList BuildASL(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Block>& block_pool, IR::AbstractSyntaxList BuildASL(Common::ObjectPool<IR::Inst>& inst_pool,
CFG& cfg, Info& info, const Profile& profile) { Common::ObjectPool<IR::Block>& block_pool, CFG& cfg, Info& info,
ObjectPool<Statement> stmt_pool{64}; const Profile& profile) {
Common::ObjectPool<Statement> stmt_pool{64};
GotoPass goto_pass{cfg, stmt_pool}; GotoPass goto_pass{cfg, stmt_pool};
Statement& root{goto_pass.RootStatement()}; Statement& root{goto_pass.RootStatement()};
IR::AbstractSyntaxList syntax_list; IR::AbstractSyntaxList syntax_list;

View File

@ -7,7 +7,6 @@
#include "shader_recompiler/ir/abstract_syntax_list.h" #include "shader_recompiler/ir/abstract_syntax_list.h"
#include "shader_recompiler/ir/basic_block.h" #include "shader_recompiler/ir/basic_block.h"
#include "shader_recompiler/ir/value.h" #include "shader_recompiler/ir/value.h"
#include "shader_recompiler/object_pool.h"
namespace Shader { namespace Shader {
struct Info; struct Info;
@ -16,8 +15,8 @@ struct Profile;
namespace Shader::Gcn { namespace Shader::Gcn {
[[nodiscard]] IR::AbstractSyntaxList BuildASL(ObjectPool<IR::Inst>& inst_pool, [[nodiscard]] IR::AbstractSyntaxList BuildASL(Common::ObjectPool<IR::Inst>& inst_pool,
ObjectPool<IR::Block>& block_pool, CFG& cfg, Common::ObjectPool<IR::Block>& block_pool, CFG& cfg,
Info& info, const Profile& profile); Info& info, const Profile& profile);
} // namespace Shader::Gcn } // namespace Shader::Gcn

View File

@ -447,6 +447,7 @@ void Translator::EmitFetch(const GcnInst& inst) {
.is_instance_data = true, .is_instance_data = true,
}); });
instance_buf_handle = s32(info.buffers.size() - 1); instance_buf_handle = s32(info.buffers.size() - 1);
info.uses_step_rates = true;
} }
const u32 num_components = AmdGpu::NumComponents(buffer.GetDataFmt()); const u32 num_components = AmdGpu::NumComponents(buffer.GetDataFmt());

View File

@ -338,6 +338,11 @@ void Translator::BUFFER_LOAD_FORMAT(u32 num_dwords, bool is_typed, bool is_forma
if (is_typed) { if (is_typed) {
info.dmft.Assign(static_cast<AmdGpu::DataFormat>(mtbuf.dfmt)); info.dmft.Assign(static_cast<AmdGpu::DataFormat>(mtbuf.dfmt));
info.nfmt.Assign(static_cast<AmdGpu::NumberFormat>(mtbuf.nfmt)); info.nfmt.Assign(static_cast<AmdGpu::NumberFormat>(mtbuf.nfmt));
ASSERT(info.nfmt == AmdGpu::NumberFormat::Float &&
(info.dmft == AmdGpu::DataFormat::Format32_32_32_32 ||
info.dmft == AmdGpu::DataFormat::Format32_32_32 ||
info.dmft == AmdGpu::DataFormat::Format32_32 ||
info.dmft == AmdGpu::DataFormat::Format32));
} }
const IR::Value handle = const IR::Value handle =

View File

@ -9,7 +9,7 @@
namespace Shader::IR { namespace Shader::IR {
Block::Block(ObjectPool<Inst>& inst_pool_) : inst_pool{&inst_pool_} {} Block::Block(Common::ObjectPool<Inst>& inst_pool_) : inst_pool{&inst_pool_} {}
Block::~Block() = default; Block::~Block() = default;

View File

@ -9,10 +9,10 @@
#include <vector> #include <vector>
#include <boost/intrusive/list.hpp> #include <boost/intrusive/list.hpp>
#include "common/object_pool.h"
#include "common/types.h" #include "common/types.h"
#include "shader_recompiler/ir/reg.h" #include "shader_recompiler/ir/reg.h"
#include "shader_recompiler/ir/value.h" #include "shader_recompiler/ir/value.h"
#include "shader_recompiler/object_pool.h"
namespace Shader::IR { namespace Shader::IR {
@ -25,7 +25,7 @@ public:
using reverse_iterator = InstructionList::reverse_iterator; using reverse_iterator = InstructionList::reverse_iterator;
using const_reverse_iterator = InstructionList::const_reverse_iterator; using const_reverse_iterator = InstructionList::const_reverse_iterator;
explicit Block(ObjectPool<Inst>& inst_pool_); explicit Block(Common::ObjectPool<Inst>& inst_pool_);
~Block(); ~Block();
Block(const Block&) = delete; Block(const Block&) = delete;
@ -153,7 +153,7 @@ public:
private: private:
/// Memory pool for instruction list /// Memory pool for instruction list
ObjectPool<Inst>* inst_pool; Common::ObjectPool<Inst>* inst_pool;
/// List of instructions in this block /// List of instructions in this block
InstructionList instructions; InstructionList instructions;

View File

@ -173,10 +173,9 @@ bool IsImageStorageInstruction(const IR::Inst& inst) {
class Descriptors { class Descriptors {
public: public:
explicit Descriptors(BufferResourceList& buffer_resources_, ImageResourceList& image_resources_, explicit Descriptors(Info& info_)
SamplerResourceList& sampler_resources_) : info{info_}, buffer_resources{info_.buffers}, image_resources{info_.images},
: buffer_resources{buffer_resources_}, image_resources{image_resources_}, sampler_resources{info_.samplers} {}
sampler_resources{sampler_resources_} {}
u32 Add(const BufferResource& desc) { u32 Add(const BufferResource& desc) {
const u32 index{Add(buffer_resources, desc, [&desc](const auto& existing) { const u32 index{Add(buffer_resources, desc, [&desc](const auto& existing) {
@ -188,6 +187,7 @@ public:
ASSERT(buffer.length == desc.length); ASSERT(buffer.length == desc.length);
buffer.is_storage |= desc.is_storage; buffer.is_storage |= desc.is_storage;
buffer.used_types |= desc.used_types; buffer.used_types |= desc.used_types;
buffer.is_written |= desc.is_written;
return index; return index;
} }
@ -201,9 +201,16 @@ public:
} }
u32 Add(const SamplerResource& desc) { u32 Add(const SamplerResource& desc) {
const u32 index{Add(sampler_resources, desc, [&desc](const auto& existing) { const u32 index{Add(sampler_resources, desc, [this, &desc](const auto& existing) {
return desc.sgpr_base == existing.sgpr_base && if (desc.sgpr_base == existing.sgpr_base &&
desc.dword_offset == existing.dword_offset; desc.dword_offset == existing.dword_offset) {
return true;
}
// Samplers with different bindings might still be the same.
const auto old_sharp =
info.ReadUd<AmdGpu::Sampler>(existing.sgpr_base, existing.dword_offset);
const auto new_sharp = info.ReadUd<AmdGpu::Sampler>(desc.sgpr_base, desc.dword_offset);
return old_sharp == new_sharp;
})}; })};
return index; return index;
} }
@ -219,6 +226,7 @@ private:
return static_cast<u32>(descriptors.size()) - 1; return static_cast<u32>(descriptors.size()) - 1;
} }
const Info& info;
BufferResourceList& buffer_resources; BufferResourceList& buffer_resources;
ImageResourceList& image_resources; ImageResourceList& image_resources;
SamplerResourceList& sampler_resources; SamplerResourceList& sampler_resources;
@ -328,16 +336,6 @@ static bool IsLoadBufferFormat(const IR::Inst& inst) {
} }
} }
static bool IsReadConstBuffer(const IR::Inst& inst) {
switch (inst.GetOpcode()) {
case IR::Opcode::ReadConstBuffer:
case IR::Opcode::ReadConstBufferU32:
return true;
default:
return false;
}
}
static u32 BufferLength(const AmdGpu::Buffer& buffer) { static u32 BufferLength(const AmdGpu::Buffer& buffer) {
const auto stride = buffer.GetStride(); const auto stride = buffer.GetStride();
if (stride < sizeof(f32)) { if (stride < sizeof(f32)) {
@ -401,30 +399,37 @@ void PatchBufferInstruction(IR::Block& block, IR::Inst& inst, Info& info,
IR::Inst* handle = inst.Arg(0).InstRecursive(); IR::Inst* handle = inst.Arg(0).InstRecursive();
IR::Inst* producer = handle->Arg(0).InstRecursive(); IR::Inst* producer = handle->Arg(0).InstRecursive();
const auto sharp = TrackSharp(producer); const auto sharp = TrackSharp(producer);
const bool is_store = IsBufferStore(inst);
buffer = info.ReadUd<AmdGpu::Buffer>(sharp.sgpr_base, sharp.dword_offset); buffer = info.ReadUd<AmdGpu::Buffer>(sharp.sgpr_base, sharp.dword_offset);
binding = descriptors.Add(BufferResource{ binding = descriptors.Add(BufferResource{
.sgpr_base = sharp.sgpr_base, .sgpr_base = sharp.sgpr_base,
.dword_offset = sharp.dword_offset, .dword_offset = sharp.dword_offset,
.length = BufferLength(buffer), .length = BufferLength(buffer),
.used_types = BufferDataType(inst, buffer.GetNumberFmt()), .used_types = BufferDataType(inst, buffer.GetNumberFmt()),
.is_storage = IsBufferStore(inst) || buffer.GetSize() > MaxUboSize, .is_storage = is_store || buffer.GetSize() > MaxUboSize,
.is_written = is_store,
}); });
} }
// Update buffer descriptor format.
const auto inst_info = inst.Flags<IR::BufferInstInfo>(); const auto inst_info = inst.Flags<IR::BufferInstInfo>();
IR::IREmitter ir{block, IR::Block::InstructionList::s_iterator_to(inst)}; auto& buffer_desc = info.buffers[binding];
// Replace handle with binding index in buffer resource list.
inst.SetArg(0, ir.Imm32(binding));
ASSERT(!buffer.swizzle_enable && !buffer.add_tid_enable);
if (inst_info.is_typed) { if (inst_info.is_typed) {
ASSERT(inst_info.nfmt == AmdGpu::NumberFormat::Float && buffer_desc.dfmt = inst_info.dmft;
(inst_info.dmft == AmdGpu::DataFormat::Format32_32_32_32 || buffer_desc.nfmt = inst_info.nfmt;
inst_info.dmft == AmdGpu::DataFormat::Format32_32_32 || } else {
inst_info.dmft == AmdGpu::DataFormat::Format32_32 || buffer_desc.dfmt = buffer.GetDataFmt();
inst_info.dmft == AmdGpu::DataFormat::Format32)); buffer_desc.nfmt = buffer.GetNumberFmt();
} }
if (IsReadConstBuffer(inst)) { // Replace handle with binding index in buffer resource list.
IR::IREmitter ir{block, IR::Block::InstructionList::s_iterator_to(inst)};
inst.SetArg(0, ir.Imm32(binding));
ASSERT(!buffer.swizzle_enable && !buffer.add_tid_enable);
// Address of constant buffer reads can be calculated at IR emittion time.
if (inst.GetOpcode() == IR::Opcode::ReadConstBuffer ||
inst.GetOpcode() == IR::Opcode::ReadConstBufferU32) {
return; return;
} }
@ -434,10 +439,14 @@ void PatchBufferInstruction(IR::Block& block, IR::Inst& inst, Info& info,
} }
} else { } else {
const u32 stride = buffer.GetStride(); const u32 stride = buffer.GetStride();
ASSERT_MSG(stride >= 4, "non-formatting load_buffer_* is not implemented for stride {}", if (stride < 4) {
stride); LOG_WARNING(Render_Vulkan,
"non-formatting load_buffer_* is not implemented for stride {}", stride);
}
} }
// Compute address of the buffer using the stride.
// Todo: What if buffer is rebound with different stride?
IR::U32 address = ir.Imm32(inst_info.inst_offset.Value()); IR::U32 address = ir.Imm32(inst_info.inst_offset.Value());
if (inst_info.index_enable) { if (inst_info.index_enable) {
const IR::U32 index = inst_info.offset_enable ? IR::U32{ir.CompositeExtract(inst.Arg(1), 0)} const IR::U32 index = inst_info.offset_enable ? IR::U32{ir.CompositeExtract(inst.Arg(1), 0)}
@ -587,39 +596,9 @@ void PatchImageInstruction(IR::Block& block, IR::Inst& inst, Info& info, Descrip
} }
void ResourceTrackingPass(IR::Program& program) { void ResourceTrackingPass(IR::Program& program) {
// When loading data from untyped buffer we don't have if it is float or integer.
// Most of the time it is float so that is the default. This pass detects float buffer loads
// combined with bitcasts and patches them to be integer loads.
for (IR::Block* const block : program.post_order_blocks) {
break;
for (IR::Inst& inst : block->Instructions()) {
if (inst.GetOpcode() != IR::Opcode::BitCastU32F32) {
continue;
}
// Replace the bitcast with a typed buffer read
IR::Inst* const arg_inst{inst.Arg(0).TryInstRecursive()};
if (!arg_inst) {
continue;
}
const auto replace{[&](IR::Opcode new_opcode) {
inst.ReplaceOpcode(new_opcode);
inst.SetArg(0, arg_inst->Arg(0));
inst.SetArg(1, arg_inst->Arg(1));
inst.SetFlags(arg_inst->Flags<u32>());
arg_inst->Invalidate();
}};
if (arg_inst->GetOpcode() == IR::Opcode::ReadConstBuffer) {
replace(IR::Opcode::ReadConstBufferU32);
}
if (arg_inst->GetOpcode() == IR::Opcode::LoadBufferF32) {
replace(IR::Opcode::LoadBufferU32);
}
}
}
// Iterate resource instructions and patch them after finding the sharp. // Iterate resource instructions and patch them after finding the sharp.
auto& info = program.info; auto& info = program.info;
Descriptors descriptors{info.buffers, info.images, info.samplers}; Descriptors descriptors{info};
for (IR::Block* const block : program.blocks) { for (IR::Block* const block : program.blocks) {
for (IR::Inst& inst : block->Instructions()) { for (IR::Inst& inst : block->Instructions()) {
if (IsBufferInstruction(inst)) { if (IsBufferInstruction(inst)) {

View File

@ -27,9 +27,9 @@ IR::BlockList GenerateBlocks(const IR::AbstractSyntaxList& syntax_list) {
return blocks; return blocks;
} }
IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Block>& block_pool, IR::Program TranslateProgram(Common::ObjectPool<IR::Inst>& inst_pool,
std::span<const u32> token, const Info&& info, Common::ObjectPool<IR::Block>& block_pool, std::span<const u32> token,
const Profile& profile) { const Info&& info, const Profile& profile) {
// Ensure first instruction is expected. // Ensure first instruction is expected.
constexpr u32 token_mov_vcchi = 0xBEEB03FF; constexpr u32 token_mov_vcchi = 0xBEEB03FF;
ASSERT_MSG(token[0] == token_mov_vcchi, "First instruction is not s_mov_b32 vcc_hi, #imm"); ASSERT_MSG(token[0] == token_mov_vcchi, "First instruction is not s_mov_b32 vcc_hi, #imm");
@ -45,7 +45,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo
} }
// Create control flow graph // Create control flow graph
ObjectPool<Gcn::Block> gcn_block_pool{64}; Common::ObjectPool<Gcn::Block> gcn_block_pool{64};
Gcn::CFG cfg{gcn_block_pool, program.ins_list}; Gcn::CFG cfg{gcn_block_pool, program.ins_list};
// Structurize control flow graph and create program. // Structurize control flow graph and create program.
@ -61,7 +61,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo
Shader::Optimization::IdentityRemovalPass(program.blocks); Shader::Optimization::IdentityRemovalPass(program.blocks);
Shader::Optimization::DeadCodeEliminationPass(program); Shader::Optimization::DeadCodeEliminationPass(program);
Shader::Optimization::CollectShaderInfoPass(program); Shader::Optimization::CollectShaderInfoPass(program);
LOG_INFO(Render_Vulkan, "{}", Shader::IR::DumpProgram(program)); LOG_DEBUG(Render_Vulkan, "{}", Shader::IR::DumpProgram(program));
return program; return program;
} }

View File

@ -3,16 +3,16 @@
#pragma once #pragma once
#include "common/object_pool.h"
#include "shader_recompiler/ir/basic_block.h" #include "shader_recompiler/ir/basic_block.h"
#include "shader_recompiler/ir/program.h" #include "shader_recompiler/ir/program.h"
#include "shader_recompiler/object_pool.h"
namespace Shader { namespace Shader {
struct Profile; struct Profile;
[[nodiscard]] IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, [[nodiscard]] IR::Program TranslateProgram(Common::ObjectPool<IR::Inst>& inst_pool,
ObjectPool<IR::Block>& block_pool, Common::ObjectPool<IR::Block>& block_pool,
std::span<const u32> code, const Info&& info, std::span<const u32> code, const Info&& info,
const Profile& profile); const Profile& profile);

View File

@ -77,8 +77,11 @@ struct BufferResource {
u32 length; u32 length;
IR::Type used_types; IR::Type used_types;
AmdGpu::Buffer inline_cbuf; AmdGpu::Buffer inline_cbuf;
bool is_storage{false}; AmdGpu::DataFormat dfmt;
bool is_instance_data{false}; AmdGpu::NumberFormat nfmt;
bool is_storage{};
bool is_instance_data{};
bool is_written{};
constexpr AmdGpu::Buffer GetVsharp(const Info& info) const noexcept; constexpr AmdGpu::Buffer GetVsharp(const Info& info) const noexcept;
}; };
@ -105,6 +108,19 @@ struct SamplerResource {
}; };
using SamplerResourceList = boost::container::static_vector<SamplerResource, 16>; using SamplerResourceList = boost::container::static_vector<SamplerResource, 16>;
struct PushData {
static constexpr size_t BufOffsetIndex = 2;
u32 step0;
u32 step1;
std::array<u8, 32> buf_offsets;
void AddOffset(u32 binding, u32 offset) {
ASSERT(offset < 64 && binding < 32);
buf_offsets[binding] = offset;
}
};
struct Info { struct Info {
struct VsInput { struct VsInput {
enum InstanceIdType : u8 { enum InstanceIdType : u8 {
@ -182,6 +198,7 @@ struct Info {
bool uses_shared_u8{}; bool uses_shared_u8{};
bool uses_shared_u16{}; bool uses_shared_u16{};
bool uses_fp16{}; bool uses_fp16{};
bool uses_step_rates{};
bool translation_failed{}; // indicates that shader has unsupported instructions bool translation_failed{}; // indicates that shader has unsupported instructions
template <typename T> template <typename T>

View File

@ -496,7 +496,7 @@ struct Liverpool {
template <typename T = VAddr> template <typename T = VAddr>
T Address() const { T Address() const {
return reinterpret_cast<T>((base_addr_lo & ~1U) | u64(base_addr_hi) << 32); return std::bit_cast<T>((base_addr_lo & ~1U) | u64(base_addr_hi) << 32);
} }
}; };

View File

@ -363,6 +363,10 @@ struct Sampler {
return raw0 != 0 || raw1 != 0; return raw0 != 0 || raw1 != 0;
} }
bool operator==(const Sampler& other) const noexcept {
return std::memcmp(this, &other, sizeof(Sampler)) == 0;
}
float LodBias() const noexcept { float LodBias() const noexcept {
return static_cast<float>(static_cast<int16_t>((lod_bias.Value() ^ 0x2000u) - 0x2000u)) / return static_cast<float>(static_cast<int16_t>((lod_bias.Value() ^ 0x2000u) - 0x2000u)) /
256.0f; 256.0f;

View File

@ -0,0 +1,227 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "common/alignment.h"
#include "common/assert.h"
#include "video_core/buffer_cache/buffer.h"
#include "video_core/renderer_vulkan/liverpool_to_vk.h"
#include "video_core/renderer_vulkan/vk_instance.h"
#include "video_core/renderer_vulkan/vk_platform.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
#include <vk_mem_alloc.h>
namespace VideoCore {
constexpr vk::BufferUsageFlags AllFlags =
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst |
vk::BufferUsageFlagBits::eUniformTexelBuffer | vk::BufferUsageFlagBits::eStorageTexelBuffer |
vk::BufferUsageFlagBits::eUniformBuffer | vk::BufferUsageFlagBits::eStorageBuffer |
vk::BufferUsageFlagBits::eIndexBuffer | vk::BufferUsageFlagBits::eVertexBuffer;
std::string_view BufferTypeName(MemoryUsage type) {
switch (type) {
case MemoryUsage::Upload:
return "Upload";
case MemoryUsage::Download:
return "Download";
case MemoryUsage::Stream:
return "Stream";
case MemoryUsage::DeviceLocal:
return "DeviceLocal";
default:
return "Invalid";
}
}
[[nodiscard]] VkMemoryPropertyFlags MemoryUsagePreferredVmaFlags(MemoryUsage usage) {
return usage != MemoryUsage::DeviceLocal ? VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
: VkMemoryPropertyFlagBits{};
}
[[nodiscard]] VmaAllocationCreateFlags MemoryUsageVmaFlags(MemoryUsage usage) {
switch (usage) {
case MemoryUsage::Upload:
case MemoryUsage::Stream:
return VMA_ALLOCATION_CREATE_MAPPED_BIT |
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT;
case MemoryUsage::Download:
return VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT;
case MemoryUsage::DeviceLocal:
return {};
}
return {};
}
[[nodiscard]] VmaMemoryUsage MemoryUsageVma(MemoryUsage usage) {
switch (usage) {
case MemoryUsage::DeviceLocal:
case MemoryUsage::Stream:
return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
case MemoryUsage::Upload:
case MemoryUsage::Download:
return VMA_MEMORY_USAGE_AUTO_PREFER_HOST;
}
return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
}
UniqueBuffer::UniqueBuffer(vk::Device device_, VmaAllocator allocator_)
: device{device_}, allocator{allocator_} {}
UniqueBuffer::~UniqueBuffer() {
if (buffer) {
vmaDestroyBuffer(allocator, buffer, allocation);
}
}
void UniqueBuffer::Create(const vk::BufferCreateInfo& buffer_ci, MemoryUsage usage,
VmaAllocationInfo* out_alloc_info) {
const VmaAllocationCreateInfo alloc_ci = {
.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage),
.usage = MemoryUsageVma(usage),
.requiredFlags = 0,
.preferredFlags = MemoryUsagePreferredVmaFlags(usage),
.pool = VK_NULL_HANDLE,
.pUserData = nullptr,
};
const VkBufferCreateInfo buffer_ci_unsafe = static_cast<VkBufferCreateInfo>(buffer_ci);
VkBuffer unsafe_buffer{};
VkResult result = vmaCreateBuffer(allocator, &buffer_ci_unsafe, &alloc_ci, &unsafe_buffer,
&allocation, out_alloc_info);
ASSERT_MSG(result == VK_SUCCESS, "Failed allocating buffer with error {}",
vk::to_string(vk::Result{result}));
buffer = vk::Buffer{unsafe_buffer};
}
Buffer::Buffer(const Vulkan::Instance& instance_, MemoryUsage usage_, VAddr cpu_addr_,
u64 size_bytes_)
: cpu_addr{cpu_addr_}, size_bytes{size_bytes_}, instance{&instance_}, usage{usage_},
buffer{instance->GetDevice(), instance->GetAllocator()} {
// Create buffer object.
const vk::BufferCreateInfo buffer_ci = {
.size = size_bytes,
.usage = AllFlags,
};
VmaAllocationInfo alloc_info{};
buffer.Create(buffer_ci, usage, &alloc_info);
if (instance->HasDebuggingToolAttached()) {
const auto device = instance->GetDevice();
Vulkan::SetObjectName(device, Handle(), "Buffer {:#x} {} KiB", cpu_addr, size_bytes / 1024);
}
// Map it if it is host visible.
VkMemoryPropertyFlags property_flags{};
vmaGetAllocationMemoryProperties(instance->GetAllocator(), buffer.allocation, &property_flags);
if (alloc_info.pMappedData) {
mapped_data = std::span<u8>{std::bit_cast<u8*>(alloc_info.pMappedData), size_bytes};
}
is_coherent = property_flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
}
vk::BufferView Buffer::View(u32 offset, u32 size, AmdGpu::DataFormat dfmt,
AmdGpu::NumberFormat nfmt) {
const auto it{std::ranges::find_if(views, [offset, size, dfmt, nfmt](const BufferView& view) {
return offset == view.offset && size == view.size && dfmt == view.dfmt && nfmt == view.nfmt;
})};
if (it != views.end()) {
return it->handle;
}
views.push_back({
.offset = offset,
.size = size,
.dfmt = dfmt,
.nfmt = nfmt,
.handle = instance->GetDevice().createBufferView({
.buffer = buffer.buffer,
.format = Vulkan::LiverpoolToVK::SurfaceFormat(dfmt, nfmt),
.offset = offset,
.range = size,
}),
});
return views.back().handle;
}
constexpr u64 WATCHES_INITIAL_RESERVE = 0x4000;
constexpr u64 WATCHES_RESERVE_CHUNK = 0x1000;
StreamBuffer::StreamBuffer(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler_,
MemoryUsage usage, u64 size_bytes)
: Buffer{instance, usage, 0, size_bytes}, scheduler{scheduler_} {
ReserveWatches(current_watches, WATCHES_INITIAL_RESERVE);
ReserveWatches(previous_watches, WATCHES_INITIAL_RESERVE);
const auto device = instance.GetDevice();
if (instance.HasDebuggingToolAttached()) {
Vulkan::SetObjectName(device, Handle(), "StreamBuffer({}): {} KiB", BufferTypeName(usage),
size_bytes / 1024);
}
}
std::pair<u8*, u64> StreamBuffer::Map(u64 size, u64 alignment) {
if (!is_coherent && usage == MemoryUsage::Stream) {
size = Common::AlignUp(size, instance->NonCoherentAtomSize());
}
ASSERT(size <= this->size_bytes);
mapped_size = size;
if (alignment > 0) {
offset = Common::AlignUp(offset, alignment);
}
if (offset + size > this->size_bytes) {
// The buffer would overflow, save the amount of used watches and reset the state.
invalidation_mark = current_watch_cursor;
current_watch_cursor = 0;
offset = 0;
// Swap watches and reset waiting cursors.
std::swap(previous_watches, current_watches);
wait_cursor = 0;
wait_bound = 0;
}
const u64 mapped_upper_bound = offset + size;
WaitPendingOperations(mapped_upper_bound);
return std::make_pair(mapped_data.data() + offset, offset);
}
void StreamBuffer::Commit() {
if (!is_coherent) {
if (usage == MemoryUsage::Download) {
vmaInvalidateAllocation(instance->GetAllocator(), buffer.allocation, offset,
mapped_size);
} else {
vmaFlushAllocation(instance->GetAllocator(), buffer.allocation, offset, mapped_size);
}
}
offset += mapped_size;
if (current_watch_cursor + 1 >= current_watches.size()) {
// Ensure that there are enough watches.
ReserveWatches(current_watches, WATCHES_RESERVE_CHUNK);
}
auto& watch = current_watches[current_watch_cursor++];
watch.upper_bound = offset;
watch.tick = scheduler.CurrentTick();
}
void StreamBuffer::ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size) {
watches.resize(watches.size() + grow_size);
}
void StreamBuffer::WaitPendingOperations(u64 requested_upper_bound) {
if (!invalidation_mark) {
return;
}
while (requested_upper_bound > wait_bound && wait_cursor < *invalidation_mark) {
auto& watch = previous_watches[wait_cursor];
wait_bound = watch.upper_bound;
scheduler.Wait(watch.tick);
++wait_cursor;
}
}
} // namespace VideoCore

View File

@ -0,0 +1,173 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <cstddef>
#include <utility>
#include <vector>
#include "common/types.h"
#include "video_core/amdgpu/resource.h"
#include "video_core/renderer_vulkan/vk_common.h"
namespace Vulkan {
class Instance;
class Scheduler;
} // namespace Vulkan
VK_DEFINE_HANDLE(VmaAllocation)
VK_DEFINE_HANDLE(VmaAllocator)
struct VmaAllocationInfo;
namespace VideoCore {
/// Hints and requirements for the backing memory type of a commit
enum class MemoryUsage {
DeviceLocal, ///< Requests device local buffer.
Upload, ///< Requires a host visible memory type optimized for CPU to GPU uploads
Download, ///< Requires a host visible memory type optimized for GPU to CPU readbacks
Stream, ///< Requests device local host visible buffer, falling back host memory.
};
struct UniqueBuffer {
explicit UniqueBuffer(vk::Device device, VmaAllocator allocator);
~UniqueBuffer();
UniqueBuffer(const UniqueBuffer&) = delete;
UniqueBuffer& operator=(const UniqueBuffer&) = delete;
UniqueBuffer(UniqueBuffer&& other)
: buffer{std::exchange(other.buffer, VK_NULL_HANDLE)},
allocator{std::exchange(other.allocator, VK_NULL_HANDLE)},
allocation{std::exchange(other.allocation, VK_NULL_HANDLE)} {}
UniqueBuffer& operator=(UniqueBuffer&& other) {
buffer = std::exchange(other.buffer, VK_NULL_HANDLE);
allocator = std::exchange(other.allocator, VK_NULL_HANDLE);
allocation = std::exchange(other.allocation, VK_NULL_HANDLE);
return *this;
}
void Create(const vk::BufferCreateInfo& image_ci, MemoryUsage usage,
VmaAllocationInfo* out_alloc_info);
operator vk::Buffer() const {
return buffer;
}
vk::Device device;
VmaAllocator allocator;
VmaAllocation allocation;
vk::Buffer buffer{};
};
class Buffer {
public:
explicit Buffer(const Vulkan::Instance& instance, MemoryUsage usage, VAddr cpu_addr_,
u64 size_bytes_);
Buffer& operator=(const Buffer&) = delete;
Buffer(const Buffer&) = delete;
Buffer& operator=(Buffer&&) = default;
Buffer(Buffer&&) = default;
vk::BufferView View(u32 offset, u32 size, AmdGpu::DataFormat dfmt, AmdGpu::NumberFormat nfmt);
/// Increases the likeliness of this being a stream buffer
void IncreaseStreamScore(int score) noexcept {
stream_score += score;
}
/// Returns the likeliness of this being a stream buffer
[[nodiscard]] int StreamScore() const noexcept {
return stream_score;
}
/// Returns true when vaddr -> vaddr+size is fully contained in the buffer
[[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept {
return addr >= cpu_addr && addr + size <= cpu_addr + SizeBytes();
}
/// Returns the base CPU address of the buffer
[[nodiscard]] VAddr CpuAddr() const noexcept {
return cpu_addr;
}
/// Returns the offset relative to the given CPU address
[[nodiscard]] u32 Offset(VAddr other_cpu_addr) const noexcept {
return static_cast<u32>(other_cpu_addr - cpu_addr);
}
size_t SizeBytes() const {
return size_bytes;
}
vk::Buffer Handle() const noexcept {
return buffer;
}
public:
VAddr cpu_addr = 0;
bool is_picked{};
bool is_coherent{};
int stream_score = 0;
size_t size_bytes = 0;
std::span<u8> mapped_data;
const Vulkan::Instance* instance{};
MemoryUsage usage;
UniqueBuffer buffer;
struct BufferView {
u32 offset;
u32 size;
AmdGpu::DataFormat dfmt;
AmdGpu::NumberFormat nfmt;
vk::BufferView handle;
};
std::vector<BufferView> views;
};
class StreamBuffer : public Buffer {
public:
explicit StreamBuffer(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
MemoryUsage usage, u64 size_bytes_);
/// Reserves a region of memory from the stream buffer.
std::pair<u8*, u64> Map(u64 size, u64 alignment = 0);
/// Ensures that reserved bytes of memory are available to the GPU.
void Commit();
/// Maps and commits a memory region with user provided data
u64 Copy(VAddr src, size_t size, size_t alignment = 0) {
const auto [data, offset] = Map(size, alignment);
std::memcpy(data, reinterpret_cast<const void*>(src), size);
Commit();
return offset;
}
private:
struct Watch {
u64 tick{};
u64 upper_bound{};
};
/// Increases the amount of watches available.
void ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size);
/// Waits pending watches until requested upper bound.
void WaitPendingOperations(u64 requested_upper_bound);
private:
Vulkan::Scheduler& scheduler;
u64 offset{};
u64 mapped_size{};
std::vector<Watch> current_watches;
std::size_t current_watch_cursor{};
std::optional<size_t> invalidation_mark;
std::vector<Watch> previous_watches;
std::size_t wait_cursor{};
u64 wait_bound{};
};
} // namespace VideoCore

View File

@ -0,0 +1,497 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include "common/alignment.h"
#include "common/scope_exit.h"
#include "shader_recompiler/runtime_info.h"
#include "video_core/amdgpu/liverpool.h"
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/renderer_vulkan/liverpool_to_vk.h"
#include "video_core/renderer_vulkan/vk_instance.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
namespace VideoCore {
static constexpr size_t StagingBufferSize = 256_MB;
static constexpr size_t UboStreamBufferSize = 64_MB;
BufferCache::BufferCache(const Vulkan::Instance& instance_, Vulkan::Scheduler& scheduler_,
const AmdGpu::Liverpool* liverpool_, PageManager& tracker_)
: instance{instance_}, scheduler{scheduler_}, liverpool{liverpool_}, tracker{tracker_},
staging_buffer{instance, scheduler, MemoryUsage::Upload, StagingBufferSize},
stream_buffer{instance, scheduler, MemoryUsage::Stream, UboStreamBufferSize},
memory_tracker{&tracker} {
// Ensure the first slot is used for the null buffer
void(slot_buffers.insert(instance, MemoryUsage::DeviceLocal, 0, 1));
}
BufferCache::~BufferCache() = default;
void BufferCache::InvalidateMemory(VAddr device_addr, u64 size) {
std::scoped_lock lk{mutex};
const bool is_tracked = IsRegionRegistered(device_addr, size);
if (!is_tracked) {
return;
}
// Mark the page as CPU modified to stop tracking writes.
SCOPE_EXIT {
memory_tracker.MarkRegionAsCpuModified(device_addr, size);
};
if (!memory_tracker.IsRegionGpuModified(device_addr, size)) {
// Page has not been modified by the GPU, nothing to do.
return;
}
}
void BufferCache::DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 size) {
boost::container::small_vector<vk::BufferCopy, 1> copies;
u64 total_size_bytes = 0;
u64 largest_copy = 0;
memory_tracker.ForEachDownloadRange<true>(
device_addr, size, [&](u64 device_addr_out, u64 range_size) {
const VAddr buffer_addr = buffer.CpuAddr();
const auto add_download = [&](VAddr start, VAddr end, u64) {
const u64 new_offset = start - buffer_addr;
const u64 new_size = end - start;
copies.push_back(vk::BufferCopy{
.srcOffset = new_offset,
.dstOffset = total_size_bytes,
.size = new_size,
});
// Align up to avoid cache conflicts
constexpr u64 align = 64ULL;
constexpr u64 mask = ~(align - 1ULL);
total_size_bytes += (new_size + align - 1) & mask;
largest_copy = std::max(largest_copy, new_size);
};
});
if (total_size_bytes == 0) {
return;
}
const auto [staging, offset] = staging_buffer.Map(total_size_bytes);
for (auto& copy : copies) {
// Modify copies to have the staging offset in mind
copy.dstOffset += offset;
}
staging_buffer.Commit();
scheduler.EndRendering();
const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.copyBuffer(buffer.buffer, staging_buffer.Handle(), copies);
scheduler.Finish();
for (const auto& copy : copies) {
const VAddr copy_device_addr = buffer.CpuAddr() + copy.srcOffset;
const u64 dst_offset = copy.dstOffset - offset;
std::memcpy(std::bit_cast<u8*>(copy_device_addr), staging + dst_offset, copy.size);
}
}
bool BufferCache::BindVertexBuffers(const Shader::Info& vs_info) {
if (vs_info.vs_inputs.empty()) {
return false;
}
std::array<vk::Buffer, NUM_VERTEX_BUFFERS> host_buffers;
std::array<vk::DeviceSize, NUM_VERTEX_BUFFERS> host_offsets;
boost::container::static_vector<AmdGpu::Buffer, NUM_VERTEX_BUFFERS> guest_buffers;
struct BufferRange {
VAddr base_address;
VAddr end_address;
vk::Buffer vk_buffer;
u64 offset;
size_t GetSize() const {
return end_address - base_address;
}
};
// Calculate buffers memory overlaps
bool has_step_rate = false;
boost::container::static_vector<BufferRange, NUM_VERTEX_BUFFERS> ranges{};
for (const auto& input : vs_info.vs_inputs) {
if (input.instance_step_rate == Shader::Info::VsInput::InstanceIdType::OverStepRate0 ||
input.instance_step_rate == Shader::Info::VsInput::InstanceIdType::OverStepRate1) {
has_step_rate = true;
continue;
}
const auto& buffer = vs_info.ReadUd<AmdGpu::Buffer>(input.sgpr_base, input.dword_offset);
if (buffer.GetSize() == 0) {
continue;
}
guest_buffers.emplace_back(buffer);
ranges.emplace_back(buffer.base_address, buffer.base_address + buffer.GetSize());
}
std::ranges::sort(ranges, [](const BufferRange& lhv, const BufferRange& rhv) {
return lhv.base_address < rhv.base_address;
});
boost::container::static_vector<BufferRange, NUM_VERTEX_BUFFERS> ranges_merged{ranges[0]};
for (auto range : ranges) {
auto& prev_range = ranges_merged.back();
if (prev_range.end_address < range.base_address) {
ranges_merged.emplace_back(range);
} else {
prev_range.end_address = std::max(prev_range.end_address, range.end_address);
}
}
// Map buffers
for (auto& range : ranges_merged) {
const auto [buffer, offset] = ObtainBuffer(range.base_address, range.GetSize(), false);
range.vk_buffer = buffer->buffer;
range.offset = offset;
}
// Bind vertex buffers
const size_t num_buffers = guest_buffers.size();
for (u32 i = 0; i < num_buffers; ++i) {
const auto& buffer = guest_buffers[i];
const auto host_buffer = std::ranges::find_if(ranges_merged, [&](const BufferRange& range) {
return (buffer.base_address >= range.base_address &&
buffer.base_address < range.end_address);
});
ASSERT(host_buffer != ranges_merged.cend());
host_buffers[i] = host_buffer->vk_buffer;
host_offsets[i] = host_buffer->offset + buffer.base_address - host_buffer->base_address;
}
if (num_buffers > 0) {
const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.bindVertexBuffers(0, num_buffers, host_buffers.data(), host_offsets.data());
}
return has_step_rate;
}
u32 BufferCache::BindIndexBuffer(bool& is_indexed, u32 index_offset) {
// Emulate QuadList primitive type with CPU made index buffer.
const auto& regs = liverpool->regs;
if (regs.primitive_type == AmdGpu::Liverpool::PrimitiveType::QuadList) {
is_indexed = true;
// Emit indices.
const u32 index_size = 3 * regs.num_indices;
const auto [data, offset] = stream_buffer.Map(index_size);
Vulkan::LiverpoolToVK::EmitQuadToTriangleListIndices(data, regs.num_indices);
stream_buffer.Commit();
// Bind index buffer.
const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.bindIndexBuffer(stream_buffer.Handle(), offset, vk::IndexType::eUint16);
return index_size / sizeof(u16);
}
if (!is_indexed) {
return regs.num_indices;
}
// Figure out index type and size.
const bool is_index16 =
regs.index_buffer_type.index_type == AmdGpu::Liverpool::IndexType::Index16;
const vk::IndexType index_type = is_index16 ? vk::IndexType::eUint16 : vk::IndexType::eUint32;
const u32 index_size = is_index16 ? sizeof(u16) : sizeof(u32);
VAddr index_address = regs.index_base_address.Address<VAddr>();
index_address += index_offset * index_size;
// Bind index buffer.
const u32 index_buffer_size = regs.num_indices * index_size;
const auto [vk_buffer, offset] = ObtainBuffer(index_address, index_buffer_size, false);
const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.bindIndexBuffer(vk_buffer->Handle(), offset, index_type);
return regs.num_indices;
}
std::pair<Buffer*, u32> BufferCache::ObtainBuffer(VAddr device_addr, u32 size, bool is_written) {
std::scoped_lock lk{mutex};
static constexpr u64 StreamThreshold = CACHING_PAGESIZE;
const bool is_gpu_dirty = memory_tracker.IsRegionGpuModified(device_addr, size);
if (!is_written && size < StreamThreshold && !is_gpu_dirty) {
// For small uniform buffers that have not been modified by gpu
// use device local stream buffer to reduce renderpass breaks.
const u64 offset = stream_buffer.Copy(device_addr, size, instance.UniformMinAlignment());
return {&stream_buffer, offset};
}
const BufferId buffer_id = FindBuffer(device_addr, size);
Buffer& buffer = slot_buffers[buffer_id];
SynchronizeBuffer(buffer, device_addr, size);
if (is_written) {
memory_tracker.MarkRegionAsGpuModified(device_addr, size);
}
return {&buffer, buffer.Offset(device_addr)};
}
bool BufferCache::IsRegionRegistered(VAddr addr, size_t size) {
const VAddr end_addr = addr + size;
const u64 page_end = Common::DivCeil(end_addr, CACHING_PAGESIZE);
for (u64 page = addr >> CACHING_PAGEBITS; page < page_end;) {
const BufferId buffer_id = page_table[page];
if (!buffer_id) {
++page;
continue;
}
Buffer& buffer = slot_buffers[buffer_id];
const VAddr buf_start_addr = buffer.CpuAddr();
const VAddr buf_end_addr = buf_start_addr + buffer.SizeBytes();
if (buf_start_addr < end_addr && addr < buf_end_addr) {
return true;
}
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
}
return false;
}
bool BufferCache::IsRegionCpuModified(VAddr addr, size_t size) {
return memory_tracker.IsRegionCpuModified(addr, size);
}
BufferId BufferCache::FindBuffer(VAddr device_addr, u32 size) {
if (device_addr == 0) {
return NULL_BUFFER_ID;
}
const u64 page = device_addr >> CACHING_PAGEBITS;
const BufferId buffer_id = page_table[page];
if (!buffer_id) {
return CreateBuffer(device_addr, size);
}
const Buffer& buffer = slot_buffers[buffer_id];
if (buffer.IsInBounds(device_addr, size)) {
return buffer_id;
}
return CreateBuffer(device_addr, size);
}
BufferCache::OverlapResult BufferCache::ResolveOverlaps(VAddr device_addr, u32 wanted_size) {
static constexpr int STREAM_LEAP_THRESHOLD = 16;
boost::container::small_vector<BufferId, 16> overlap_ids;
VAddr begin = device_addr;
VAddr end = device_addr + wanted_size;
int stream_score = 0;
bool has_stream_leap = false;
const auto expand_begin = [&](VAddr add_value) {
static constexpr VAddr min_page = CACHING_PAGESIZE + DEVICE_PAGESIZE;
if (add_value > begin - min_page) {
begin = min_page;
device_addr = DEVICE_PAGESIZE;
return;
}
begin -= add_value;
device_addr = begin - CACHING_PAGESIZE;
};
const auto expand_end = [&](VAddr add_value) {
static constexpr VAddr max_page = 1ULL << MemoryTracker::MAX_CPU_PAGE_BITS;
if (add_value > max_page - end) {
end = max_page;
return;
}
end += add_value;
};
if (begin == 0) {
return OverlapResult{
.ids = std::move(overlap_ids),
.begin = begin,
.end = end,
.has_stream_leap = has_stream_leap,
};
}
for (; device_addr >> CACHING_PAGEBITS < Common::DivCeil(end, CACHING_PAGESIZE);
device_addr += CACHING_PAGESIZE) {
const BufferId overlap_id = page_table[device_addr >> CACHING_PAGEBITS];
if (!overlap_id) {
continue;
}
Buffer& overlap = slot_buffers[overlap_id];
if (overlap.is_picked) {
continue;
}
overlap_ids.push_back(overlap_id);
overlap.is_picked = true;
const VAddr overlap_device_addr = overlap.CpuAddr();
const bool expands_left = overlap_device_addr < begin;
if (expands_left) {
begin = overlap_device_addr;
}
const VAddr overlap_end = overlap_device_addr + overlap.SizeBytes();
const bool expands_right = overlap_end > end;
if (overlap_end > end) {
end = overlap_end;
}
stream_score += overlap.StreamScore();
if (stream_score > STREAM_LEAP_THRESHOLD && !has_stream_leap) {
// When this memory region has been joined a bunch of times, we assume it's being used
// as a stream buffer. Increase the size to skip constantly recreating buffers.
has_stream_leap = true;
if (expands_right) {
expand_begin(CACHING_PAGESIZE * 128);
}
if (expands_left) {
expand_end(CACHING_PAGESIZE * 128);
}
}
}
return OverlapResult{
.ids = std::move(overlap_ids),
.begin = begin,
.end = end,
.has_stream_leap = has_stream_leap,
};
}
void BufferCache::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id,
bool accumulate_stream_score) {
Buffer& new_buffer = slot_buffers[new_buffer_id];
Buffer& overlap = slot_buffers[overlap_id];
if (accumulate_stream_score) {
new_buffer.IncreaseStreamScore(overlap.StreamScore() + 1);
}
const size_t dst_base_offset = overlap.CpuAddr() - new_buffer.CpuAddr();
const vk::BufferCopy copy = {
.srcOffset = 0,
.dstOffset = dst_base_offset,
.size = overlap.SizeBytes(),
};
scheduler.EndRendering();
const auto cmdbuf = scheduler.CommandBuffer();
static constexpr vk::MemoryBarrier READ_BARRIER{
.srcAccessMask = vk::AccessFlagBits::eMemoryWrite,
.dstAccessMask = vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite,
};
static constexpr vk::MemoryBarrier WRITE_BARRIER{
.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
.dstAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
};
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
READ_BARRIER, {}, {});
cmdbuf.copyBuffer(overlap.buffer, new_buffer.buffer, copy);
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eAllCommands,
vk::DependencyFlagBits::eByRegion, WRITE_BARRIER, {}, {});
DeleteBuffer(overlap_id, true);
}
BufferId BufferCache::CreateBuffer(VAddr device_addr, u32 wanted_size) {
const VAddr device_addr_end = Common::AlignUp(device_addr + wanted_size, CACHING_PAGESIZE);
device_addr = Common::AlignDown(device_addr, CACHING_PAGESIZE);
wanted_size = static_cast<u32>(device_addr_end - device_addr);
const OverlapResult overlap = ResolveOverlaps(device_addr, wanted_size);
const u32 size = static_cast<u32>(overlap.end - overlap.begin);
const BufferId new_buffer_id =
slot_buffers.insert(instance, MemoryUsage::DeviceLocal, overlap.begin, size);
auto& new_buffer = slot_buffers[new_buffer_id];
const size_t size_bytes = new_buffer.SizeBytes();
const auto cmdbuf = scheduler.CommandBuffer();
scheduler.EndRendering();
cmdbuf.fillBuffer(new_buffer.buffer, 0, size_bytes, 0);
for (const BufferId overlap_id : overlap.ids) {
JoinOverlap(new_buffer_id, overlap_id, !overlap.has_stream_leap);
}
Register(new_buffer_id);
return new_buffer_id;
}
void BufferCache::Register(BufferId buffer_id) {
ChangeRegister<true>(buffer_id);
}
void BufferCache::Unregister(BufferId buffer_id) {
ChangeRegister<false>(buffer_id);
}
template <bool insert>
void BufferCache::ChangeRegister(BufferId buffer_id) {
Buffer& buffer = slot_buffers[buffer_id];
const auto size = buffer.SizeBytes();
const VAddr device_addr_begin = buffer.CpuAddr();
const VAddr device_addr_end = device_addr_begin + size;
const u64 page_begin = device_addr_begin / CACHING_PAGESIZE;
const u64 page_end = Common::DivCeil(device_addr_end, CACHING_PAGESIZE);
for (u64 page = page_begin; page != page_end; ++page) {
if constexpr (insert) {
page_table[page] = buffer_id;
} else {
page_table[page] = BufferId{};
}
}
}
bool BufferCache::SynchronizeBuffer(Buffer& buffer, VAddr device_addr, u32 size) {
boost::container::small_vector<vk::BufferCopy, 4> copies;
u64 total_size_bytes = 0;
u64 largest_copy = 0;
VAddr buffer_start = buffer.CpuAddr();
const auto add_copy = [&](VAddr device_addr_out, u64 range_size) {
copies.push_back(vk::BufferCopy{
.srcOffset = total_size_bytes,
.dstOffset = device_addr_out - buffer_start,
.size = range_size,
});
total_size_bytes += range_size;
largest_copy = std::max(largest_copy, range_size);
};
memory_tracker.ForEachUploadRange(device_addr, size, [&](u64 device_addr_out, u64 range_size) {
add_copy(device_addr_out, range_size);
// Prevent uploading to gpu modified regions.
// gpu_modified_ranges.ForEachNotInRange(device_addr_out, range_size, add_copy);
});
if (total_size_bytes == 0) {
return true;
}
vk::Buffer src_buffer = staging_buffer.Handle();
if (total_size_bytes < StagingBufferSize) {
const auto [staging, offset] = staging_buffer.Map(total_size_bytes);
for (auto& copy : copies) {
u8* const src_pointer = staging + copy.srcOffset;
const VAddr device_addr = buffer.CpuAddr() + copy.dstOffset;
std::memcpy(src_pointer, std::bit_cast<const u8*>(device_addr), copy.size);
// Apply the staging offset
copy.srcOffset += offset;
}
staging_buffer.Commit();
} else {
// For large one time transfers use a temporary host buffer.
// RenderDoc can lag quite a bit if the stream buffer is too large.
Buffer temp_buffer{instance, MemoryUsage::Upload, 0, total_size_bytes};
src_buffer = temp_buffer.Handle();
u8* const staging = temp_buffer.mapped_data.data();
for (auto& copy : copies) {
u8* const src_pointer = staging + copy.srcOffset;
const VAddr device_addr = buffer.CpuAddr() + copy.dstOffset;
std::memcpy(src_pointer, std::bit_cast<const u8*>(device_addr), copy.size);
}
scheduler.DeferOperation([buffer = std::move(temp_buffer)]() mutable {});
}
scheduler.EndRendering();
const auto cmdbuf = scheduler.CommandBuffer();
static constexpr vk::MemoryBarrier READ_BARRIER{
.srcAccessMask = vk::AccessFlagBits::eMemoryWrite,
.dstAccessMask = vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite,
};
static constexpr vk::MemoryBarrier WRITE_BARRIER{
.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
.dstAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
};
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
READ_BARRIER, {}, {});
cmdbuf.copyBuffer(src_buffer, buffer.buffer, copies);
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eAllCommands,
vk::DependencyFlagBits::eByRegion, WRITE_BARRIER, {}, {});
return false;
}
void BufferCache::DeleteBuffer(BufferId buffer_id, bool do_not_mark) {
// Mark the whole buffer as CPU written to stop tracking CPU writes
if (!do_not_mark) {
Buffer& buffer = slot_buffers[buffer_id];
memory_tracker.MarkRegionAsCpuModified(buffer.CpuAddr(), buffer.SizeBytes());
}
Unregister(buffer_id);
scheduler.DeferOperation([this, buffer_id] { slot_buffers.erase(buffer_id); });
}
} // namespace VideoCore

View File

@ -0,0 +1,129 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <array>
#include <mutex>
#include <boost/container/small_vector.hpp>
#include <boost/icl/interval_map.hpp>
#include <tsl/robin_map.h>
#include "common/div_ceil.h"
#include "common/slot_vector.h"
#include "common/types.h"
#include "video_core/buffer_cache/buffer.h"
#include "video_core/buffer_cache/memory_tracker_base.h"
#include "video_core/multi_level_page_table.h"
namespace AmdGpu {
struct Liverpool;
}
namespace Shader {
struct Info;
}
namespace VideoCore {
using BufferId = Common::SlotId;
static constexpr BufferId NULL_BUFFER_ID{0};
static constexpr u32 NUM_VERTEX_BUFFERS = 32;
class BufferCache {
public:
static constexpr u32 CACHING_PAGEBITS = 12;
static constexpr u64 CACHING_PAGESIZE = u64{1} << CACHING_PAGEBITS;
static constexpr u64 DEVICE_PAGESIZE = 4_KB;
struct Traits {
using Entry = BufferId;
static constexpr size_t AddressSpaceBits = 39;
static constexpr size_t FirstLevelBits = 14;
static constexpr size_t PageBits = CACHING_PAGEBITS;
};
using PageTable = MultiLevelPageTable<Traits>;
struct OverlapResult {
boost::container::small_vector<BufferId, 16> ids;
VAddr begin;
VAddr end;
bool has_stream_leap = false;
};
public:
explicit BufferCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
const AmdGpu::Liverpool* liverpool, PageManager& tracker);
~BufferCache();
/// Invalidates any buffer in the logical page range.
void InvalidateMemory(VAddr device_addr, u64 size);
/// Binds host vertex buffers for the current draw.
bool BindVertexBuffers(const Shader::Info& vs_info);
/// Bind host index buffer for the current draw.
u32 BindIndexBuffer(bool& is_indexed, u32 index_offset);
/// Obtains a buffer for the specified region.
[[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(VAddr gpu_addr, u32 size, bool is_written);
/// Return true when a region is registered on the cache
[[nodiscard]] bool IsRegionRegistered(VAddr addr, size_t size);
/// Return true when a CPU region is modified from the CPU
[[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
private:
template <typename Func>
void ForEachBufferInRange(VAddr device_addr, u64 size, Func&& func) {
const u64 page_end = Common::DivCeil(device_addr + size, CACHING_PAGESIZE);
for (u64 page = device_addr >> CACHING_PAGEBITS; page < page_end;) {
const BufferId buffer_id = page_table[page];
if (!buffer_id) {
++page;
continue;
}
Buffer& buffer = slot_buffers[buffer_id];
func(buffer_id, buffer);
const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
}
}
void DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 size);
[[nodiscard]] BufferId FindBuffer(VAddr device_addr, u32 size);
[[nodiscard]] OverlapResult ResolveOverlaps(VAddr device_addr, u32 wanted_size);
void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
[[nodiscard]] BufferId CreateBuffer(VAddr device_addr, u32 wanted_size);
void Register(BufferId buffer_id);
void Unregister(BufferId buffer_id);
template <bool insert>
void ChangeRegister(BufferId buffer_id);
bool SynchronizeBuffer(Buffer& buffer, VAddr device_addr, u32 size);
void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false);
const Vulkan::Instance& instance;
Vulkan::Scheduler& scheduler;
const AmdGpu::Liverpool* liverpool;
PageManager& tracker;
StreamBuffer staging_buffer;
StreamBuffer stream_buffer;
std::recursive_mutex mutex;
Common::SlotVector<Buffer> slot_buffers;
MemoryTracker memory_tracker;
PageTable page_table;
};
} // namespace VideoCore

View File

@ -0,0 +1,175 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <algorithm>
#include <deque>
#include <type_traits>
#include <vector>
#include "common/types.h"
#include "video_core/buffer_cache/word_manager.h"
namespace VideoCore {
class MemoryTracker {
public:
static constexpr size_t MAX_CPU_PAGE_BITS = 39;
static constexpr size_t HIGHER_PAGE_BITS = 22;
static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS;
static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL;
static constexpr size_t NUM_HIGH_PAGES = 1ULL << (MAX_CPU_PAGE_BITS - HIGHER_PAGE_BITS);
static constexpr size_t MANAGER_POOL_SIZE = 32;
static constexpr size_t WORDS_STACK_NEEDED = HIGHER_PAGE_SIZE / BYTES_PER_WORD;
using Manager = WordManager<WORDS_STACK_NEEDED>;
public:
explicit MemoryTracker(PageManager* tracker_) : tracker{tracker_} {}
~MemoryTracker() = default;
/// Returns true if a region has been modified from the CPU
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
return IteratePages<true>(
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
return manager->template IsRegionModified<Type::CPU>(offset, size);
});
}
/// Returns true if a region has been modified from the GPU
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
return IteratePages<false>(
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
return manager->template IsRegionModified<Type::GPU>(offset, size);
});
}
/// Mark region as CPU modified, notifying the device_tracker about this change
void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
IteratePages<true>(dirty_cpu_addr, query_size,
[](Manager* manager, u64 offset, size_t size) {
manager->template ChangeRegionState<Type::CPU, true>(
manager->GetCpuAddr() + offset, size);
});
}
/// Unmark region as CPU modified, notifying the device_tracker about this change
void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
IteratePages<true>(dirty_cpu_addr, query_size,
[](Manager* manager, u64 offset, size_t size) {
manager->template ChangeRegionState<Type::CPU, false>(
manager->GetCpuAddr() + offset, size);
});
}
/// Mark region as modified from the host GPU
void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
IteratePages<true>(dirty_cpu_addr, query_size,
[](Manager* manager, u64 offset, size_t size) {
manager->template ChangeRegionState<Type::GPU, true>(
manager->GetCpuAddr() + offset, size);
});
}
/// Unmark region as modified from the host GPU
void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
IteratePages<true>(dirty_cpu_addr, query_size,
[](Manager* manager, u64 offset, size_t size) {
manager->template ChangeRegionState<Type::GPU, false>(
manager->GetCpuAddr() + offset, size);
});
}
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
template <typename Func>
void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
IteratePages<true>(query_cpu_range, query_size,
[&func](Manager* manager, u64 offset, size_t size) {
manager->template ForEachModifiedRange<Type::CPU, true>(
manager->GetCpuAddr() + offset, size, func);
});
}
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
template <bool clear, typename Func>
void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
IteratePages<false>(query_cpu_range, query_size,
[&func](Manager* manager, u64 offset, size_t size) {
if constexpr (clear) {
manager->template ForEachModifiedRange<Type::GPU, true>(
manager->GetCpuAddr() + offset, size, func);
} else {
manager->template ForEachModifiedRange<Type::GPU, false>(
manager->GetCpuAddr() + offset, size, func);
}
});
}
private:
/**
* @brief IteratePages Iterates L2 word manager page table.
* @param cpu_address Start byte cpu address
* @param size Size in bytes of the region of iterate.
* @param func Callback for each word manager.
* @return
*/
template <bool create_region_on_fail, typename Func>
bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
using FuncReturn = typename std::invoke_result<Func, Manager*, u64, size_t>::type;
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
std::size_t remaining_size{size};
std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
while (remaining_size > 0) {
const std::size_t copy_amount{
std::min<std::size_t>(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
auto* manager{top_tier[page_index]};
if (manager) {
if constexpr (BOOL_BREAK) {
if (func(manager, page_offset, copy_amount)) {
return true;
}
} else {
func(manager, page_offset, copy_amount);
}
} else if constexpr (create_region_on_fail) {
CreateRegion(page_index);
manager = top_tier[page_index];
if constexpr (BOOL_BREAK) {
if (func(manager, page_offset, copy_amount)) {
return true;
}
} else {
func(manager, page_offset, copy_amount);
}
}
page_index++;
page_offset = 0;
remaining_size -= copy_amount;
}
return false;
}
void CreateRegion(std::size_t page_index) {
const VAddr base_cpu_addr = page_index << HIGHER_PAGE_BITS;
if (free_managers.empty()) {
manager_pool.emplace_back();
auto& last_pool = manager_pool.back();
for (size_t i = 0; i < MANAGER_POOL_SIZE; i++) {
std::construct_at(&last_pool[i], tracker, 0, HIGHER_PAGE_SIZE);
free_managers.push_back(&last_pool[i]);
}
}
// Each manager tracks a 4_MB virtual address space.
auto* new_manager = free_managers.back();
new_manager->SetCpuAddress(base_cpu_addr);
free_managers.pop_back();
top_tier[page_index] = new_manager;
}
PageManager* tracker;
std::deque<std::array<Manager, MANAGER_POOL_SIZE>> manager_pool;
std::vector<Manager*> free_managers;
std::array<Manager*, NUM_HIGH_PAGES> top_tier{};
};
} // namespace VideoCore

View File

@ -0,0 +1,159 @@
// SPDX-FileCopyrightText: 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <boost/icl/interval_map.hpp>
#include <boost/pool/pool.hpp>
#include <boost/pool/pool_alloc.hpp>
#include <boost/pool/poolfwd.hpp>
#include "common/types.h"
namespace VideoCore {
template <class T>
using RangeSetsAllocator =
boost::fast_pool_allocator<T, boost::default_user_allocator_new_delete,
boost::details::pool::default_mutex, 1024, 2048>;
struct RangeSet {
using IntervalSet =
boost::icl::interval_set<VAddr, std::less,
ICL_INTERVAL_INSTANCE(ICL_INTERVAL_DEFAULT, VAddr, std::less),
RangeSetsAllocator>;
using IntervalType = typename IntervalSet::interval_type;
explicit RangeSet() = default;
~RangeSet() = default;
void Add(VAddr base_address, size_t size) {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
m_ranges_set.add(interval);
}
void Subtract(VAddr base_address, size_t size) {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
m_ranges_set.subtract(interval);
}
template <typename Func>
void ForEach(Func&& func) const {
if (m_ranges_set.empty()) {
return;
}
auto it = m_ranges_set.begin();
auto end_it = m_ranges_set.end();
for (; it != end_it; it++) {
const VAddr inter_addr_end = it->upper();
const VAddr inter_addr = it->lower();
func(inter_addr, inter_addr_end);
}
}
template <typename Func>
void ForEachInRange(VAddr base_addr, size_t size, Func&& func) const {
if (m_ranges_set.empty()) {
return;
}
const VAddr start_address = base_addr;
const VAddr end_address = start_address + size;
const IntervalType search_interval{start_address, end_address};
auto it = m_ranges_set.lower_bound(search_interval);
if (it == m_ranges_set.end()) {
return;
}
auto end_it = m_ranges_set.upper_bound(search_interval);
for (; it != end_it; it++) {
VAddr inter_addr_end = it->upper();
VAddr inter_addr = it->lower();
if (inter_addr_end > end_address) {
inter_addr_end = end_address;
}
if (inter_addr < start_address) {
inter_addr = start_address;
}
func(inter_addr, inter_addr_end);
}
}
IntervalSet m_ranges_set;
};
class RangeMap {
public:
using IntervalMap =
boost::icl::interval_map<VAddr, u64, boost::icl::partial_absorber, std::less,
boost::icl::inplace_plus, boost::icl::inter_section,
ICL_INTERVAL_INSTANCE(ICL_INTERVAL_DEFAULT, VAddr, std::less),
RangeSetsAllocator>;
using IntervalType = typename IntervalMap::interval_type;
public:
RangeMap() = default;
~RangeMap() = default;
RangeMap(RangeMap const&) = delete;
RangeMap& operator=(RangeMap const&) = delete;
RangeMap(RangeMap&& other);
RangeMap& operator=(RangeMap&& other);
void Add(VAddr base_address, size_t size, u64 value) {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
m_ranges_map.add({interval, value});
}
void Subtract(VAddr base_address, size_t size) {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
m_ranges_map -= interval;
}
template <typename Func>
void ForEachInRange(VAddr base_addr, size_t size, Func&& func) const {
if (m_ranges_map.empty()) {
return;
}
const VAddr start_address = base_addr;
const VAddr end_address = start_address + size;
const IntervalType search_interval{start_address, end_address};
auto it = m_ranges_map.lower_bound(search_interval);
if (it == m_ranges_map.end()) {
return;
}
auto end_it = m_ranges_map.upper_bound(search_interval);
for (; it != end_it; it++) {
VAddr inter_addr_end = it->first.upper();
VAddr inter_addr = it->first.lower();
if (inter_addr_end > end_address) {
inter_addr_end = end_address;
}
if (inter_addr < start_address) {
inter_addr = start_address;
}
func(inter_addr, inter_addr_end, it->second);
}
}
template <typename Func>
void ForEachNotInRange(VAddr base_addr, size_t size, Func&& func) const {
const VAddr end_addr = base_addr + size;
ForEachInRange(base_addr, size, [&](VAddr range_addr, VAddr range_end, u64) {
if (size_t gap_size = range_addr - base_addr; gap_size != 0) {
func(base_addr, gap_size);
}
base_addr = range_end;
});
if (base_addr != end_addr) {
func(base_addr, end_addr - base_addr);
}
}
private:
IntervalMap m_ranges_map;
};
} // namespace VideoCore

View File

@ -0,0 +1,398 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <algorithm>
#include <span>
#include <utility>
#include "common/div_ceil.h"
#include "common/types.h"
#include "video_core/page_manager.h"
namespace VideoCore {
constexpr u64 PAGES_PER_WORD = 64;
constexpr u64 BYTES_PER_PAGE = 4_KB;
constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
enum class Type {
CPU,
GPU,
Untracked,
};
/// Vector tracking modified pages tightly packed with small vector optimization
template <size_t stack_words = 1>
struct WordsArray {
/// Returns the pointer to the words state
[[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
return is_short ? stack.data() : heap;
}
/// Returns the pointer to the words state
[[nodiscard]] u64* Pointer(bool is_short) noexcept {
return is_short ? stack.data() : heap;
}
std::array<u64, stack_words> stack{}; ///< Small buffers storage
u64* heap; ///< Not-small buffers pointer to the storage
};
template <size_t stack_words = 1>
struct Words {
explicit Words() = default;
explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
num_words = Common::DivCeil(size_bytes, BYTES_PER_WORD);
if (IsShort()) {
cpu.stack.fill(~u64{0});
gpu.stack.fill(0);
untracked.stack.fill(~u64{0});
} else {
// Share allocation between CPU and GPU pages and set their default values
u64* const alloc = new u64[num_words * 3];
cpu.heap = alloc;
gpu.heap = alloc + num_words;
untracked.heap = alloc + num_words * 2;
std::fill_n(cpu.heap, num_words, ~u64{0});
std::fill_n(gpu.heap, num_words, 0);
std::fill_n(untracked.heap, num_words, ~u64{0});
}
// Clean up tailing bits
const u64 last_word_size = size_bytes % BYTES_PER_WORD;
const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
const u64 last_word = (~u64{0} << shift) >> shift;
cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
}
~Words() {
Release();
}
Words& operator=(Words&& rhs) noexcept {
Release();
size_bytes = rhs.size_bytes;
num_words = rhs.num_words;
cpu = rhs.cpu;
gpu = rhs.gpu;
untracked = rhs.untracked;
rhs.cpu.heap = nullptr;
return *this;
}
Words(Words&& rhs) noexcept
: size_bytes{rhs.size_bytes}, num_words{rhs.num_words}, cpu{rhs.cpu}, gpu{rhs.gpu},
untracked{rhs.untracked} {
rhs.cpu.heap = nullptr;
}
Words& operator=(const Words&) = delete;
Words(const Words&) = delete;
/// Returns true when the buffer fits in the small vector optimization
[[nodiscard]] bool IsShort() const noexcept {
return num_words <= stack_words;
}
/// Returns the number of words of the buffer
[[nodiscard]] size_t NumWords() const noexcept {
return num_words;
}
/// Release buffer resources
void Release() {
if (!IsShort()) {
// CPU written words is the base for the heap allocation
delete[] cpu.heap;
}
}
template <Type type>
std::span<u64> Span() noexcept {
if constexpr (type == Type::CPU) {
return std::span<u64>(cpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::GPU) {
return std::span<u64>(gpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::Untracked) {
return std::span<u64>(untracked.Pointer(IsShort()), num_words);
}
}
template <Type type>
std::span<const u64> Span() const noexcept {
if constexpr (type == Type::CPU) {
return std::span<const u64>(cpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::GPU) {
return std::span<const u64>(gpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::Untracked) {
return std::span<const u64>(untracked.Pointer(IsShort()), num_words);
}
}
u64 size_bytes = 0;
size_t num_words = 0;
WordsArray<stack_words> cpu;
WordsArray<stack_words> gpu;
WordsArray<stack_words> untracked;
};
template <size_t stack_words = 1>
class WordManager {
public:
explicit WordManager(PageManager* tracker_, VAddr cpu_addr_, u64 size_bytes)
: tracker{tracker_}, cpu_addr{cpu_addr_}, words{size_bytes} {}
explicit WordManager() = default;
void SetCpuAddress(VAddr new_cpu_addr) {
cpu_addr = new_cpu_addr;
}
VAddr GetCpuAddr() const {
return cpu_addr;
}
static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) {
constexpr size_t number_bits = sizeof(u64) * 8;
const size_t limit_page_end = number_bits - std::min(page_end, number_bits);
u64 bits = (word >> page_start) << page_start;
bits = (bits << limit_page_end) >> limit_page_end;
return bits;
}
static std::pair<size_t, size_t> GetWordPage(VAddr address) {
const size_t converted_address = static_cast<size_t>(address);
const size_t word_number = converted_address / BYTES_PER_WORD;
const size_t amount_pages = converted_address % BYTES_PER_WORD;
return std::make_pair(word_number, amount_pages / BYTES_PER_PAGE);
}
template <typename Func>
void IterateWords(size_t offset, size_t size, Func&& func) const {
using FuncReturn = std::invoke_result_t<Func, std::size_t, u64>;
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
const size_t start = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset), 0LL));
const size_t end = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset + size), 0LL));
if (start >= SizeBytes() || end <= start) {
return;
}
auto [start_word, start_page] = GetWordPage(start);
auto [end_word, end_page] = GetWordPage(end + BYTES_PER_PAGE - 1ULL);
const size_t num_words = NumWords();
start_word = std::min(start_word, num_words);
end_word = std::min(end_word, num_words);
const size_t diff = end_word - start_word;
end_word += (end_page + PAGES_PER_WORD - 1ULL) / PAGES_PER_WORD;
end_word = std::min(end_word, num_words);
end_page += diff * PAGES_PER_WORD;
constexpr u64 base_mask{~0ULL};
for (size_t word_index = start_word; word_index < end_word; word_index++) {
const u64 mask = ExtractBits(base_mask, start_page, end_page);
start_page = 0;
end_page -= PAGES_PER_WORD;
if constexpr (BOOL_BREAK) {
if (func(word_index, mask)) {
return;
}
} else {
func(word_index, mask);
}
}
}
template <typename Func>
void IteratePages(u64 mask, Func&& func) const {
size_t offset = 0;
while (mask != 0) {
const size_t empty_bits = std::countr_zero(mask);
offset += empty_bits;
mask = mask >> empty_bits;
const size_t continuous_bits = std::countr_one(mask);
func(offset, continuous_bits);
mask = continuous_bits < PAGES_PER_WORD ? (mask >> continuous_bits) : 0;
offset += continuous_bits;
}
}
/**
* Change the state of a range of pages
*
* @param dirty_addr Base address to mark or unmark as modified
* @param size Size in bytes to mark or unmark as modified
*/
template <Type type, bool enable>
void ChangeRegionState(u64 dirty_addr, u64 size) noexcept(type == Type::GPU) {
std::span<u64> state_words = words.template Span<type>();
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
IterateWords(dirty_addr - cpu_addr, size, [&](size_t index, u64 mask) {
if constexpr (type == Type::CPU) {
NotifyPageTracker<!enable>(index, untracked_words[index], mask);
}
if constexpr (enable) {
state_words[index] |= mask;
if constexpr (type == Type::CPU) {
untracked_words[index] |= mask;
}
} else {
state_words[index] &= ~mask;
if constexpr (type == Type::CPU) {
untracked_words[index] &= ~mask;
}
}
});
}
/**
* Loop over each page in the given range, turn off those bits and notify the tracker if
* needed. Call the given function on each turned off range.
*
* @param query_cpu_range Base CPU address to loop over
* @param size Size in bytes of the CPU range to loop over
* @param func Function to call for each turned off region
*/
template <Type type, bool clear, typename Func>
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) {
static_assert(type != Type::Untracked);
std::span<u64> state_words = words.template Span<type>();
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
const size_t offset = query_cpu_range - cpu_addr;
bool pending = false;
size_t pending_offset{};
size_t pending_pointer{};
const auto release = [&]() {
func(cpu_addr + pending_offset * BYTES_PER_PAGE,
(pending_pointer - pending_offset) * BYTES_PER_PAGE);
};
IterateWords(offset, size, [&](size_t index, u64 mask) {
if constexpr (type == Type::GPU) {
mask &= ~untracked_words[index];
}
const u64 word = state_words[index] & mask;
if constexpr (clear) {
if constexpr (type == Type::CPU) {
NotifyPageTracker<true>(index, untracked_words[index], mask);
}
state_words[index] &= ~mask;
if constexpr (type == Type::CPU) {
untracked_words[index] &= ~mask;
}
}
const size_t base_offset = index * PAGES_PER_WORD;
IteratePages(word, [&](size_t pages_offset, size_t pages_size) {
const auto reset = [&]() {
pending_offset = base_offset + pages_offset;
pending_pointer = base_offset + pages_offset + pages_size;
};
if (!pending) {
reset();
pending = true;
return;
}
if (pending_pointer == base_offset + pages_offset) {
pending_pointer += pages_size;
return;
}
release();
reset();
});
});
if (pending) {
release();
}
}
/**
* Returns true when a region has been modified
*
* @param offset Offset in bytes from the start of the buffer
* @param size Size in bytes of the region to query for modifications
*/
template <Type type>
[[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
static_assert(type != Type::Untracked);
const std::span<const u64> state_words = words.template Span<type>();
[[maybe_unused]] const std::span<const u64> untracked_words =
words.template Span<Type::Untracked>();
bool result = false;
IterateWords(offset, size, [&](size_t index, u64 mask) {
if constexpr (type == Type::GPU) {
mask &= ~untracked_words[index];
}
const u64 word = state_words[index] & mask;
if (word != 0) {
result = true;
return true;
}
return false;
});
return result;
}
/// Returns the number of words of the manager
[[nodiscard]] size_t NumWords() const noexcept {
return words.NumWords();
}
/// Returns the size in bytes of the manager
[[nodiscard]] u64 SizeBytes() const noexcept {
return words.size_bytes;
}
/// Returns true when the buffer fits in the small vector optimization
[[nodiscard]] bool IsShort() const noexcept {
return words.IsShort();
}
private:
template <Type type>
u64* Array() noexcept {
if constexpr (type == Type::CPU) {
return words.cpu.Pointer(IsShort());
} else if constexpr (type == Type::GPU) {
return words.gpu.Pointer(IsShort());
} else if constexpr (type == Type::Untracked) {
return words.untracked.Pointer(IsShort());
}
}
template <Type type>
const u64* Array() const noexcept {
if constexpr (type == Type::CPU) {
return words.cpu.Pointer(IsShort());
} else if constexpr (type == Type::GPU) {
return words.gpu.Pointer(IsShort());
} else if constexpr (type == Type::Untracked) {
return words.untracked.Pointer(IsShort());
}
}
/**
* Notify tracker about changes in the CPU tracking state of a word in the buffer
*
* @param word_index Index to the word to notify to the tracker
* @param current_bits Current state of the word
* @param new_bits New state of the word
*
* @tparam add_to_tracker True when the tracker should start tracking the new pages
*/
template <bool add_to_tracker>
void NotifyPageTracker(u64 word_index, u64 current_bits, u64 new_bits) const {
u64 changed_bits = (add_to_tracker ? current_bits : ~current_bits) & new_bits;
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
IteratePages(changed_bits, [&](size_t offset, size_t size) {
tracker->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE, size * BYTES_PER_PAGE,
add_to_tracker ? 1 : -1);
});
}
PageManager* tracker;
VAddr cpu_addr = 0;
Words<stack_words> words;
};
} // namespace VideoCore

View File

@ -0,0 +1,65 @@
// SPDX-FileCopyrightText: 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <type_traits>
#include <utility>
#include <vector>
#include "common/object_pool.h"
#include "common/types.h"
namespace VideoCore {
template <class Traits>
class MultiLevelPageTable final {
using Entry = typename Traits::Entry;
static constexpr size_t AddressSpaceBits = Traits::AddressSpaceBits;
static constexpr size_t FirstLevelBits = Traits::FirstLevelBits;
static constexpr size_t PageBits = Traits::PageBits;
static constexpr size_t FirstLevelShift = AddressSpaceBits - FirstLevelBits;
static constexpr size_t SecondLevelBits = FirstLevelShift - PageBits;
static constexpr size_t NumEntriesPerL1Page = 1ULL << SecondLevelBits;
using L1Page = std::array<Entry, NumEntriesPerL1Page>;
public:
explicit MultiLevelPageTable() : first_level_map{1ULL << FirstLevelBits, nullptr} {}
~MultiLevelPageTable() noexcept = default;
[[nodiscard]] Entry* find(size_t page) {
const size_t l1_page = page >> SecondLevelBits;
const size_t l2_page = page & (NumEntriesPerL1Page - 1);
if (!first_level_map[l1_page]) {
return nullptr;
}
return &(*first_level_map[l1_page])[l2_page];
}
[[nodiscard]] const Entry& operator[](size_t page) const {
const size_t l1_page = page >> SecondLevelBits;
const size_t l2_page = page & (NumEntriesPerL1Page - 1);
if (!first_level_map[l1_page]) {
first_level_map[l1_page] = page_alloc.Create();
}
return (*first_level_map[l1_page])[l2_page];
}
[[nodiscard]] Entry& operator[](size_t page) {
const size_t l1_page = page >> SecondLevelBits;
const size_t l2_page = page & (NumEntriesPerL1Page - 1);
if (!first_level_map[l1_page]) {
first_level_map[l1_page] = page_alloc.Create();
}
return (*first_level_map[l1_page])[l2_page];
}
private:
std::vector<L1Page*> first_level_map{};
Common::ObjectPool<L1Page> page_alloc;
};
} // namespace VideoCore

View File

@ -0,0 +1,260 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <thread>
#include "common/alignment.h"
#include "common/assert.h"
#include "common/error.h"
#include "video_core/page_manager.h"
#include "video_core/renderer_vulkan/vk_rasterizer.h"
#ifndef _WIN64
#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#ifdef ENABLE_USERFAULTFD
#include <linux/userfaultfd.h>
#endif
#else
#include <windows.h>
#endif
namespace VideoCore {
constexpr size_t PAGESIZE = 4_KB;
constexpr size_t PAGEBITS = 12;
#ifdef _WIN64
struct PageManager::Impl {
Impl(Vulkan::Rasterizer* rasterizer_) {
rasterizer = rasterizer_;
veh_handle = AddVectoredExceptionHandler(0, GuestFaultSignalHandler);
ASSERT_MSG(veh_handle, "Failed to register an exception handler");
}
void OnMap(VAddr address, size_t size) {}
void OnUnmap(VAddr address, size_t size) {}
void Protect(VAddr address, size_t size, bool allow_write) {
DWORD prot = allow_write ? PAGE_READWRITE : PAGE_READONLY;
DWORD old_prot{};
BOOL result = VirtualProtect(std::bit_cast<LPVOID>(address), size, prot, &old_prot);
ASSERT_MSG(result != 0, "Region protection failed");
}
static LONG WINAPI GuestFaultSignalHandler(EXCEPTION_POINTERS* pExp) noexcept {
const u32 ec = pExp->ExceptionRecord->ExceptionCode;
if (ec == EXCEPTION_ACCESS_VIOLATION) {
const auto info = pExp->ExceptionRecord->ExceptionInformation;
if (info[0] == 1) { // Write violation
rasterizer->InvalidateMemory(info[1], sizeof(u64));
return EXCEPTION_CONTINUE_EXECUTION;
} /* else {
UNREACHABLE();
}*/
}
return EXCEPTION_CONTINUE_SEARCH; // pass further
}
inline static Vulkan::Rasterizer* rasterizer;
void* veh_handle{};
};
#elif ENABLE_USERFAULTFD
struct PageManager::Impl {
Impl(Vulkan::Rasterizer* rasterizer_) : rasterizer{rasterizer_} {
uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
ASSERT_MSG(uffd != -1, "{}", Common::GetLastErrorMsg());
// Request uffdio features from kernel.
uffdio_api api;
api.api = UFFD_API;
api.features = UFFD_FEATURE_THREAD_ID;
const int ret = ioctl(uffd, UFFDIO_API, &api);
ASSERT(ret == 0 && api.api == UFFD_API);
// Create uffd handler thread
ufd_thread = std::jthread([&](std::stop_token token) { UffdHandler(token); });
}
void OnMap(VAddr address, size_t size) {
uffdio_register reg;
reg.range.start = address;
reg.range.len = size;
reg.mode = UFFDIO_REGISTER_MODE_WP;
const int ret = ioctl(uffd, UFFDIO_REGISTER, &reg);
ASSERT_MSG(ret != -1, "Uffdio register failed");
}
void OnUnmap(VAddr address, size_t size) {
uffdio_range range;
range.start = address;
range.len = size;
const int ret = ioctl(uffd, UFFDIO_UNREGISTER, &range);
ASSERT_MSG(ret != -1, "Uffdio unregister failed");
}
void Protect(VAddr address, size_t size, bool allow_write) {
uffdio_writeprotect wp;
wp.range.start = address;
wp.range.len = size;
wp.mode = allow_write ? 0 : UFFDIO_WRITEPROTECT_MODE_WP;
const int ret = ioctl(uffd, UFFDIO_WRITEPROTECT, &wp);
ASSERT_MSG(ret != -1, "Uffdio writeprotect failed with error: {}",
Common::GetLastErrorMsg());
}
void UffdHandler(std::stop_token token) {
while (!token.stop_requested()) {
pollfd pollfd;
pollfd.fd = uffd;
pollfd.events = POLLIN;
// Block until the descriptor is ready for data reads.
const int pollres = poll(&pollfd, 1, -1);
switch (pollres) {
case -1:
perror("Poll userfaultfd");
continue;
break;
case 0:
continue;
case 1:
break;
default:
UNREACHABLE_MSG("Unexpected number of descriptors {} out of poll", pollres);
}
// We don't want an error condition to have occured.
ASSERT_MSG(!(pollfd.revents & POLLERR), "POLLERR on userfaultfd");
// We waited until there is data to read, we don't care about anything else.
if (!(pollfd.revents & POLLIN)) {
continue;
}
// Read message from kernel.
uffd_msg msg;
const int readret = read(uffd, &msg, sizeof(msg));
ASSERT_MSG(readret != -1 || errno == EAGAIN, "Unexpected result of uffd read");
if (errno == EAGAIN) {
continue;
}
ASSERT_MSG(readret == sizeof(msg), "Unexpected short read, exiting");
ASSERT(msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP);
// Notify rasterizer about the fault.
const VAddr addr = msg.arg.pagefault.address;
const VAddr addr_page = Common::AlignDown(addr, PAGESIZE);
rasterizer->InvalidateMemory(addr_page, PAGESIZE);
}
}
Vulkan::Rasterizer* rasterizer;
std::jthread ufd_thread;
int uffd;
};
#else
struct PageManager::Impl {
Impl(Vulkan::Rasterizer* rasterizer_) {
rasterizer = rasterizer_;
#ifdef __APPLE__
// Read-only memory write results in SIGBUS on Apple.
static constexpr int SignalType = SIGBUS;
#else
static constexpr int SignalType = SIGSEGV;
#endif
sigset_t signal_mask;
sigemptyset(&signal_mask);
sigaddset(&signal_mask, SignalType);
using HandlerType = decltype(sigaction::sa_sigaction);
struct sigaction guest_access_fault {};
guest_access_fault.sa_flags = SA_SIGINFO | SA_ONSTACK;
guest_access_fault.sa_sigaction = &GuestFaultSignalHandler;
guest_access_fault.sa_mask = signal_mask;
sigaction(SignalType, &guest_access_fault, nullptr);
}
void OnMap(VAddr address, size_t size) {}
void OnUnmap(VAddr address, size_t size) {}
void Protect(VAddr address, size_t size, bool allow_write) {
mprotect(reinterpret_cast<void*>(address), size,
PROT_READ | (allow_write ? PROT_WRITE : 0));
}
static void GuestFaultSignalHandler(int sig, siginfo_t* info, void* raw_context) {
ucontext_t* ctx = reinterpret_cast<ucontext_t*>(raw_context);
const VAddr address = reinterpret_cast<VAddr>(info->si_addr);
#ifdef __APPLE__
const u32 err = ctx->uc_mcontext->__es.__err;
#else
const greg_t err = ctx->uc_mcontext.gregs[REG_ERR];
#endif
if (err & 0x2) {
rasterizer->InvalidateMemory(address, sizeof(u64));
} else {
// Read not supported!
UNREACHABLE();
}
}
inline static Vulkan::Rasterizer* rasterizer;
};
#endif
PageManager::PageManager(Vulkan::Rasterizer* rasterizer_)
: impl{std::make_unique<Impl>(rasterizer_)}, rasterizer{rasterizer_} {}
PageManager::~PageManager() = default;
void PageManager::OnGpuMap(VAddr address, size_t size) {
impl->OnMap(address, size);
}
void PageManager::OnGpuUnmap(VAddr address, size_t size) {
impl->OnUnmap(address, size);
}
void PageManager::UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta) {
static constexpr u64 PageShift = 12;
std::scoped_lock lk{mutex};
const u64 num_pages = ((addr + size - 1) >> PageShift) - (addr >> PageShift) + 1;
const u64 page_start = addr >> PageShift;
const u64 page_end = page_start + num_pages;
const auto pages_interval =
decltype(cached_pages)::interval_type::right_open(page_start, page_end);
if (delta > 0) {
cached_pages.add({pages_interval, delta});
}
const auto& range = cached_pages.equal_range(pages_interval);
for (const auto& [range, count] : boost::make_iterator_range(range)) {
const auto interval = range & pages_interval;
const VAddr interval_start_addr = boost::icl::first(interval) << PageShift;
const VAddr interval_end_addr = boost::icl::last_next(interval) << PageShift;
const u32 interval_size = interval_end_addr - interval_start_addr;
if (delta > 0 && count == delta) {
impl->Protect(interval_start_addr, interval_size, false);
} else if (delta < 0 && count == -delta) {
impl->Protect(interval_start_addr, interval_size, true);
} else {
ASSERT(count >= 0);
}
}
if (delta < 0) {
cached_pages.add({pages_interval, delta});
}
}
} // namespace VideoCore

View File

@ -0,0 +1,39 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <memory>
#include <mutex>
#include <boost/icl/interval_map.hpp>
#include "common/types.h"
namespace Vulkan {
class Rasterizer;
}
namespace VideoCore {
class PageManager {
public:
explicit PageManager(Vulkan::Rasterizer* rasterizer);
~PageManager();
/// Register a range of mapped gpu memory.
void OnGpuMap(VAddr address, size_t size);
/// Unregister a range of gpu memory that was unmapped.
void OnGpuUnmap(VAddr address, size_t size);
/// Increase/decrease the number of surface in pages touching the specified region
void UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta);
private:
struct Impl;
std::unique_ptr<Impl> impl;
Vulkan::Rasterizer* rasterizer;
std::mutex mutex;
boost::icl::interval_map<VAddr, s32> cached_pages;
};
} // namespace VideoCore

View File

@ -67,8 +67,8 @@ RendererVulkan::RendererVulkan(Frontend::WindowSDL& window_, AmdGpu::Liverpool*
: window{window_}, liverpool{liverpool_}, : window{window_}, liverpool{liverpool_},
instance{window, Config::getGpuId(), Config::vkValidationEnabled()}, draw_scheduler{instance}, instance{window, Config::getGpuId(), Config::vkValidationEnabled()}, draw_scheduler{instance},
present_scheduler{instance}, flip_scheduler{instance}, swapchain{instance, window}, present_scheduler{instance}, flip_scheduler{instance}, swapchain{instance, window},
texture_cache{instance, draw_scheduler} { rasterizer{std::make_unique<Rasterizer>(instance, draw_scheduler, liverpool)},
rasterizer = std::make_unique<Rasterizer>(instance, draw_scheduler, texture_cache, liverpool); texture_cache{rasterizer->GetTextureCache()} {
const u32 num_images = swapchain.GetImageCount(); const u32 num_images = swapchain.GetImageCount();
const vk::Device device = instance.GetDevice(); const vk::Device device = instance.GetDevice();

View File

@ -47,7 +47,7 @@ public:
Frame* PrepareFrame(const Libraries::VideoOut::BufferAttributeGroup& attribute, Frame* PrepareFrame(const Libraries::VideoOut::BufferAttributeGroup& attribute,
VAddr cpu_address, bool is_eop) { VAddr cpu_address, bool is_eop) {
const auto info = VideoCore::ImageInfo{attribute, cpu_address}; const auto info = VideoCore::ImageInfo{attribute, cpu_address};
const auto image_id = texture_cache.FindImage(info, cpu_address); const auto image_id = texture_cache.FindImage(info, false);
auto& image = texture_cache.GetImage(image_id); auto& image = texture_cache.GetImage(image_id);
return PrepareFrameInternal(image, is_eop); return PrepareFrameInternal(image, is_eop);
} }
@ -61,7 +61,7 @@ public:
const Libraries::VideoOut::BufferAttributeGroup& attribute, VAddr cpu_address) { const Libraries::VideoOut::BufferAttributeGroup& attribute, VAddr cpu_address) {
vo_buffers_addr.emplace_back(cpu_address); vo_buffers_addr.emplace_back(cpu_address);
const auto info = VideoCore::ImageInfo{attribute, cpu_address}; const auto info = VideoCore::ImageInfo{attribute, cpu_address};
const auto image_id = texture_cache.FindImage(info, cpu_address); const auto image_id = texture_cache.FindImage(info, false);
return texture_cache.GetImage(image_id); return texture_cache.GetImage(image_id);
} }
@ -88,7 +88,7 @@ private:
Scheduler flip_scheduler; Scheduler flip_scheduler;
Swapchain swapchain; Swapchain swapchain;
std::unique_ptr<Rasterizer> rasterizer; std::unique_ptr<Rasterizer> rasterizer;
VideoCore::TextureCache texture_cache; VideoCore::TextureCache& texture_cache;
vk::UniqueCommandPool command_pool; vk::UniqueCommandPool command_pool;
std::vector<Frame> present_frames; std::vector<Frame> present_frames;
std::queue<Frame*> free_queue; std::queue<Frame*> free_queue;

View File

@ -3,11 +3,10 @@
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
#include "common/alignment.h" #include "common/alignment.h"
#include "core/memory.h" #include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/renderer_vulkan/vk_compute_pipeline.h" #include "video_core/renderer_vulkan/vk_compute_pipeline.h"
#include "video_core/renderer_vulkan/vk_instance.h" #include "video_core/renderer_vulkan/vk_instance.h"
#include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
#include "video_core/texture_cache/texture_cache.h" #include "video_core/texture_cache/texture_cache.h"
namespace Vulkan { namespace Vulkan {
@ -51,6 +50,12 @@ ComputePipeline::ComputePipeline(const Instance& instance_, Scheduler& scheduler
}); });
} }
const vk::PushConstantRange push_constants = {
.stageFlags = vk::ShaderStageFlagBits::eCompute,
.offset = 0,
.size = sizeof(Shader::PushData),
};
const vk::DescriptorSetLayoutCreateInfo desc_layout_ci = { const vk::DescriptorSetLayoutCreateInfo desc_layout_ci = {
.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR, .flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR,
.bindingCount = static_cast<u32>(bindings.size()), .bindingCount = static_cast<u32>(bindings.size()),
@ -62,8 +67,8 @@ ComputePipeline::ComputePipeline(const Instance& instance_, Scheduler& scheduler
const vk::PipelineLayoutCreateInfo layout_info = { const vk::PipelineLayoutCreateInfo layout_info = {
.setLayoutCount = 1U, .setLayoutCount = 1U,
.pSetLayouts = &set_layout, .pSetLayouts = &set_layout,
.pushConstantRangeCount = 0, .pushConstantRangeCount = 1U,
.pPushConstantRanges = nullptr, .pPushConstantRanges = &push_constants,
}; };
pipeline_layout = instance.GetDevice().createPipelineLayoutUnique(layout_info); pipeline_layout = instance.GetDevice().createPipelineLayoutUnique(layout_info);
@ -82,35 +87,18 @@ ComputePipeline::ComputePipeline(const Instance& instance_, Scheduler& scheduler
ComputePipeline::~ComputePipeline() = default; ComputePipeline::~ComputePipeline() = default;
bool ComputePipeline::BindResources(Core::MemoryManager* memory, StreamBuffer& staging, bool ComputePipeline::BindResources(VideoCore::BufferCache& buffer_cache,
VideoCore::TextureCache& texture_cache) const { VideoCore::TextureCache& texture_cache) const {
// Bind resource buffers and textures. // Bind resource buffers and textures.
boost::container::static_vector<vk::DescriptorBufferInfo, 16> buffer_infos; boost::container::static_vector<vk::DescriptorBufferInfo, 16> buffer_infos;
boost::container::static_vector<vk::DescriptorImageInfo, 16> image_infos; boost::container::static_vector<vk::DescriptorImageInfo, 16> image_infos;
boost::container::small_vector<vk::WriteDescriptorSet, 16> set_writes; boost::container::small_vector<vk::WriteDescriptorSet, 16> set_writes;
Shader::PushData push_data{};
u32 binding{}; u32 binding{};
for (const auto& buffer : info.buffers) { for (u32 i = 0; const auto& buffer : info.buffers) {
const auto vsharp = buffer.GetVsharp(info); const auto vsharp = buffer.GetVsharp(info);
const u32 size = vsharp.GetSize();
const VAddr address = vsharp.base_address; const VAddr address = vsharp.base_address;
if (buffer.is_storage) {
texture_cache.OnCpuWrite(address);
}
const u32 offset = staging.Copy(address, size,
buffer.is_storage ? instance.StorageMinAlignment()
: instance.UniformMinAlignment());
buffer_infos.emplace_back(staging.Handle(), offset, size);
set_writes.push_back({
.dstSet = VK_NULL_HANDLE,
.dstBinding = binding++,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = buffer.is_storage ? vk::DescriptorType::eStorageBuffer
: vk::DescriptorType::eUniformBuffer,
.pBufferInfo = &buffer_infos.back(),
});
// Most of the time when a metadata is updated with a shader it gets cleared. It means we // Most of the time when a metadata is updated with a shader it gets cleared. It means we
// can skip the whole dispatch and update the tracked state instead. Also, it is not // can skip the whole dispatch and update the tracked state instead. Also, it is not
// intended to be consumed and in such rare cases (e.g. HTile introspection, CRAA) we will // intended to be consumed and in such rare cases (e.g. HTile introspection, CRAA) we will
@ -125,6 +113,31 @@ bool ComputePipeline::BindResources(Core::MemoryManager* memory, StreamBuffer& s
LOG_WARNING(Render_Vulkan, "Unexpected metadata read by a CS shader (buffer)"); LOG_WARNING(Render_Vulkan, "Unexpected metadata read by a CS shader (buffer)");
} }
} }
const u32 size = vsharp.GetSize();
if (buffer.is_written) {
texture_cache.InvalidateMemory(address, size);
}
const u32 alignment =
buffer.is_storage ? instance.StorageMinAlignment() : instance.UniformMinAlignment();
const auto [vk_buffer, offset] =
buffer_cache.ObtainBuffer(address, size, buffer.is_written);
const u32 offset_aligned = Common::AlignDown(offset, alignment);
const u32 adjust = offset - offset_aligned;
if (adjust != 0) {
ASSERT(adjust % 4 == 0);
push_data.AddOffset(i, adjust);
}
buffer_infos.emplace_back(vk_buffer->Handle(), offset_aligned, size + adjust);
set_writes.push_back({
.dstSet = VK_NULL_HANDLE,
.dstBinding = binding++,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = buffer.is_storage ? vk::DescriptorType::eStorageBuffer
: vk::DescriptorType::eUniformBuffer,
.pBufferInfo = &buffer_infos.back(),
});
i++;
} }
for (const auto& image_desc : info.images) { for (const auto& image_desc : info.images) {
@ -168,6 +181,8 @@ bool ComputePipeline::BindResources(Core::MemoryManager* memory, StreamBuffer& s
} }
const auto cmdbuf = scheduler.CommandBuffer(); const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.pushConstants(*pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0u, sizeof(push_data),
&push_data);
cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, *pipeline_layout, 0, set_writes); cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, *pipeline_layout, 0, set_writes);
return true; return true;
} }

View File

@ -6,19 +6,15 @@
#include "shader_recompiler/runtime_info.h" #include "shader_recompiler/runtime_info.h"
#include "video_core/renderer_vulkan/vk_common.h" #include "video_core/renderer_vulkan/vk_common.h"
namespace Core {
class MemoryManager;
}
namespace VideoCore { namespace VideoCore {
class BufferCache;
class TextureCache; class TextureCache;
} } // namespace VideoCore
namespace Vulkan { namespace Vulkan {
class Instance; class Instance;
class Scheduler; class Scheduler;
class StreamBuffer;
class ComputePipeline { class ComputePipeline {
public: public:
@ -31,7 +27,7 @@ public:
return *pipeline; return *pipeline;
} }
bool BindResources(Core::MemoryManager* memory, StreamBuffer& staging, bool BindResources(VideoCore::BufferCache& buffer_cache,
VideoCore::TextureCache& texture_cache) const; VideoCore::TextureCache& texture_cache) const;
private: private:

View File

@ -5,13 +5,13 @@
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
#include <boost/container/static_vector.hpp> #include <boost/container/static_vector.hpp>
#include "common/alignment.h"
#include "common/assert.h" #include "common/assert.h"
#include "core/memory.h"
#include "video_core/amdgpu/resource.h" #include "video_core/amdgpu/resource.h"
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/renderer_vulkan/vk_graphics_pipeline.h" #include "video_core/renderer_vulkan/vk_graphics_pipeline.h"
#include "video_core/renderer_vulkan/vk_instance.h" #include "video_core/renderer_vulkan/vk_instance.h"
#include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
#include "video_core/texture_cache/texture_cache.h" #include "video_core/texture_cache/texture_cache.h"
namespace Vulkan { namespace Vulkan {
@ -32,9 +32,9 @@ GraphicsPipeline::GraphicsPipeline(const Instance& instance_, Scheduler& schedul
BuildDescSetLayout(); BuildDescSetLayout();
const vk::PushConstantRange push_constants = { const vk::PushConstantRange push_constants = {
.stageFlags = vk::ShaderStageFlagBits::eVertex, .stageFlags = vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment,
.offset = 0, .offset = 0,
.size = 2 * sizeof(u32), .size = sizeof(Shader::PushData),
}; };
const vk::DescriptorSetLayout set_layout = *desc_layout; const vk::DescriptorSetLayout set_layout = *desc_layout;
@ -328,25 +328,43 @@ void GraphicsPipeline::BuildDescSetLayout() {
desc_layout = instance.GetDevice().createDescriptorSetLayoutUnique(desc_layout_ci); desc_layout = instance.GetDevice().createDescriptorSetLayoutUnique(desc_layout_ci);
} }
void GraphicsPipeline::BindResources(Core::MemoryManager* memory, StreamBuffer& staging, void GraphicsPipeline::BindResources(const Liverpool::Regs& regs,
VideoCore::BufferCache& buffer_cache,
VideoCore::TextureCache& texture_cache) const { VideoCore::TextureCache& texture_cache) const {
BindVertexBuffers(staging);
// Bind resource buffers and textures. // Bind resource buffers and textures.
boost::container::static_vector<vk::DescriptorBufferInfo, 16> buffer_infos; boost::container::static_vector<vk::DescriptorBufferInfo, 16> buffer_infos;
boost::container::static_vector<vk::DescriptorImageInfo, 32> image_infos; boost::container::static_vector<vk::DescriptorImageInfo, 32> image_infos;
boost::container::small_vector<vk::WriteDescriptorSet, 16> set_writes; boost::container::small_vector<vk::WriteDescriptorSet, 16> set_writes;
Shader::PushData push_data{};
u32 binding{}; u32 binding{};
for (const auto& stage : stages) { for (const auto& stage : stages) {
for (const auto& buffer : stage.buffers) { if (stage.uses_step_rates) {
push_data.step0 = regs.vgt_instance_step_rate_0;
push_data.step1 = regs.vgt_instance_step_rate_1;
}
for (u32 i = 0; const auto& buffer : stage.buffers) {
const auto vsharp = buffer.GetVsharp(stage); const auto vsharp = buffer.GetVsharp(stage);
if (vsharp) {
const VAddr address = vsharp.base_address; const VAddr address = vsharp.base_address;
if (texture_cache.IsMeta(address)) {
LOG_WARNING(Render_Vulkan, "Unexpected metadata read by a PS shader (buffer)");
}
const u32 size = vsharp.GetSize(); const u32 size = vsharp.GetSize();
const u32 offset = staging.Copy(address, size, const u32 alignment = buffer.is_storage ? instance.StorageMinAlignment()
buffer.is_storage ? instance.StorageMinAlignment() : instance.UniformMinAlignment();
: instance.UniformMinAlignment()); const auto [vk_buffer, offset] =
buffer_infos.emplace_back(staging.Handle(), offset, size); buffer_cache.ObtainBuffer(address, size, buffer.is_written);
const u32 offset_aligned = Common::AlignDown(offset, alignment);
const u32 adjust = offset - offset_aligned;
if (adjust != 0) {
ASSERT(adjust % 4 == 0);
push_data.AddOffset(i, adjust);
}
buffer_infos.emplace_back(vk_buffer->Handle(), offset_aligned, size + adjust);
} else {
buffer_infos.emplace_back(VK_NULL_HANDLE, 0, VK_WHOLE_SIZE);
}
set_writes.push_back({ set_writes.push_back({
.dstSet = VK_NULL_HANDLE, .dstSet = VK_NULL_HANDLE,
.dstBinding = binding++, .dstBinding = binding++,
@ -356,10 +374,7 @@ void GraphicsPipeline::BindResources(Core::MemoryManager* memory, StreamBuffer&
: vk::DescriptorType::eUniformBuffer, : vk::DescriptorType::eUniformBuffer,
.pBufferInfo = &buffer_infos.back(), .pBufferInfo = &buffer_infos.back(),
}); });
i++;
if (texture_cache.IsMeta(address)) {
LOG_WARNING(Render_Vulkan, "Unexpected metadata read by a PS shader (buffer)");
}
} }
boost::container::static_vector<AmdGpu::Image, 16> tsharps; boost::container::static_vector<AmdGpu::Image, 16> tsharps;
@ -406,86 +421,15 @@ void GraphicsPipeline::BindResources(Core::MemoryManager* memory, StreamBuffer&
} }
} }
if (!set_writes.empty()) {
const auto cmdbuf = scheduler.CommandBuffer(); const auto cmdbuf = scheduler.CommandBuffer();
if (!set_writes.empty()) {
cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eGraphics, *pipeline_layout, 0, cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eGraphics, *pipeline_layout, 0,
set_writes); set_writes);
} }
} cmdbuf.pushConstants(*pipeline_layout,
vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment, 0U,
void GraphicsPipeline::BindVertexBuffers(StreamBuffer& staging) const { sizeof(push_data), &push_data);
const auto& vs_info = stages[u32(Shader::Stage::Vertex)]; cmdbuf.bindPipeline(vk::PipelineBindPoint::eGraphics, Handle());
if (vs_info.vs_inputs.empty()) {
return;
}
std::array<vk::Buffer, MaxVertexBufferCount> host_buffers;
std::array<vk::DeviceSize, MaxVertexBufferCount> host_offsets;
boost::container::static_vector<AmdGpu::Buffer, MaxVertexBufferCount> guest_buffers;
struct BufferRange {
VAddr base_address;
VAddr end_address;
u64 offset; // offset in the mapped memory
size_t GetSize() const {
return end_address - base_address;
}
};
// Calculate buffers memory overlaps
boost::container::static_vector<BufferRange, MaxVertexBufferCount> ranges{};
for (const auto& input : vs_info.vs_inputs) {
if (input.instance_step_rate == Shader::Info::VsInput::InstanceIdType::OverStepRate0 ||
input.instance_step_rate == Shader::Info::VsInput::InstanceIdType::OverStepRate1) {
continue;
}
const auto& buffer = vs_info.ReadUd<AmdGpu::Buffer>(input.sgpr_base, input.dword_offset);
if (buffer.GetSize() == 0) {
continue;
}
guest_buffers.emplace_back(buffer);
ranges.emplace_back(buffer.base_address, buffer.base_address + buffer.GetSize());
}
std::ranges::sort(ranges, [](const BufferRange& lhv, const BufferRange& rhv) {
return lhv.base_address < rhv.base_address;
});
boost::container::static_vector<BufferRange, MaxVertexBufferCount> ranges_merged{ranges[0]};
for (auto range : ranges) {
auto& prev_range = ranges_merged.back();
if (prev_range.end_address < range.base_address) {
ranges_merged.emplace_back(range);
} else {
prev_range.end_address = std::max(prev_range.end_address, range.end_address);
}
}
// Map buffers
for (auto& range : ranges_merged) {
range.offset = staging.Copy(range.base_address, range.GetSize(), 4);
}
// Bind vertex buffers
const size_t num_buffers = guest_buffers.size();
for (u32 i = 0; i < num_buffers; ++i) {
const auto& buffer = guest_buffers[i];
const auto& host_buffer = std::ranges::find_if(
ranges_merged.cbegin(), ranges_merged.cend(), [&](const BufferRange& range) {
return (buffer.base_address >= range.base_address &&
buffer.base_address < range.end_address);
});
assert(host_buffer != ranges_merged.cend());
host_buffers[i] = staging.Handle();
host_offsets[i] = host_buffer->offset + buffer.base_address - host_buffer->base_address;
}
if (num_buffers > 0) {
const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.bindVertexBuffers(0, num_buffers, host_buffers.data(), host_offsets.data());
}
} }
} // namespace Vulkan } // namespace Vulkan

View File

@ -7,13 +7,10 @@
#include "video_core/renderer_vulkan/liverpool_to_vk.h" #include "video_core/renderer_vulkan/liverpool_to_vk.h"
#include "video_core/renderer_vulkan/vk_common.h" #include "video_core/renderer_vulkan/vk_common.h"
namespace Core {
class MemoryManager;
}
namespace VideoCore { namespace VideoCore {
class BufferCache;
class TextureCache; class TextureCache;
} } // namespace VideoCore
namespace Vulkan { namespace Vulkan {
@ -22,7 +19,6 @@ static constexpr u32 MaxShaderStages = 5;
class Instance; class Instance;
class Scheduler; class Scheduler;
class StreamBuffer;
using Liverpool = AmdGpu::Liverpool; using Liverpool = AmdGpu::Liverpool;
@ -64,7 +60,7 @@ public:
std::array<vk::ShaderModule, MaxShaderStages> modules); std::array<vk::ShaderModule, MaxShaderStages> modules);
~GraphicsPipeline(); ~GraphicsPipeline();
void BindResources(Core::MemoryManager* memory, StreamBuffer& staging, void BindResources(const Liverpool::Regs& regs, VideoCore::BufferCache& buffer_cache,
VideoCore::TextureCache& texture_cache) const; VideoCore::TextureCache& texture_cache) const;
vk::Pipeline Handle() const noexcept { vk::Pipeline Handle() const noexcept {
@ -75,6 +71,10 @@ public:
return *pipeline_layout; return *pipeline_layout;
} }
const Shader::Info& GetStage(Shader::Stage stage) const noexcept {
return stages[u32(stage)];
}
bool IsEmbeddedVs() const noexcept { bool IsEmbeddedVs() const noexcept {
static constexpr size_t EmbeddedVsHash = 0x9b2da5cf47f8c29f; static constexpr size_t EmbeddedVsHash = 0x9b2da5cf47f8c29f;
return key.stage_hashes[u32(Shader::Stage::Vertex)] == EmbeddedVsHash; return key.stage_hashes[u32(Shader::Stage::Vertex)] == EmbeddedVsHash;
@ -90,7 +90,6 @@ public:
private: private:
void BuildDescSetLayout(); void BuildDescSetLayout();
void BindVertexBuffers(StreamBuffer& staging) const;
private: private:
const Instance& instance; const Instance& instance;

View File

@ -204,7 +204,8 @@ bool Instance::CreateDevice() {
// The next two extensions are required to be available together in order to support write masks // The next two extensions are required to be available together in order to support write masks
color_write_en = add_extension(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME); color_write_en = add_extension(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
color_write_en &= add_extension(VK_EXT_EXTENDED_DYNAMIC_STATE_3_EXTENSION_NAME); color_write_en &= add_extension(VK_EXT_EXTENDED_DYNAMIC_STATE_3_EXTENSION_NAME);
const auto calibrated_timestamps = add_extension(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME); const bool calibrated_timestamps = add_extension(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME);
const bool robustness = add_extension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
// These extensions are promoted by Vulkan 1.3, but for greater compatibility we use Vulkan 1.2 // These extensions are promoted by Vulkan 1.3, but for greater compatibility we use Vulkan 1.2
// with extensions. // with extensions.
@ -303,12 +304,19 @@ bool Instance::CreateDevice() {
.workgroupMemoryExplicitLayoutScalarBlockLayout = true, .workgroupMemoryExplicitLayoutScalarBlockLayout = true,
.workgroupMemoryExplicitLayout8BitAccess = true, .workgroupMemoryExplicitLayout8BitAccess = true,
.workgroupMemoryExplicitLayout16BitAccess = true, .workgroupMemoryExplicitLayout16BitAccess = true,
}}; },
vk::PhysicalDeviceRobustness2FeaturesEXT{
.nullDescriptor = true,
},
};
if (!color_write_en) { if (!color_write_en) {
device_chain.unlink<vk::PhysicalDeviceColorWriteEnableFeaturesEXT>(); device_chain.unlink<vk::PhysicalDeviceColorWriteEnableFeaturesEXT>();
device_chain.unlink<vk::PhysicalDeviceExtendedDynamicState3FeaturesEXT>(); device_chain.unlink<vk::PhysicalDeviceExtendedDynamicState3FeaturesEXT>();
} }
if (!robustness) {
device_chain.unlink<vk::PhysicalDeviceRobustness2FeaturesEXT>();
}
try { try {
device = physical_device.createDeviceUnique(device_chain.get()); device = physical_device.createDeviceUnique(device_chain.get());

View File

@ -5,7 +5,6 @@
#include <tsl/robin_map.h> #include <tsl/robin_map.h>
#include "shader_recompiler/ir/basic_block.h" #include "shader_recompiler/ir/basic_block.h"
#include "shader_recompiler/object_pool.h"
#include "shader_recompiler/profile.h" #include "shader_recompiler/profile.h"
#include "video_core/renderer_vulkan/vk_compute_pipeline.h" #include "video_core/renderer_vulkan/vk_compute_pipeline.h"
#include "video_core/renderer_vulkan/vk_graphics_pipeline.h" #include "video_core/renderer_vulkan/vk_graphics_pipeline.h"
@ -51,8 +50,8 @@ private:
Shader::Profile profile{}; Shader::Profile profile{};
GraphicsPipelineKey graphics_key{}; GraphicsPipelineKey graphics_key{};
u64 compute_key{}; u64 compute_key{};
Shader::ObjectPool<Shader::IR::Inst> inst_pool; Common::ObjectPool<Shader::IR::Inst> inst_pool;
Shader::ObjectPool<Shader::IR::Block> block_pool; Common::ObjectPool<Shader::IR::Block> block_pool;
}; };
} // namespace Vulkan } // namespace Vulkan

View File

@ -13,22 +13,17 @@
namespace Vulkan { namespace Vulkan {
static constexpr vk::BufferUsageFlags VertexIndexFlags =
vk::BufferUsageFlagBits::eVertexBuffer | vk::BufferUsageFlagBits::eIndexBuffer |
vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eUniformBuffer |
vk::BufferUsageFlagBits::eStorageBuffer;
Rasterizer::Rasterizer(const Instance& instance_, Scheduler& scheduler_, Rasterizer::Rasterizer(const Instance& instance_, Scheduler& scheduler_,
VideoCore::TextureCache& texture_cache_, AmdGpu::Liverpool* liverpool_) AmdGpu::Liverpool* liverpool_)
: instance{instance_}, scheduler{scheduler_}, texture_cache{texture_cache_}, : instance{instance_}, scheduler{scheduler_}, page_manager{this},
liverpool{liverpool_}, memory{Core::Memory::Instance()}, buffer_cache{instance, scheduler, liverpool_, page_manager},
pipeline_cache{instance, scheduler, liverpool}, texture_cache{instance, scheduler, buffer_cache, page_manager}, liverpool{liverpool_},
vertex_index_buffer{instance, scheduler, VertexIndexFlags, 2_GB, BufferType::Upload} { memory{Core::Memory::Instance()}, pipeline_cache{instance, scheduler, liverpool} {
if (!Config::nullGpu()) { if (!Config::nullGpu()) {
liverpool->BindRasterizer(this); liverpool->BindRasterizer(this);
} }
memory->SetRasterizer(this);
memory->SetInstance(&instance); wfi_event = instance.GetDevice().createEventUnique({});
} }
Rasterizer::~Rasterizer() = default; Rasterizer::~Rasterizer() = default;
@ -38,29 +33,24 @@ void Rasterizer::Draw(bool is_indexed, u32 index_offset) {
const auto cmdbuf = scheduler.CommandBuffer(); const auto cmdbuf = scheduler.CommandBuffer();
const auto& regs = liverpool->regs; const auto& regs = liverpool->regs;
const u32 num_indices = SetupIndexBuffer(is_indexed, index_offset);
const GraphicsPipeline* pipeline = pipeline_cache.GetGraphicsPipeline(); const GraphicsPipeline* pipeline = pipeline_cache.GetGraphicsPipeline();
if (!pipeline) { if (!pipeline) {
return; return;
} }
try { try {
pipeline->BindResources(memory, vertex_index_buffer, texture_cache); pipeline->BindResources(regs, buffer_cache, texture_cache);
} catch (...) { } catch (...) {
UNREACHABLE(); UNREACHABLE();
} }
const auto& vs_info = pipeline->GetStage(Shader::Stage::Vertex);
buffer_cache.BindVertexBuffers(vs_info);
const u32 num_indices = buffer_cache.BindIndexBuffer(is_indexed, index_offset);
BeginRendering(); BeginRendering();
UpdateDynamicState(*pipeline); UpdateDynamicState(*pipeline);
cmdbuf.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline->Handle());
const u32 step_rates[] = {
regs.vgt_instance_step_rate_0,
regs.vgt_instance_step_rate_1,
};
cmdbuf.pushConstants(pipeline->GetLayout(), vk::ShaderStageFlagBits::eVertex, 0u,
sizeof(step_rates), &step_rates);
if (is_indexed) { if (is_indexed) {
cmdbuf.drawIndexed(num_indices, regs.num_instances.NumInstances(), 0, 0, 0); cmdbuf.drawIndexed(num_indices, regs.num_instances.NumInstances(), 0, 0, 0);
} else { } else {
@ -82,8 +72,7 @@ void Rasterizer::DispatchDirect() {
} }
try { try {
const auto has_resources = const auto has_resources = pipeline->BindResources(buffer_cache, texture_cache);
pipeline->BindResources(memory, vertex_index_buffer, texture_cache);
if (!has_resources) { if (!has_resources) {
return; return;
} }
@ -131,7 +120,7 @@ void Rasterizer::BeginRendering() {
state.color_images[state.num_color_attachments] = image.image; state.color_images[state.num_color_attachments] = image.image;
state.color_attachments[state.num_color_attachments++] = { state.color_attachments[state.num_color_attachments++] = {
.imageView = *image_view.image_view, .imageView = *image_view.image_view,
.imageLayout = vk::ImageLayout::eGeneral, .imageLayout = vk::ImageLayout::eColorAttachmentOptimal,
.loadOp = is_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad, .loadOp = is_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad,
.storeOp = vk::AttachmentStoreOp::eStore, .storeOp = vk::AttachmentStoreOp::eStore,
.clearValue = .clearValue =
@ -168,45 +157,19 @@ void Rasterizer::BeginRendering() {
scheduler.BeginRendering(state); scheduler.BeginRendering(state);
} }
u32 Rasterizer::SetupIndexBuffer(bool& is_indexed, u32 index_offset) { void Rasterizer::InvalidateMemory(VAddr addr, u64 size) {
// Emulate QuadList primitive type with CPU made index buffer. buffer_cache.InvalidateMemory(addr, size);
const auto& regs = liverpool->regs; texture_cache.InvalidateMemory(addr, size);
if (liverpool->regs.primitive_type == Liverpool::PrimitiveType::QuadList) { }
// ASSERT_MSG(!is_indexed, "Using QuadList primitive with indexed draw");
is_indexed = true;
// Emit indices. void Rasterizer::MapMemory(VAddr addr, u64 size) {
const u32 index_size = 3 * regs.num_indices; page_manager.OnGpuMap(addr, size);
const auto [data, offset, _] = vertex_index_buffer.Map(index_size); }
LiverpoolToVK::EmitQuadToTriangleListIndices(data, regs.num_indices);
vertex_index_buffer.Commit(index_size);
// Bind index buffer. void Rasterizer::UnmapMemory(VAddr addr, u64 size) {
const auto cmdbuf = scheduler.CommandBuffer(); buffer_cache.InvalidateMemory(addr, size);
cmdbuf.bindIndexBuffer(vertex_index_buffer.Handle(), offset, vk::IndexType::eUint16); texture_cache.UnmapMemory(addr, size);
return index_size / sizeof(u16); page_manager.OnGpuUnmap(addr, size);
}
if (!is_indexed) {
return regs.num_indices;
}
// Figure out index type and size.
const bool is_index16 = regs.index_buffer_type.index_type == Liverpool::IndexType::Index16;
const vk::IndexType index_type = is_index16 ? vk::IndexType::eUint16 : vk::IndexType::eUint32;
const u32 index_size = is_index16 ? sizeof(u16) : sizeof(u32);
// Upload index data to stream buffer.
const auto index_address = regs.index_base_address.Address<const void*>();
const u32 index_buffer_size = (index_offset + regs.num_indices) * index_size;
const auto [data, offset, _] = vertex_index_buffer.Map(index_buffer_size);
std::memcpy(data, index_address, index_buffer_size);
vertex_index_buffer.Commit(index_buffer_size);
// Bind index buffer.
const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.bindIndexBuffer(vertex_index_buffer.Handle(), offset + index_offset * index_size,
index_type);
return regs.num_indices;
} }
void Rasterizer::UpdateDynamicState(const GraphicsPipeline& pipeline) { void Rasterizer::UpdateDynamicState(const GraphicsPipeline& pipeline) {

View File

@ -3,8 +3,10 @@
#pragma once #pragma once
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/page_manager.h"
#include "video_core/renderer_vulkan/vk_pipeline_cache.h" #include "video_core/renderer_vulkan/vk_pipeline_cache.h"
#include "video_core/renderer_vulkan/vk_stream_buffer.h" #include "video_core/texture_cache/texture_cache.h"
namespace AmdGpu { namespace AmdGpu {
struct Liverpool; struct Liverpool;
@ -14,10 +16,6 @@ namespace Core {
class MemoryManager; class MemoryManager;
} }
namespace VideoCore {
class TextureCache;
}
namespace Vulkan { namespace Vulkan {
class Scheduler; class Scheduler;
@ -26,9 +24,13 @@ class GraphicsPipeline;
class Rasterizer { class Rasterizer {
public: public:
explicit Rasterizer(const Instance& instance, Scheduler& scheduler, explicit Rasterizer(const Instance& instance, Scheduler& scheduler,
VideoCore::TextureCache& texture_cache, AmdGpu::Liverpool* liverpool); AmdGpu::Liverpool* liverpool);
~Rasterizer(); ~Rasterizer();
[[nodiscard]] VideoCore::TextureCache& GetTextureCache() noexcept {
return texture_cache;
}
void Draw(bool is_indexed, u32 index_offset = 0); void Draw(bool is_indexed, u32 index_offset = 0);
void DispatchDirect(); void DispatchDirect();
@ -36,12 +38,13 @@ public:
void ScopeMarkerBegin(const std::string& str); void ScopeMarkerBegin(const std::string& str);
void ScopeMarkerEnd(); void ScopeMarkerEnd();
void InvalidateMemory(VAddr addr, u64 size);
void MapMemory(VAddr addr, u64 size);
void UnmapMemory(VAddr addr, u64 size);
u64 Flush(); u64 Flush();
private: private:
u32 SetupIndexBuffer(bool& is_indexed, u32 index_offset);
void MapMemory(VAddr addr, size_t size);
void BeginRendering(); void BeginRendering();
void UpdateDynamicState(const GraphicsPipeline& pipeline); void UpdateDynamicState(const GraphicsPipeline& pipeline);
@ -51,11 +54,13 @@ private:
private: private:
const Instance& instance; const Instance& instance;
Scheduler& scheduler; Scheduler& scheduler;
VideoCore::TextureCache& texture_cache; VideoCore::PageManager page_manager;
VideoCore::BufferCache buffer_cache;
VideoCore::TextureCache texture_cache;
AmdGpu::Liverpool* liverpool; AmdGpu::Liverpool* liverpool;
Core::MemoryManager* memory; Core::MemoryManager* memory;
PipelineCache pipeline_cache; PipelineCache pipeline_cache;
StreamBuffer vertex_index_buffer; vk::UniqueEvent wfi_event;
}; };
} // namespace Vulkan } // namespace Vulkan

View File

@ -6,6 +6,7 @@
#include <condition_variable> #include <condition_variable>
#include <boost/container/static_vector.hpp> #include <boost/container/static_vector.hpp>
#include "common/types.h" #include "common/types.h"
#include "common/unique_function.h"
#include "video_core/renderer_vulkan/vk_master_semaphore.h" #include "video_core/renderer_vulkan/vk_master_semaphore.h"
#include "video_core/renderer_vulkan/vk_resource_pool.h" #include "video_core/renderer_vulkan/vk_resource_pool.h"
@ -97,8 +98,8 @@ public:
} }
/// Defers an operation until the gpu has reached the current cpu tick. /// Defers an operation until the gpu has reached the current cpu tick.
void DeferOperation(auto&& func) { void DeferOperation(Common::UniqueFunction<void>&& func) {
pending_ops.emplace(func, CurrentTick()); pending_ops.emplace(std::move(func), CurrentTick());
} }
static std::mutex submit_mutex; static std::mutex submit_mutex;
@ -115,7 +116,7 @@ private:
vk::CommandBuffer current_cmdbuf; vk::CommandBuffer current_cmdbuf;
std::condition_variable_any event_cv; std::condition_variable_any event_cv;
struct PendingOp { struct PendingOp {
std::function<void()> callback; Common::UniqueFunction<void> callback;
u64 gpu_tick; u64 gpu_tick;
}; };
std::queue<PendingOp> pending_ops; std::queue<PendingOp> pending_ops;

View File

@ -1,241 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include "common/alignment.h"
#include "common/assert.h"
#include "video_core/renderer_vulkan/vk_instance.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
namespace Vulkan {
namespace {
std::string_view BufferTypeName(BufferType type) {
switch (type) {
case BufferType::Upload:
return "Upload";
case BufferType::Download:
return "Download";
case BufferType::Stream:
return "Stream";
default:
return "Invalid";
}
}
vk::MemoryPropertyFlags MakePropertyFlags(BufferType type) {
switch (type) {
case BufferType::Upload:
return vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent;
case BufferType::Download:
return vk::MemoryPropertyFlagBits::eHostVisible |
vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached;
case BufferType::Stream:
return vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible |
vk::MemoryPropertyFlagBits::eHostCoherent;
default:
UNREACHABLE_MSG("Unknown buffer type {}", static_cast<u32>(type));
return vk::MemoryPropertyFlagBits::eHostVisible;
}
}
static std::optional<u32> FindMemoryType(const vk::PhysicalDeviceMemoryProperties& properties,
vk::MemoryPropertyFlags wanted) {
for (u32 i = 0; i < properties.memoryTypeCount; ++i) {
const auto flags = properties.memoryTypes[i].propertyFlags;
if ((flags & wanted) == wanted) {
return i;
}
}
return std::nullopt;
}
/// Get the preferred host visible memory type.
u32 GetMemoryType(const vk::PhysicalDeviceMemoryProperties& properties, BufferType type) {
vk::MemoryPropertyFlags flags = MakePropertyFlags(type);
std::optional preferred_type = FindMemoryType(properties, flags);
constexpr std::array remove_flags = {
vk::MemoryPropertyFlagBits::eHostCached,
vk::MemoryPropertyFlagBits::eHostCoherent,
};
for (u32 i = 0; i < remove_flags.size() && !preferred_type; i++) {
flags &= ~remove_flags[i];
preferred_type = FindMemoryType(properties, flags);
}
ASSERT_MSG(preferred_type, "No suitable memory type found");
return preferred_type.value();
}
constexpr u64 WATCHES_INITIAL_RESERVE = 0x4000;
constexpr u64 WATCHES_RESERVE_CHUNK = 0x1000;
} // Anonymous namespace
StreamBuffer::StreamBuffer(const Instance& instance_, Scheduler& scheduler_,
vk::BufferUsageFlags usage_, u64 size, BufferType type_)
: instance{instance_}, scheduler{scheduler_}, device{instance.GetDevice()},
stream_buffer_size{size}, usage{usage_}, type{type_} {
CreateBuffers(size);
ReserveWatches(current_watches, WATCHES_INITIAL_RESERVE);
ReserveWatches(previous_watches, WATCHES_INITIAL_RESERVE);
}
StreamBuffer::~StreamBuffer() {
device.unmapMemory(memory);
device.destroyBuffer(buffer);
device.freeMemory(memory);
}
std::tuple<u8*, u64, bool> StreamBuffer::Map(u64 size, u64 alignment) {
if (!is_coherent && type == BufferType::Stream) {
size = Common::AlignUp(size, instance.NonCoherentAtomSize());
}
ASSERT(size <= stream_buffer_size);
mapped_size = size;
if (alignment > 0) {
offset = Common::AlignUp(offset, alignment);
}
bool invalidate{false};
if (offset + size > stream_buffer_size) {
// The buffer would overflow, save the amount of used watches and reset the state.
invalidate = true;
invalidation_mark = current_watch_cursor;
current_watch_cursor = 0;
offset = 0;
// Swap watches and reset waiting cursors.
std::swap(previous_watches, current_watches);
wait_cursor = 0;
wait_bound = 0;
}
const u64 mapped_upper_bound = offset + size;
WaitPendingOperations(mapped_upper_bound);
return std::make_tuple(mapped + offset, offset, invalidate);
}
void StreamBuffer::Commit(u64 size) {
if (!is_coherent && type == BufferType::Stream) {
size = Common::AlignUp(size, instance.NonCoherentAtomSize());
}
ASSERT_MSG(size <= mapped_size, "Reserved size {} is too small compared to {}", mapped_size,
size);
const vk::MappedMemoryRange range = {
.memory = memory,
.offset = offset,
.size = size,
};
if (!is_coherent && type == BufferType::Download) {
device.invalidateMappedMemoryRanges(range);
} else if (!is_coherent) {
device.flushMappedMemoryRanges(range);
}
offset += size;
if (current_watch_cursor + 1 >= current_watches.size()) {
// Ensure that there are enough watches.
ReserveWatches(current_watches, WATCHES_RESERVE_CHUNK);
}
auto& watch = current_watches[current_watch_cursor++];
watch.upper_bound = offset;
watch.tick = scheduler.CurrentTick();
}
void StreamBuffer::CreateBuffers(u64 prefered_size) {
const vk::Device device = instance.GetDevice();
const auto memory_properties = instance.GetPhysicalDevice().getMemoryProperties();
const u32 preferred_type = GetMemoryType(memory_properties, type);
const vk::MemoryType mem_type = memory_properties.memoryTypes[preferred_type];
const u32 preferred_heap = mem_type.heapIndex;
is_coherent =
static_cast<bool>(mem_type.propertyFlags & vk::MemoryPropertyFlagBits::eHostCoherent);
// Substract from the preferred heap size some bytes to avoid getting out of memory.
const vk::DeviceSize heap_size = memory_properties.memoryHeaps[preferred_heap].size;
// As per DXVK's example, using `heap_size / 2`
const vk::DeviceSize allocable_size = heap_size / 2;
buffer = device.createBuffer({
.size = std::min(prefered_size, allocable_size),
.usage = usage,
});
const auto requirements_chain =
device
.getBufferMemoryRequirements2<vk::MemoryRequirements2, vk::MemoryDedicatedRequirements>(
{.buffer = buffer});
const auto& requirements = requirements_chain.get<vk::MemoryRequirements2>();
const auto& dedicated_requirements = requirements_chain.get<vk::MemoryDedicatedRequirements>();
stream_buffer_size = static_cast<u64>(requirements.memoryRequirements.size);
LOG_INFO(Render_Vulkan, "Creating {} buffer with size {} KiB with flags {}",
BufferTypeName(type), stream_buffer_size / 1024,
vk::to_string(mem_type.propertyFlags));
if (dedicated_requirements.prefersDedicatedAllocation) {
vk::StructureChain<vk::MemoryAllocateInfo, vk::MemoryDedicatedAllocateInfo> alloc_chain =
{};
auto& alloc_info = alloc_chain.get<vk::MemoryAllocateInfo>();
alloc_info.allocationSize = requirements.memoryRequirements.size;
alloc_info.memoryTypeIndex = preferred_type;
auto& dedicated_alloc_info = alloc_chain.get<vk::MemoryDedicatedAllocateInfo>();
dedicated_alloc_info.buffer = buffer;
memory = device.allocateMemory(alloc_chain.get());
} else {
memory = device.allocateMemory({
.allocationSize = requirements.memoryRequirements.size,
.memoryTypeIndex = preferred_type,
});
}
device.bindBufferMemory(buffer, memory, 0);
mapped = reinterpret_cast<u8*>(device.mapMemory(memory, 0, VK_WHOLE_SIZE));
if (instance.HasDebuggingToolAttached()) {
SetObjectName(device, buffer, "StreamBuffer({}): {} KiB {}", BufferTypeName(type),
stream_buffer_size / 1024, vk::to_string(mem_type.propertyFlags));
SetObjectName(device, memory, "StreamBufferMemory({}): {} Kib {}", BufferTypeName(type),
stream_buffer_size / 1024, vk::to_string(mem_type.propertyFlags));
}
}
void StreamBuffer::ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size) {
watches.resize(watches.size() + grow_size);
}
void StreamBuffer::WaitPendingOperations(u64 requested_upper_bound) {
if (!invalidation_mark) {
return;
}
while (requested_upper_bound > wait_bound && wait_cursor < *invalidation_mark) {
auto& watch = previous_watches[wait_cursor];
wait_bound = watch.upper_bound;
scheduler.Wait(watch.tick);
++wait_cursor;
}
}
u64 StreamBuffer::Copy(VAddr src, size_t size, size_t alignment /*= 0*/) {
const auto [data, offset, _] = Map(size, alignment);
std::memcpy(data, reinterpret_cast<const void*>(src), size);
Commit(size);
return offset;
}
} // namespace Vulkan

View File

@ -1,89 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <optional>
#include <span>
#include <tuple>
#include <vector>
#include "common/types.h"
#include "video_core/renderer_vulkan/vk_common.h"
namespace Vulkan {
enum class BufferType : u32 {
Upload = 0,
Download = 1,
Stream = 2,
};
class Instance;
class Scheduler;
class StreamBuffer final {
static constexpr std::size_t MAX_BUFFER_VIEWS = 3;
public:
explicit StreamBuffer(const Instance& instance, Scheduler& scheduler,
vk::BufferUsageFlags usage, u64 size,
BufferType type = BufferType::Stream);
~StreamBuffer();
/**
* Reserves a region of memory from the stream buffer.
* @param size Size to reserve.
* @returns A pair of a raw memory pointer (with offset added), and the buffer offset
*/
std::tuple<u8*, u64, bool> Map(u64 size, u64 alignment = 0);
/// Ensures that "size" bytes of memory are available to the GPU, potentially recording a copy.
void Commit(u64 size);
/// Maps and commits a memory region with user provided data
u64 Copy(VAddr src, size_t size, size_t alignment = 0);
vk::Buffer Handle() const noexcept {
return buffer;
}
private:
struct Watch {
u64 tick{};
u64 upper_bound{};
};
/// Creates Vulkan buffer handles committing the required the required memory.
void CreateBuffers(u64 prefered_size);
/// Increases the amount of watches available.
void ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size);
void WaitPendingOperations(u64 requested_upper_bound);
private:
const Instance& instance; ///< Vulkan instance.
Scheduler& scheduler; ///< Command scheduler.
vk::Device device;
vk::Buffer buffer; ///< Mapped buffer.
vk::DeviceMemory memory; ///< Memory allocation.
u8* mapped{}; ///< Pointer to the mapped memory
u64 stream_buffer_size{}; ///< Stream buffer size.
vk::BufferUsageFlags usage{};
BufferType type;
u64 offset{}; ///< Buffer iterator.
u64 mapped_size{}; ///< Size reserved for the current copy.
bool is_coherent{}; ///< True if the buffer is coherent
std::vector<Watch> current_watches; ///< Watches recorded in the current iteration.
std::size_t current_watch_cursor{}; ///< Count of watches, reset on invalidation.
std::optional<std::size_t> invalidation_mark; ///< Number of watches used in the previous cycle.
std::vector<Watch> previous_watches; ///< Watches used in the previous iteration.
std::size_t wait_cursor{}; ///< Last watch being waited for completion.
u64 wait_bound{}; ///< Highest offset being watched for completion.
};
} // namespace Vulkan

View File

@ -260,7 +260,7 @@ ImageInfo::ImageInfo(const AmdGpu::Image& image) noexcept {
case AmdGpu::TilingMode::Display_MacroTiled: case AmdGpu::TilingMode::Display_MacroTiled:
case AmdGpu::TilingMode::Texture_MacroTiled: case AmdGpu::TilingMode::Texture_MacroTiled:
case AmdGpu::TilingMode::Depth_MacroTiled: { case AmdGpu::TilingMode::Depth_MacroTiled: {
ASSERT(!props.is_cube && !props.is_block); // ASSERT(!props.is_cube && !props.is_block);
ASSERT(num_samples == 1); ASSERT(num_samples == 1);
std::tie(mip_info.pitch, mip_info.size) = std::tie(mip_info.pitch, mip_info.size) =
ImageSizeMacroTiled(mip_w, mip_h, bpp, num_samples, image.tiling_index); ImageSizeMacroTiled(mip_w, mip_h, bpp, num_samples, image.tiling_index);

View File

@ -61,23 +61,24 @@ vk::Format TrySwizzleFormat(vk::Format format, u32 dst_sel) {
return format; return format;
} }
ImageViewInfo::ImageViewInfo(const AmdGpu::Image& image, bool is_storage) noexcept ImageViewInfo::ImageViewInfo(const AmdGpu::Image& image, bool is_storage_) noexcept
: is_storage{is_storage} { : is_storage{is_storage_} {
type = ConvertImageViewType(image.GetType()); type = ConvertImageViewType(image.GetType());
format = Vulkan::LiverpoolToVK::SurfaceFormat(image.GetDataFmt(), image.GetNumberFmt()); format = Vulkan::LiverpoolToVK::SurfaceFormat(image.GetDataFmt(), image.GetNumberFmt());
range.base.level = image.base_level; range.base.level = image.base_level;
range.base.layer = image.base_array; range.base.layer = image.base_array;
range.extent.levels = image.last_level + 1; range.extent.levels = image.last_level + 1;
range.extent.layers = image.last_array + 1; range.extent.layers = image.last_array + 1;
if (!is_storage) {
mapping.r = ConvertComponentSwizzle(image.dst_sel_x); mapping.r = ConvertComponentSwizzle(image.dst_sel_x);
mapping.g = ConvertComponentSwizzle(image.dst_sel_y); mapping.g = ConvertComponentSwizzle(image.dst_sel_y);
mapping.b = ConvertComponentSwizzle(image.dst_sel_z); mapping.b = ConvertComponentSwizzle(image.dst_sel_z);
mapping.a = ConvertComponentSwizzle(image.dst_sel_w); mapping.a = ConvertComponentSwizzle(image.dst_sel_w);
}
// Check for unfortunate case of storage images being swizzled // Check for unfortunate case of storage images being swizzled
const u32 num_comps = AmdGpu::NumComponents(image.GetDataFmt()); const u32 num_comps = AmdGpu::NumComponents(image.GetDataFmt());
const u32 dst_sel = image.DstSelect(); const u32 dst_sel = image.DstSelect();
if (is_storage && !IsIdentityMapping(dst_sel, num_comps)) { if (is_storage && !IsIdentityMapping(dst_sel, num_comps)) {
mapping = vk::ComponentMapping{};
if (auto new_format = TrySwizzleFormat(format, dst_sel); new_format != format) { if (auto new_format = TrySwizzleFormat(format, dst_sel); new_format != format) {
format = new_format; format = new_format;
return; return;

View File

@ -3,103 +3,22 @@
#include <xxhash.h> #include <xxhash.h>
#include "common/assert.h" #include "common/assert.h"
#include "common/config.h" #include "video_core/page_manager.h"
#include "core/virtual_memory.h"
#include "video_core/renderer_vulkan/vk_instance.h" #include "video_core/renderer_vulkan/vk_instance.h"
#include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/texture_cache/texture_cache.h" #include "video_core/texture_cache/texture_cache.h"
#include "video_core/texture_cache/tile_manager.h" #include "video_core/texture_cache/tile_manager.h"
#ifndef _WIN64
#include <signal.h>
#include <sys/mman.h>
#define PAGE_NOACCESS PROT_NONE
#define PAGE_READWRITE (PROT_READ | PROT_WRITE)
#define PAGE_READONLY PROT_READ
#else
#include <windows.h>
void mprotect(void* addr, size_t len, int prot) {
DWORD old_prot{};
BOOL result = VirtualProtect(addr, len, prot, &old_prot);
ASSERT_MSG(result != 0, "Region protection failed");
}
#endif
namespace VideoCore { namespace VideoCore {
static TextureCache* g_texture_cache = nullptr;
#ifndef _WIN64
void GuestFaultSignalHandler(int sig, siginfo_t* info, void* raw_context) {
ucontext_t* ctx = reinterpret_cast<ucontext_t*>(raw_context);
const VAddr address = reinterpret_cast<VAddr>(info->si_addr);
#ifdef __APPLE__
const u32 err = ctx->uc_mcontext->__es.__err;
#else
const greg_t err = ctx->uc_mcontext.gregs[REG_ERR];
#endif
if (err & 0x2) {
g_texture_cache->OnCpuWrite(address);
} else {
// Read not supported!
UNREACHABLE();
}
}
#else
LONG WINAPI GuestFaultSignalHandler(EXCEPTION_POINTERS* pExp) noexcept {
const u32 ec = pExp->ExceptionRecord->ExceptionCode;
if (ec == EXCEPTION_ACCESS_VIOLATION) {
const auto info = pExp->ExceptionRecord->ExceptionInformation;
if (info[0] == 1) { // Write violation
g_texture_cache->OnCpuWrite(info[1]);
return EXCEPTION_CONTINUE_EXECUTION;
} /* else {
UNREACHABLE();
}*/
}
return EXCEPTION_CONTINUE_SEARCH; // pass further
}
#endif
static constexpr u64 StreamBufferSize = 512_MB; static constexpr u64 StreamBufferSize = 512_MB;
static constexpr u64 PageShift = 12; static constexpr u64 PageShift = 12;
TextureCache::TextureCache(const Vulkan::Instance& instance_, Vulkan::Scheduler& scheduler_) TextureCache::TextureCache(const Vulkan::Instance& instance_, Vulkan::Scheduler& scheduler_,
: instance{instance_}, scheduler{scheduler_}, BufferCache& buffer_cache_, PageManager& tracker_)
staging{instance, scheduler, vk::BufferUsageFlagBits::eTransferSrc, StreamBufferSize, : instance{instance_}, scheduler{scheduler_}, buffer_cache{buffer_cache_}, tracker{tracker_},
Vulkan::BufferType::Upload}, staging{instance, scheduler, MemoryUsage::Upload, StreamBufferSize},
tile_manager{instance, scheduler} { tile_manager{instance, scheduler} {
#ifndef _WIN64
#ifdef __APPLE__
// Read-only memory write results in SIGBUS on Apple.
static constexpr int SignalType = SIGBUS;
#else
static constexpr int SignalType = SIGSEGV;
#endif
sigset_t signal_mask;
sigemptyset(&signal_mask);
sigaddset(&signal_mask, SignalType);
using HandlerType = decltype(sigaction::sa_sigaction);
struct sigaction guest_access_fault {};
guest_access_fault.sa_flags = SA_SIGINFO | SA_ONSTACK;
guest_access_fault.sa_sigaction = &GuestFaultSignalHandler;
guest_access_fault.sa_mask = signal_mask;
sigaction(SignalType, &guest_access_fault, nullptr);
#else
veh_handle = AddVectoredExceptionHandler(0, GuestFaultSignalHandler);
ASSERT_MSG(veh_handle, "Failed to register an exception handler");
#endif
g_texture_cache = this;
ImageInfo info; ImageInfo info;
info.pixel_format = vk::Format::eR8G8B8A8Unorm; info.pixel_format = vk::Format::eR8G8B8A8Unorm;
info.type = vk::ImageType::e2D; info.type = vk::ImageType::e2D;
@ -110,15 +29,11 @@ TextureCache::TextureCache(const Vulkan::Instance& instance_, Vulkan::Scheduler&
void(slot_image_views.insert(instance, view_info, slot_images[null_id], null_id)); void(slot_image_views.insert(instance, view_info, slot_images[null_id], null_id));
} }
TextureCache::~TextureCache() { TextureCache::~TextureCache() = default;
#if _WIN64
RemoveVectoredExceptionHandler(veh_handle);
#endif
}
void TextureCache::OnCpuWrite(VAddr address) { void TextureCache::InvalidateMemory(VAddr address, size_t size) {
std::unique_lock lock{m_page_table}; std::unique_lock lock{mutex};
ForEachImageInRegion(address, 1 << PageShift, [&](ImageId image_id, Image& image) { ForEachImageInRegion(address, size, [&](ImageId image_id, Image& image) {
// Ensure image is reuploaded when accessed again. // Ensure image is reuploaded when accessed again.
image.flags |= ImageFlagBits::CpuModified; image.flags |= ImageFlagBits::CpuModified;
// Untrack image, so the range is unprotected and the guest can write freely. // Untrack image, so the range is unprotected and the guest can write freely.
@ -126,8 +41,28 @@ void TextureCache::OnCpuWrite(VAddr address) {
}); });
} }
void TextureCache::UnmapMemory(VAddr cpu_addr, size_t size) {
std::scoped_lock lk{mutex};
boost::container::small_vector<ImageId, 16> deleted_images;
ForEachImageInRegion(cpu_addr, size, [&](ImageId id, Image&) { deleted_images.push_back(id); });
for (const ImageId id : deleted_images) {
Image& image = slot_images[id];
if (True(image.flags & ImageFlagBits::Tracked)) {
UntrackImage(image, id);
}
// TODO: Download image data back to host.
UnregisterImage(id);
DeleteImage(id);
}
}
ImageId TextureCache::FindImage(const ImageInfo& info, bool refresh_on_create) { ImageId TextureCache::FindImage(const ImageInfo& info, bool refresh_on_create) {
std::unique_lock lock{m_page_table}; if (info.guest_address == 0) [[unlikely]] {
return NULL_IMAGE_VIEW_ID;
}
std::unique_lock lock{mutex};
boost::container::small_vector<ImageId, 2> image_ids; boost::container::small_vector<ImageId, 2> image_ids;
ForEachImageInRegion( ForEachImageInRegion(
info.guest_address, info.guest_size_bytes, [&](ImageId image_id, Image& image) { info.guest_address, info.guest_size_bytes, [&](ImageId image_id, Image& image) {
@ -183,10 +118,6 @@ ImageView& TextureCache::RegisterImageView(ImageId image_id, const ImageViewInfo
} }
ImageView& TextureCache::FindTexture(const ImageInfo& info, const ImageViewInfo& view_info) { ImageView& TextureCache::FindTexture(const ImageInfo& info, const ImageViewInfo& view_info) {
if (info.guest_address == 0) [[unlikely]] {
return slot_image_views[NULL_IMAGE_VIEW_ID];
}
const ImageId image_id = FindImage(info); const ImageId image_id = FindImage(info);
Image& image = slot_images[image_id]; Image& image = slot_images[image_id];
auto& usage = image.info.usage; auto& usage = image.info.usage;
@ -310,10 +241,7 @@ void TextureCache::RefreshImage(Image& image) {
buffer = *upload_buffer; buffer = *upload_buffer;
} else { } else {
// Upload data to the staging buffer. // Upload data to the staging buffer.
const auto [data, offset_, _] = staging.Map(image.info.guest_size_bytes, 16); offset = staging.Copy(image.info.guest_address, image.info.guest_size_bytes, 16);
std::memcpy(data, (void*)image.info.guest_address, image.info.guest_size_bytes);
staging.Commit(image.info.guest_size_bytes);
offset = offset_;
} }
const auto& num_layers = image.info.resources.layers; const auto& num_layers = image.info.resources.layers;
@ -344,9 +272,6 @@ void TextureCache::RefreshImage(Image& image) {
} }
cmdbuf.copyBufferToImage(buffer, image.image, vk::ImageLayout::eTransferDstOptimal, image_copy); cmdbuf.copyBufferToImage(buffer, image.image, vk::ImageLayout::eTransferDstOptimal, image_copy);
image.Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eMemoryRead);
} }
vk::Sampler TextureCache::GetSampler(const AmdGpu::Sampler& sampler) { vk::Sampler TextureCache::GetSampler(const AmdGpu::Sampler& sampler) {
@ -362,8 +287,6 @@ void TextureCache::RegisterImage(ImageId image_id) {
image.flags |= ImageFlagBits::Registered; image.flags |= ImageFlagBits::Registered;
ForEachPage(image.cpu_addr, image.info.guest_size_bytes, ForEachPage(image.cpu_addr, image.info.guest_size_bytes,
[this, image_id](u64 page) { page_table[page].push_back(image_id); }); [this, image_id](u64 page) { page_table[page].push_back(image_id); });
image.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits::eNone);
} }
void TextureCache::UnregisterImage(ImageId image_id) { void TextureCache::UnregisterImage(ImageId image_id) {
@ -373,11 +296,11 @@ void TextureCache::UnregisterImage(ImageId image_id) {
image.flags &= ~ImageFlagBits::Registered; image.flags &= ~ImageFlagBits::Registered;
ForEachPage(image.cpu_addr, image.info.guest_size_bytes, [this, image_id](u64 page) { ForEachPage(image.cpu_addr, image.info.guest_size_bytes, [this, image_id](u64 page) {
const auto page_it = page_table.find(page); const auto page_it = page_table.find(page);
if (page_it == page_table.end()) { if (page_it == nullptr) {
ASSERT_MSG(false, "Unregistering unregistered page=0x{:x}", page << PageShift); UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PageShift);
return; return;
} }
auto& image_ids = page_it.value(); auto& image_ids = *page_it;
const auto vector_it = std::ranges::find(image_ids, image_id); const auto vector_it = std::ranges::find(image_ids, image_id);
if (vector_it == image_ids.end()) { if (vector_it == image_ids.end()) {
ASSERT_MSG(false, "Unregistering unregistered image in page=0x{:x}", page << PageShift); ASSERT_MSG(false, "Unregistering unregistered image in page=0x{:x}", page << PageShift);
@ -393,7 +316,7 @@ void TextureCache::TrackImage(Image& image, ImageId image_id) {
return; return;
} }
image.flags |= ImageFlagBits::Tracked; image.flags |= ImageFlagBits::Tracked;
UpdatePagesCachedCount(image.cpu_addr, image.info.guest_size_bytes, 1); tracker.UpdatePagesCachedCount(image.cpu_addr, image.info.guest_size_bytes, 1);
} }
void TextureCache::UntrackImage(Image& image, ImageId image_id) { void TextureCache::UntrackImage(Image& image, ImageId image_id) {
@ -401,40 +324,34 @@ void TextureCache::UntrackImage(Image& image, ImageId image_id) {
return; return;
} }
image.flags &= ~ImageFlagBits::Tracked; image.flags &= ~ImageFlagBits::Tracked;
UpdatePagesCachedCount(image.cpu_addr, image.info.guest_size_bytes, -1); tracker.UpdatePagesCachedCount(image.cpu_addr, image.info.guest_size_bytes, -1);
} }
void TextureCache::UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta) { void TextureCache::DeleteImage(ImageId image_id) {
std::scoped_lock lk{mutex}; Image& image = slot_images[image_id];
const u64 num_pages = ((addr + size - 1) >> PageShift) - (addr >> PageShift) + 1; ASSERT_MSG(False(image.flags & ImageFlagBits::Tracked), "Image was not untracked");
const u64 page_start = addr >> PageShift; ASSERT_MSG(False(image.flags & ImageFlagBits::Registered), "Image was not unregistered");
const u64 page_end = page_start + num_pages;
const auto pages_interval = // Remove any registered meta areas.
decltype(cached_pages)::interval_type::right_open(page_start, page_end); const auto& meta_info = image.info.meta_info;
if (delta > 0) { if (meta_info.cmask_addr) {
cached_pages.add({pages_interval, delta}); surface_metas.erase(meta_info.cmask_addr);
}
if (meta_info.fmask_addr) {
surface_metas.erase(meta_info.fmask_addr);
}
if (meta_info.htile_addr) {
surface_metas.erase(meta_info.htile_addr);
} }
const auto& range = cached_pages.equal_range(pages_interval); // Reclaim image and any image views it references.
for (const auto& [range, count] : boost::make_iterator_range(range)) { scheduler.DeferOperation([this, image_id] {
const auto interval = range & pages_interval; Image& image = slot_images[image_id];
const VAddr interval_start_addr = boost::icl::first(interval) << PageShift; for (const ImageViewId image_view_id : image.image_view_ids) {
const VAddr interval_end_addr = boost::icl::last_next(interval) << PageShift; slot_image_views.erase(image_view_id);
const u32 interval_size = interval_end_addr - interval_start_addr;
void* addr = reinterpret_cast<void*>(interval_start_addr);
if (delta > 0 && count == delta) {
mprotect(addr, interval_size, PAGE_READONLY);
} else if (delta < 0 && count == -delta) {
mprotect(addr, interval_size, PAGE_READWRITE);
} else {
ASSERT(count >= 0);
}
}
if (delta < 0) {
cached_pages.add({pages_interval, delta});
} }
slot_images.erase(image_id);
});
} }
} // namespace VideoCore } // namespace VideoCore

View File

@ -4,12 +4,11 @@
#pragma once #pragma once
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
#include <boost/icl/interval_map.hpp>
#include <tsl/robin_map.h> #include <tsl/robin_map.h>
#include "common/slot_vector.h" #include "common/slot_vector.h"
#include "video_core/amdgpu/resource.h" #include "video_core/amdgpu/resource.h"
#include "video_core/renderer_vulkan/vk_stream_buffer.h" #include "video_core/multi_level_page_table.h"
#include "video_core/texture_cache/image.h" #include "video_core/texture_cache/image.h"
#include "video_core/texture_cache/image_view.h" #include "video_core/texture_cache/image_view.h"
#include "video_core/texture_cache/sampler.h" #include "video_core/texture_cache/sampler.h"
@ -21,31 +20,28 @@ struct BufferAttributeGroup;
namespace VideoCore { namespace VideoCore {
class BufferCache;
class PageManager;
class TextureCache { class TextureCache {
// This is the page shift for adding images into the hash map. It isn't related to struct Traits {
// the page size of the guest or the host and is chosen for convenience. A number too using Entry = boost::container::small_vector<ImageId, 16>;
// small will increase the number of hash map lookups per image, while too large will static constexpr size_t AddressSpaceBits = 39;
// increase the number of images per page. static constexpr size_t FirstLevelBits = 9;
static constexpr u64 PageBits = 20; static constexpr size_t PageBits = 22;
static constexpr u64 PageMask = (1ULL << PageBits) - 1;
struct MetaDataInfo {
enum class Type {
CMask,
FMask,
HTile,
};
Type type;
bool is_cleared;
}; };
using PageTable = MultiLevelPageTable<Traits>;
public: public:
explicit TextureCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler); explicit TextureCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
BufferCache& buffer_cache, PageManager& tracker);
~TextureCache(); ~TextureCache();
/// Invalidates any image in the logical page range. /// Invalidates any image in the logical page range.
void OnCpuWrite(VAddr address); void InvalidateMemory(VAddr address, size_t size);
/// Evicts any images that overlap the unmapped range.
void UnmapMemory(VAddr cpu_addr, size_t size);
/// Retrieves the image handle of the image with the provided attributes. /// Retrieves the image handle of the image with the provided attributes.
[[nodiscard]] ImageId FindImage(const ImageInfo& info, bool refresh_on_create = true); [[nodiscard]] ImageId FindImage(const ImageInfo& info, bool refresh_on_create = true);
@ -101,8 +97,8 @@ private:
template <typename Func> template <typename Func>
static void ForEachPage(PAddr addr, size_t size, Func&& func) { static void ForEachPage(PAddr addr, size_t size, Func&& func) {
static constexpr bool RETURNS_BOOL = std::is_same_v<std::invoke_result<Func, u64>, bool>; static constexpr bool RETURNS_BOOL = std::is_same_v<std::invoke_result<Func, u64>, bool>;
const u64 page_end = (addr + size - 1) >> PageBits; const u64 page_end = (addr + size - 1) >> Traits::PageBits;
for (u64 page = addr >> PageBits; page <= page_end; ++page) { for (u64 page = addr >> Traits::PageBits; page <= page_end; ++page) {
if constexpr (RETURNS_BOOL) { if constexpr (RETURNS_BOOL) {
if (func(page)) { if (func(page)) {
break; break;
@ -120,14 +116,14 @@ private:
boost::container::small_vector<ImageId, 32> images; boost::container::small_vector<ImageId, 32> images;
ForEachPage(cpu_addr, size, [this, &images, cpu_addr, size, func](u64 page) { ForEachPage(cpu_addr, size, [this, &images, cpu_addr, size, func](u64 page) {
const auto it = page_table.find(page); const auto it = page_table.find(page);
if (it == page_table.end()) { if (it == nullptr) {
if constexpr (BOOL_BREAK) { if constexpr (BOOL_BREAK) {
return false; return false;
} else { } else {
return; return;
} }
} }
for (const ImageId image_id : it->second) { for (const ImageId image_id : *it) {
Image& image = slot_images[image_id]; Image& image = slot_images[image_id];
if (image.flags & ImageFlagBits::Picked) { if (image.flags & ImageFlagBits::Picked) {
continue; continue;
@ -166,25 +162,32 @@ private:
/// Stop tracking CPU reads and writes for image /// Stop tracking CPU reads and writes for image
void UntrackImage(Image& image, ImageId image_id); void UntrackImage(Image& image, ImageId image_id);
/// Increase/decrease the number of surface in pages touching the specified region /// Removes the image and any views/surface metas that reference it.
void UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta); void DeleteImage(ImageId image_id);
private: private:
const Vulkan::Instance& instance; const Vulkan::Instance& instance;
Vulkan::Scheduler& scheduler; Vulkan::Scheduler& scheduler;
Vulkan::StreamBuffer staging; BufferCache& buffer_cache;
PageManager& tracker;
StreamBuffer staging;
TileManager tile_manager; TileManager tile_manager;
Common::SlotVector<Image> slot_images; Common::SlotVector<Image> slot_images;
Common::SlotVector<ImageView> slot_image_views; Common::SlotVector<ImageView> slot_image_views;
tsl::robin_map<u64, Sampler> samplers; tsl::robin_map<u64, Sampler> samplers;
tsl::robin_pg_map<u64, std::vector<ImageId>> page_table; PageTable page_table;
boost::icl::interval_map<VAddr, s32> cached_pages;
tsl::robin_map<VAddr, MetaDataInfo> surface_metas;
std::mutex mutex; std::mutex mutex;
#ifdef _WIN64
void* veh_handle{}; struct MetaDataInfo {
#endif enum class Type {
std::mutex m_page_table; CMask,
FMask,
HTile,
};
Type type;
bool is_cleared;
};
tsl::robin_map<VAddr, MetaDataInfo> surface_metas;
}; };
} // namespace VideoCore } // namespace VideoCore

View File

@ -183,10 +183,12 @@ vk::Format DemoteImageFormatForDetiling(vk::Format format) {
case vk::Format::eB8G8R8A8Srgb: case vk::Format::eB8G8R8A8Srgb:
case vk::Format::eB8G8R8A8Unorm: case vk::Format::eB8G8R8A8Unorm:
case vk::Format::eR8G8B8A8Unorm: case vk::Format::eR8G8B8A8Unorm:
case vk::Format::eR8G8B8A8Uint:
case vk::Format::eR32Sfloat: case vk::Format::eR32Sfloat:
case vk::Format::eR32Uint: case vk::Format::eR32Uint:
case vk::Format::eR16G16Sfloat: case vk::Format::eR16G16Sfloat:
return vk::Format::eR32Uint; return vk::Format::eR32Uint;
case vk::Format::eBc1RgbaSrgbBlock:
case vk::Format::eBc1RgbaUnormBlock: case vk::Format::eBc1RgbaUnormBlock:
case vk::Format::eBc4UnormBlock: case vk::Format::eBc4UnormBlock:
case vk::Format::eR32G32Sfloat: case vk::Format::eR32G32Sfloat:
@ -200,11 +202,20 @@ vk::Format DemoteImageFormatForDetiling(vk::Format format) {
case vk::Format::eBc5UnormBlock: case vk::Format::eBc5UnormBlock:
case vk::Format::eBc7SrgbBlock: case vk::Format::eBc7SrgbBlock:
case vk::Format::eBc7UnormBlock: case vk::Format::eBc7UnormBlock:
case vk::Format::eBc6HUfloatBlock:
case vk::Format::eR32G32B32A32Sfloat:
return vk::Format::eR32G32B32A32Uint; return vk::Format::eR32G32B32A32Uint;
default: default:
break; break;
} }
// Log missing formats only once to avoid spamming the log.
static constexpr size_t MaxFormatIndex = 256;
static std::array<bool, MaxFormatIndex> logged_formats{};
if (const u32 index = u32(format); !logged_formats[index]) {
LOG_ERROR(Render_Vulkan, "Unexpected format for demotion {}", vk::to_string(format)); LOG_ERROR(Render_Vulkan, "Unexpected format for demotion {}", vk::to_string(format));
logged_formats[index] = true;
}
return format; return format;
} }
@ -236,8 +247,11 @@ struct DetilerParams {
u32 sizes[14]; u32 sizes[14];
}; };
static constexpr size_t StreamBufferSize = 128_MB;
TileManager::TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler) TileManager::TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler)
: instance{instance}, scheduler{scheduler} { : instance{instance}, scheduler{scheduler},
stream_buffer{instance, scheduler, MemoryUsage::Stream, StreamBufferSize} {
static const std::array detiler_shaders{ static const std::array detiler_shaders{
HostShaders::DETILE_M8X1_COMP, HostShaders::DETILE_M8X2_COMP, HostShaders::DETILE_M8X1_COMP, HostShaders::DETILE_M8X2_COMP,
HostShaders::DETILE_M32X1_COMP, HostShaders::DETILE_M32X2_COMP, HostShaders::DETILE_M32X1_COMP, HostShaders::DETILE_M32X2_COMP,
@ -336,8 +350,7 @@ TileManager::ScratchBuffer TileManager::AllocBuffer(u32 size, bool is_storage /*
.flags = !is_storage ? VMA_ALLOCATION_CREATE_HOST_ACCESS_ALLOW_TRANSFER_INSTEAD_BIT | .flags = !is_storage ? VMA_ALLOCATION_CREATE_HOST_ACCESS_ALLOW_TRANSFER_INSTEAD_BIT |
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT
: static_cast<VmaAllocationCreateFlags>(0), : static_cast<VmaAllocationCreateFlags>(0),
.usage = is_large_buffer ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
: VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.requiredFlags = !is_storage ? VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT .requiredFlags = !is_storage ? VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT
: static_cast<VkMemoryPropertyFlags>(0), : static_cast<VkMemoryPropertyFlags>(0),
}; };
@ -373,37 +386,46 @@ std::optional<vk::Buffer> TileManager::TryDetile(Image& image) {
const auto* detiler = GetDetiler(image); const auto* detiler = GetDetiler(image);
if (!detiler) { if (!detiler) {
if (image.info.tiling_mode != AmdGpu::TilingMode::Texture_MacroTiled) {
LOG_ERROR(Render_Vulkan, "Unsupported tiled image: {} ({})", LOG_ERROR(Render_Vulkan, "Unsupported tiled image: {} ({})",
vk::to_string(image.info.pixel_format), NameOf(image.info.tiling_mode)); vk::to_string(image.info.pixel_format), NameOf(image.info.tiling_mode));
}
return std::nullopt; return std::nullopt;
} }
// Prepare input buffer // Prepare input buffer
auto in_buffer = AllocBuffer(image.info.guest_size_bytes); const u32 image_size = image.info.guest_size_bytes;
Upload(in_buffer, reinterpret_cast<const void*>(image.info.guest_address), const auto [in_buffer, in_offset] = [&] -> std::pair<vk::Buffer, u32> {
image.info.guest_size_bytes); // Use stream buffer for smaller textures.
if (image_size <= StreamBufferSize) {
u32 offset = stream_buffer.Copy(image.info.guest_address, image_size);
return {stream_buffer.Handle(), offset};
}
// Request temporary host buffer for larger sizes.
auto in_buffer = AllocBuffer(image_size);
const auto addr = reinterpret_cast<const void*>(image.info.guest_address);
Upload(in_buffer, addr, image_size);
scheduler.DeferOperation([=, this]() { FreeBuffer(in_buffer); });
return {in_buffer.first, 0};
}();
// Prepare output buffer // Prepare output buffer
auto out_buffer = AllocBuffer(image.info.guest_size_bytes, true); auto out_buffer = AllocBuffer(image_size, true);
scheduler.DeferOperation([=, this]() { FreeBuffer(out_buffer); });
scheduler.DeferOperation([=, this]() {
FreeBuffer(in_buffer);
FreeBuffer(out_buffer);
});
auto cmdbuf = scheduler.CommandBuffer(); auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.bindPipeline(vk::PipelineBindPoint::eCompute, *detiler->pl); cmdbuf.bindPipeline(vk::PipelineBindPoint::eCompute, *detiler->pl);
const vk::DescriptorBufferInfo input_buffer_info{ const vk::DescriptorBufferInfo input_buffer_info{
.buffer = in_buffer.first, .buffer = in_buffer,
.offset = 0, .offset = in_offset,
.range = image.info.guest_size_bytes, .range = image_size,
}; };
const vk::DescriptorBufferInfo output_buffer_info{ const vk::DescriptorBufferInfo output_buffer_info{
.buffer = out_buffer.first, .buffer = out_buffer.first,
.offset = 0, .offset = 0,
.range = image.info.guest_size_bytes, .range = image_size,
}; };
std::vector<vk::WriteDescriptorSet> set_writes{ std::vector<vk::WriteDescriptorSet> set_writes{
@ -442,16 +464,16 @@ std::optional<vk::Buffer> TileManager::TryDetile(Image& image) {
cmdbuf.pushConstants(*detiler->pl_layout, vk::ShaderStageFlagBits::eCompute, 0u, sizeof(params), cmdbuf.pushConstants(*detiler->pl_layout, vk::ShaderStageFlagBits::eCompute, 0u, sizeof(params),
&params); &params);
ASSERT((image.info.guest_size_bytes % 64) == 0); ASSERT((image_size % 64) == 0);
const auto bpp = image.info.num_bits * (image.info.props.is_block ? 16u : 1u); const auto bpp = image.info.num_bits * (image.info.props.is_block ? 16u : 1u);
const auto num_tiles = image.info.guest_size_bytes / (64 * (bpp / 8)); const auto num_tiles = image_size / (64 * (bpp / 8));
cmdbuf.dispatch(num_tiles, 1, 1); cmdbuf.dispatch(num_tiles, 1, 1);
const vk::BufferMemoryBarrier post_barrier{ const vk::BufferMemoryBarrier post_barrier{
.srcAccessMask = vk::AccessFlagBits::eShaderWrite, .srcAccessMask = vk::AccessFlagBits::eShaderWrite,
.dstAccessMask = vk::AccessFlagBits::eTransferRead, .dstAccessMask = vk::AccessFlagBits::eTransferRead,
.buffer = out_buffer.first, .buffer = out_buffer.first,
.size = image.info.guest_size_bytes, .size = image_size,
}; };
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader, cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion, vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,

View File

@ -4,7 +4,7 @@
#pragma once #pragma once
#include "common/types.h" #include "common/types.h"
#include "video_core/renderer_vulkan/vk_stream_buffer.h" #include "video_core/buffer_cache/buffer.h"
#include "video_core/texture_cache/image.h" #include "video_core/texture_cache/image.h"
namespace VideoCore { namespace VideoCore {
@ -34,7 +34,7 @@ struct DetilerContext {
class TileManager { class TileManager {
public: public:
using ScratchBuffer = std::pair<VkBuffer, VmaAllocation>; using ScratchBuffer = std::pair<vk::Buffer, VmaAllocation>;
TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler); TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler);
~TileManager(); ~TileManager();
@ -51,6 +51,7 @@ private:
private: private:
const Vulkan::Instance& instance; const Vulkan::Instance& instance;
Vulkan::Scheduler& scheduler; Vulkan::Scheduler& scheduler;
StreamBuffer stream_buffer;
std::array<DetilerContext, DetilerType::Max> detilers; std::array<DetilerContext, DetilerType::Max> detilers;
}; };