Compare commits
5 Commits
main
...
new-buffer
Author | SHA1 | Date |
---|---|---|
IndecisiveTurtle | 2c98063e76 | |
IndecisiveTurtle | b9938cb1d8 | |
IndecisiveTurtle | f3fd02812e | |
IndecisiveTurtle | b387ce9bbd | |
IndecisiveTurtle | efcabbfd2a |
|
@ -283,6 +283,7 @@ set(COMMON src/common/logging/backend.cpp
|
||||||
src/common/native_clock.h
|
src/common/native_clock.h
|
||||||
src/common/path_util.cpp
|
src/common/path_util.cpp
|
||||||
src/common/path_util.h
|
src/common/path_util.h
|
||||||
|
src/common/object_pool.h
|
||||||
src/common/polyfill_thread.h
|
src/common/polyfill_thread.h
|
||||||
src/common/rdtsc.cpp
|
src/common/rdtsc.cpp
|
||||||
src/common/rdtsc.h
|
src/common/rdtsc.h
|
||||||
|
@ -294,6 +295,7 @@ set(COMMON src/common/logging/backend.cpp
|
||||||
src/common/thread.h
|
src/common/thread.h
|
||||||
src/common/types.h
|
src/common/types.h
|
||||||
src/common/uint128.h
|
src/common/uint128.h
|
||||||
|
src/common/unique_function.h
|
||||||
src/common/version.h
|
src/common/version.h
|
||||||
src/common/ntapi.h
|
src/common/ntapi.h
|
||||||
src/common/ntapi.cpp
|
src/common/ntapi.cpp
|
||||||
|
@ -367,7 +369,6 @@ set(CORE src/core/aerolib/stubs.cpp
|
||||||
)
|
)
|
||||||
|
|
||||||
set(SHADER_RECOMPILER src/shader_recompiler/exception.h
|
set(SHADER_RECOMPILER src/shader_recompiler/exception.h
|
||||||
src/shader_recompiler/object_pool.h
|
|
||||||
src/shader_recompiler/profile.h
|
src/shader_recompiler/profile.h
|
||||||
src/shader_recompiler/recompiler.cpp
|
src/shader_recompiler/recompiler.cpp
|
||||||
src/shader_recompiler/recompiler.h
|
src/shader_recompiler/recompiler.h
|
||||||
|
@ -451,6 +452,13 @@ set(VIDEO_CORE src/video_core/amdgpu/liverpool.cpp
|
||||||
src/video_core/amdgpu/pm4_cmds.h
|
src/video_core/amdgpu/pm4_cmds.h
|
||||||
src/video_core/amdgpu/pm4_opcodes.h
|
src/video_core/amdgpu/pm4_opcodes.h
|
||||||
src/video_core/amdgpu/resource.h
|
src/video_core/amdgpu/resource.h
|
||||||
|
src/video_core/buffer_cache/buffer.cpp
|
||||||
|
src/video_core/buffer_cache/buffer.h
|
||||||
|
src/video_core/buffer_cache/buffer_cache.cpp
|
||||||
|
src/video_core/buffer_cache/buffer_cache.h
|
||||||
|
src/video_core/buffer_cache/memory_tracker_base.h
|
||||||
|
src/video_core/buffer_cache/range_set.h
|
||||||
|
src/video_core/buffer_cache/word_manager.h
|
||||||
src/video_core/renderer_vulkan/liverpool_to_vk.cpp
|
src/video_core/renderer_vulkan/liverpool_to_vk.cpp
|
||||||
src/video_core/renderer_vulkan/liverpool_to_vk.h
|
src/video_core/renderer_vulkan/liverpool_to_vk.h
|
||||||
src/video_core/renderer_vulkan/renderer_vulkan.cpp
|
src/video_core/renderer_vulkan/renderer_vulkan.cpp
|
||||||
|
@ -479,8 +487,6 @@ set(VIDEO_CORE src/video_core/amdgpu/liverpool.cpp
|
||||||
src/video_core/renderer_vulkan/vk_scheduler.h
|
src/video_core/renderer_vulkan/vk_scheduler.h
|
||||||
src/video_core/renderer_vulkan/vk_shader_util.cpp
|
src/video_core/renderer_vulkan/vk_shader_util.cpp
|
||||||
src/video_core/renderer_vulkan/vk_shader_util.h
|
src/video_core/renderer_vulkan/vk_shader_util.h
|
||||||
src/video_core/renderer_vulkan/vk_stream_buffer.cpp
|
|
||||||
src/video_core/renderer_vulkan/vk_stream_buffer.h
|
|
||||||
src/video_core/renderer_vulkan/vk_swapchain.cpp
|
src/video_core/renderer_vulkan/vk_swapchain.cpp
|
||||||
src/video_core/renderer_vulkan/vk_swapchain.h
|
src/video_core/renderer_vulkan/vk_swapchain.h
|
||||||
src/video_core/texture_cache/image.cpp
|
src/video_core/texture_cache/image.cpp
|
||||||
|
@ -496,6 +502,9 @@ set(VIDEO_CORE src/video_core/amdgpu/liverpool.cpp
|
||||||
src/video_core/texture_cache/tile_manager.cpp
|
src/video_core/texture_cache/tile_manager.cpp
|
||||||
src/video_core/texture_cache/tile_manager.h
|
src/video_core/texture_cache/tile_manager.h
|
||||||
src/video_core/texture_cache/types.h
|
src/video_core/texture_cache/types.h
|
||||||
|
src/video_core/page_manager.cpp
|
||||||
|
src/video_core/page_manager.h
|
||||||
|
src/video_core/multi_level_page_table.h
|
||||||
src/video_core/renderdoc.cpp
|
src/video_core/renderdoc.cpp
|
||||||
src/video_core/renderdoc.h
|
src/video_core/renderdoc.h
|
||||||
)
|
)
|
||||||
|
|
|
@ -8,7 +8,7 @@
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
namespace Shader {
|
namespace Common {
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
requires std::is_destructible_v<T>
|
requires std::is_destructible_v<T>
|
||||||
|
@ -104,4 +104,4 @@ private:
|
||||||
size_t new_chunk_size{};
|
size_t new_chunk_size{};
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace Shader
|
} // namespace Common
|
|
@ -0,0 +1,61 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
namespace Common {
|
||||||
|
|
||||||
|
/// General purpose function wrapper similar to std::function.
|
||||||
|
/// Unlike std::function, the captured values don't have to be copyable.
|
||||||
|
/// This class can be moved but not copied.
|
||||||
|
template <typename ResultType, typename... Args>
|
||||||
|
class UniqueFunction {
|
||||||
|
class CallableBase {
|
||||||
|
public:
|
||||||
|
virtual ~CallableBase() = default;
|
||||||
|
virtual ResultType operator()(Args&&...) = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
template <typename Functor>
|
||||||
|
class Callable final : public CallableBase {
|
||||||
|
public:
|
||||||
|
Callable(Functor&& functor_) : functor{std::move(functor_)} {}
|
||||||
|
~Callable() override = default;
|
||||||
|
|
||||||
|
ResultType operator()(Args&&... args) override {
|
||||||
|
return functor(std::forward<Args>(args)...);
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
Functor functor;
|
||||||
|
};
|
||||||
|
|
||||||
|
public:
|
||||||
|
UniqueFunction() = default;
|
||||||
|
|
||||||
|
template <typename Functor>
|
||||||
|
UniqueFunction(Functor&& functor)
|
||||||
|
: callable{std::make_unique<Callable<Functor>>(std::move(functor))} {}
|
||||||
|
|
||||||
|
UniqueFunction& operator=(UniqueFunction&& rhs) noexcept = default;
|
||||||
|
UniqueFunction(UniqueFunction&& rhs) noexcept = default;
|
||||||
|
|
||||||
|
UniqueFunction& operator=(const UniqueFunction&) = delete;
|
||||||
|
UniqueFunction(const UniqueFunction&) = delete;
|
||||||
|
|
||||||
|
ResultType operator()(Args&&... args) const {
|
||||||
|
return (*callable)(std::forward<Args>(args)...);
|
||||||
|
}
|
||||||
|
|
||||||
|
explicit operator bool() const noexcept {
|
||||||
|
return static_cast<bool>(callable);
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::unique_ptr<CallableBase> callable;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace Common
|
|
@ -465,7 +465,7 @@ int PS4_SYSV_ABI scePthreadMutexDestroy(ScePthreadMutex* mutex) {
|
||||||
|
|
||||||
int result = pthread_mutex_destroy(&(*mutex)->pth_mutex);
|
int result = pthread_mutex_destroy(&(*mutex)->pth_mutex);
|
||||||
|
|
||||||
LOG_INFO(Kernel_Pthread, "name={}, result={}", (*mutex)->name, result);
|
LOG_DEBUG(Kernel_Pthread, "name={}, result={}", (*mutex)->name, result);
|
||||||
|
|
||||||
delete *mutex;
|
delete *mutex;
|
||||||
*mutex = nullptr;
|
*mutex = nullptr;
|
||||||
|
@ -725,7 +725,7 @@ int PS4_SYSV_ABI scePthreadCondDestroy(ScePthreadCond* cond) {
|
||||||
}
|
}
|
||||||
int result = pthread_cond_destroy(&(*cond)->cond);
|
int result = pthread_cond_destroy(&(*cond)->cond);
|
||||||
|
|
||||||
LOG_INFO(Kernel_Pthread, "scePthreadCondDestroy, result={}", result);
|
LOG_DEBUG(Kernel_Pthread, "scePthreadCondDestroy, result={}", result);
|
||||||
|
|
||||||
delete *cond;
|
delete *cond;
|
||||||
*cond = nullptr;
|
*cond = nullptr;
|
||||||
|
@ -811,8 +811,6 @@ int PS4_SYSV_ABI posix_pthread_cond_timedwait(ScePthreadCond* cond, ScePthreadMu
|
||||||
}
|
}
|
||||||
|
|
||||||
int PS4_SYSV_ABI posix_pthread_cond_broadcast(ScePthreadCond* cond) {
|
int PS4_SYSV_ABI posix_pthread_cond_broadcast(ScePthreadCond* cond) {
|
||||||
LOG_INFO(Kernel_Pthread,
|
|
||||||
"posix posix_pthread_cond_broadcast redirect to scePthreadCondBroadcast");
|
|
||||||
int result = scePthreadCondBroadcast(cond);
|
int result = scePthreadCondBroadcast(cond);
|
||||||
if (result != 0) {
|
if (result != 0) {
|
||||||
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
|
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
|
||||||
|
@ -824,7 +822,6 @@ int PS4_SYSV_ABI posix_pthread_cond_broadcast(ScePthreadCond* cond) {
|
||||||
}
|
}
|
||||||
|
|
||||||
int PS4_SYSV_ABI posix_pthread_mutexattr_init(ScePthreadMutexattr* attr) {
|
int PS4_SYSV_ABI posix_pthread_mutexattr_init(ScePthreadMutexattr* attr) {
|
||||||
// LOG_INFO(Kernel_Pthread, "posix pthread_mutexattr_init redirect to scePthreadMutexattrInit");
|
|
||||||
int result = scePthreadMutexattrInit(attr);
|
int result = scePthreadMutexattrInit(attr);
|
||||||
if (result < 0) {
|
if (result < 0) {
|
||||||
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
|
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
|
||||||
|
@ -836,7 +833,6 @@ int PS4_SYSV_ABI posix_pthread_mutexattr_init(ScePthreadMutexattr* attr) {
|
||||||
}
|
}
|
||||||
|
|
||||||
int PS4_SYSV_ABI posix_pthread_mutexattr_settype(ScePthreadMutexattr* attr, int type) {
|
int PS4_SYSV_ABI posix_pthread_mutexattr_settype(ScePthreadMutexattr* attr, int type) {
|
||||||
// LOG_INFO(Kernel_Pthread, "posix pthread_mutex_init redirect to scePthreadMutexInit");
|
|
||||||
int result = scePthreadMutexattrSettype(attr, type);
|
int result = scePthreadMutexattrSettype(attr, type);
|
||||||
if (result < 0) {
|
if (result < 0) {
|
||||||
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
|
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
|
||||||
|
@ -861,7 +857,6 @@ int PS4_SYSV_ABI posix_pthread_once(pthread_once_t* once_control, void (*init_ro
|
||||||
|
|
||||||
int PS4_SYSV_ABI posix_pthread_mutexattr_setprotocol(ScePthreadMutexattr* attr, int protocol) {
|
int PS4_SYSV_ABI posix_pthread_mutexattr_setprotocol(ScePthreadMutexattr* attr, int protocol) {
|
||||||
int result = scePthreadMutexattrSetprotocol(attr, protocol);
|
int result = scePthreadMutexattrSetprotocol(attr, protocol);
|
||||||
LOG_INFO(Kernel_Pthread, "redirect to scePthreadMutexattrSetprotocol: result = {}", result);
|
|
||||||
if (result < 0) {
|
if (result < 0) {
|
||||||
UNREACHABLE();
|
UNREACHABLE();
|
||||||
}
|
}
|
||||||
|
@ -1295,8 +1290,6 @@ int PS4_SYSV_ABI posix_pthread_attr_setdetachstate(ScePthreadAttr* attr, int det
|
||||||
int PS4_SYSV_ABI posix_pthread_create_name_np(ScePthread* thread, const ScePthreadAttr* attr,
|
int PS4_SYSV_ABI posix_pthread_create_name_np(ScePthread* thread, const ScePthreadAttr* attr,
|
||||||
PthreadEntryFunc start_routine, void* arg,
|
PthreadEntryFunc start_routine, void* arg,
|
||||||
const char* name) {
|
const char* name) {
|
||||||
LOG_INFO(Kernel_Pthread, "posix pthread_create redirect to scePthreadCreate: name = {}", name);
|
|
||||||
|
|
||||||
int result = scePthreadCreate(thread, attr, start_routine, arg, name);
|
int result = scePthreadCreate(thread, attr, start_routine, arg, name);
|
||||||
if (result != 0) {
|
if (result != 0) {
|
||||||
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
|
int rt = result > SCE_KERNEL_ERROR_UNKNOWN && result <= SCE_KERNEL_ERROR_ESTOP
|
||||||
|
@ -1343,17 +1336,11 @@ int PS4_SYSV_ABI posix_pthread_cond_init(ScePthreadCond* cond, const ScePthreadC
|
||||||
|
|
||||||
int PS4_SYSV_ABI posix_pthread_cond_signal(ScePthreadCond* cond) {
|
int PS4_SYSV_ABI posix_pthread_cond_signal(ScePthreadCond* cond) {
|
||||||
int result = scePthreadCondSignal(cond);
|
int result = scePthreadCondSignal(cond);
|
||||||
LOG_INFO(Kernel_Pthread,
|
|
||||||
"posix posix_pthread_cond_signal redirect to scePthreadCondSignal, result = {}",
|
|
||||||
result);
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
int PS4_SYSV_ABI posix_pthread_cond_destroy(ScePthreadCond* cond) {
|
int PS4_SYSV_ABI posix_pthread_cond_destroy(ScePthreadCond* cond) {
|
||||||
int result = scePthreadCondDestroy(cond);
|
int result = scePthreadCondDestroy(cond);
|
||||||
LOG_INFO(Kernel_Pthread,
|
|
||||||
"posix posix_pthread_cond_destroy redirect to scePthreadCondDestroy, result = {}",
|
|
||||||
result);
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -470,7 +470,7 @@ int PS4_SYSV_ABI scePadSetUserColor() {
|
||||||
}
|
}
|
||||||
|
|
||||||
int PS4_SYSV_ABI scePadSetVibration(s32 handle, const OrbisPadVibrationParam* pParam) {
|
int PS4_SYSV_ABI scePadSetVibration(s32 handle, const OrbisPadVibrationParam* pParam) {
|
||||||
LOG_ERROR(Lib_Pad, "(STUBBED) called");
|
LOG_DEBUG(Lib_Pad, "(STUBBED) called");
|
||||||
return ORBIS_OK;
|
return ORBIS_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
#include "core/libraries/error_codes.h"
|
#include "core/libraries/error_codes.h"
|
||||||
#include "core/libraries/kernel/memory_management.h"
|
#include "core/libraries/kernel/memory_management.h"
|
||||||
#include "core/memory.h"
|
#include "core/memory.h"
|
||||||
#include "video_core/renderer_vulkan/vk_instance.h"
|
#include "video_core/renderer_vulkan/vk_rasterizer.h"
|
||||||
|
|
||||||
namespace Core {
|
namespace Core {
|
||||||
|
|
||||||
|
@ -172,7 +172,7 @@ int MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, size_t size, M
|
||||||
|
|
||||||
if (type == VMAType::Direct) {
|
if (type == VMAType::Direct) {
|
||||||
new_vma.phys_base = phys_addr;
|
new_vma.phys_base = phys_addr;
|
||||||
MapVulkanMemory(mapped_addr, size);
|
rasterizer->MapMemory(mapped_addr, size);
|
||||||
}
|
}
|
||||||
if (type == VMAType::Flexible) {
|
if (type == VMAType::Flexible) {
|
||||||
flexible_usage += size;
|
flexible_usage += size;
|
||||||
|
@ -222,7 +222,7 @@ void MemoryManager::UnmapMemory(VAddr virtual_addr, size_t size) {
|
||||||
const auto type = it->second.type;
|
const auto type = it->second.type;
|
||||||
const bool has_backing = type == VMAType::Direct || type == VMAType::File;
|
const bool has_backing = type == VMAType::Direct || type == VMAType::File;
|
||||||
if (type == VMAType::Direct) {
|
if (type == VMAType::Direct) {
|
||||||
UnmapVulkanMemory(virtual_addr, size);
|
rasterizer->UnmapMemory(virtual_addr, size);
|
||||||
}
|
}
|
||||||
if (type == VMAType::Flexible) {
|
if (type == VMAType::Flexible) {
|
||||||
flexible_usage -= size;
|
flexible_usage -= size;
|
||||||
|
@ -263,7 +263,7 @@ int MemoryManager::QueryProtection(VAddr addr, void** start, void** end, u32* pr
|
||||||
}
|
}
|
||||||
|
|
||||||
int MemoryManager::VirtualQuery(VAddr addr, int flags,
|
int MemoryManager::VirtualQuery(VAddr addr, int flags,
|
||||||
Libraries::Kernel::OrbisVirtualQueryInfo* info) {
|
::Libraries::Kernel::OrbisVirtualQueryInfo* info) {
|
||||||
std::scoped_lock lk{mutex};
|
std::scoped_lock lk{mutex};
|
||||||
|
|
||||||
auto it = FindVMA(addr);
|
auto it = FindVMA(addr);
|
||||||
|
@ -293,7 +293,7 @@ int MemoryManager::VirtualQuery(VAddr addr, int flags,
|
||||||
}
|
}
|
||||||
|
|
||||||
int MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next,
|
int MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next,
|
||||||
Libraries::Kernel::OrbisQueryInfo* out_info) {
|
::Libraries::Kernel::OrbisQueryInfo* out_info) {
|
||||||
std::scoped_lock lk{mutex};
|
std::scoped_lock lk{mutex};
|
||||||
|
|
||||||
auto dmem_area = FindDmemArea(addr);
|
auto dmem_area = FindDmemArea(addr);
|
||||||
|
@ -333,13 +333,6 @@ int MemoryManager::DirectQueryAvailable(PAddr search_start, PAddr search_end, si
|
||||||
return ORBIS_OK;
|
return ORBIS_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::pair<vk::Buffer, size_t> MemoryManager::GetVulkanBuffer(VAddr addr) {
|
|
||||||
auto it = mapped_memories.upper_bound(addr);
|
|
||||||
it = std::prev(it);
|
|
||||||
ASSERT(it != mapped_memories.end() && it->first <= addr);
|
|
||||||
return std::make_pair(*it->second.buffer, addr - it->first);
|
|
||||||
}
|
|
||||||
|
|
||||||
void MemoryManager::NameVirtualRange(VAddr virtual_addr, size_t size, std::string_view name) {
|
void MemoryManager::NameVirtualRange(VAddr virtual_addr, size_t size, std::string_view name) {
|
||||||
auto it = FindVMA(virtual_addr);
|
auto it = FindVMA(virtual_addr);
|
||||||
|
|
||||||
|
@ -455,85 +448,6 @@ MemoryManager::DMemHandle MemoryManager::Split(DMemHandle dmem_handle, size_t of
|
||||||
return dmem_map.emplace_hint(std::next(dmem_handle), new_area.base, new_area);
|
return dmem_map.emplace_hint(std::next(dmem_handle), new_area.base, new_area);
|
||||||
};
|
};
|
||||||
|
|
||||||
void MemoryManager::MapVulkanMemory(VAddr addr, size_t size) {
|
|
||||||
return;
|
|
||||||
const vk::Device device = instance->GetDevice();
|
|
||||||
const auto memory_props = instance->GetPhysicalDevice().getMemoryProperties();
|
|
||||||
void* host_pointer = reinterpret_cast<void*>(addr);
|
|
||||||
const auto host_mem_props = device.getMemoryHostPointerPropertiesEXT(
|
|
||||||
vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, host_pointer);
|
|
||||||
ASSERT(host_mem_props.memoryTypeBits != 0);
|
|
||||||
|
|
||||||
int mapped_memory_type = -1;
|
|
||||||
auto find_mem_type_with_flag = [&](const vk::MemoryPropertyFlags flags) {
|
|
||||||
u32 host_mem_types = host_mem_props.memoryTypeBits;
|
|
||||||
while (host_mem_types != 0) {
|
|
||||||
// Try to find a cached memory type
|
|
||||||
mapped_memory_type = std::countr_zero(host_mem_types);
|
|
||||||
host_mem_types -= (1 << mapped_memory_type);
|
|
||||||
|
|
||||||
if ((memory_props.memoryTypes[mapped_memory_type].propertyFlags & flags) == flags) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
mapped_memory_type = -1;
|
|
||||||
};
|
|
||||||
|
|
||||||
// First try to find a memory that is both coherent and cached
|
|
||||||
find_mem_type_with_flag(vk::MemoryPropertyFlagBits::eHostCoherent |
|
|
||||||
vk::MemoryPropertyFlagBits::eHostCached);
|
|
||||||
if (mapped_memory_type == -1)
|
|
||||||
// Then only coherent (lower performance)
|
|
||||||
find_mem_type_with_flag(vk::MemoryPropertyFlagBits::eHostCoherent);
|
|
||||||
|
|
||||||
if (mapped_memory_type == -1) {
|
|
||||||
LOG_CRITICAL(Render_Vulkan, "No coherent memory available for memory mapping");
|
|
||||||
mapped_memory_type = std::countr_zero(host_mem_props.memoryTypeBits);
|
|
||||||
}
|
|
||||||
|
|
||||||
const vk::StructureChain alloc_info = {
|
|
||||||
vk::MemoryAllocateInfo{
|
|
||||||
.allocationSize = size,
|
|
||||||
.memoryTypeIndex = static_cast<uint32_t>(mapped_memory_type),
|
|
||||||
},
|
|
||||||
vk::ImportMemoryHostPointerInfoEXT{
|
|
||||||
.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT,
|
|
||||||
.pHostPointer = host_pointer,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
const auto [it, new_memory] = mapped_memories.try_emplace(addr);
|
|
||||||
ASSERT_MSG(new_memory, "Attempting to remap already mapped vulkan memory");
|
|
||||||
|
|
||||||
auto& memory = it->second;
|
|
||||||
memory.backing = device.allocateMemoryUnique(alloc_info.get());
|
|
||||||
|
|
||||||
constexpr vk::BufferUsageFlags MapFlags =
|
|
||||||
vk::BufferUsageFlagBits::eIndexBuffer | vk::BufferUsageFlagBits::eVertexBuffer |
|
|
||||||
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst |
|
|
||||||
vk::BufferUsageFlagBits::eUniformBuffer | vk::BufferUsageFlagBits::eStorageBuffer;
|
|
||||||
|
|
||||||
const vk::StructureChain buffer_info = {
|
|
||||||
vk::BufferCreateInfo{
|
|
||||||
.size = size,
|
|
||||||
.usage = MapFlags,
|
|
||||||
.sharingMode = vk::SharingMode::eExclusive,
|
|
||||||
},
|
|
||||||
vk::ExternalMemoryBufferCreateInfoKHR{
|
|
||||||
.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT,
|
|
||||||
}};
|
|
||||||
memory.buffer = device.createBufferUnique(buffer_info.get());
|
|
||||||
device.bindBufferMemory(*memory.buffer, *memory.backing, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
void MemoryManager::UnmapVulkanMemory(VAddr addr, size_t size) {
|
|
||||||
return;
|
|
||||||
const auto it = mapped_memories.find(addr);
|
|
||||||
ASSERT(it != mapped_memories.end() && it->second.buffer_size == size);
|
|
||||||
mapped_memories.erase(it);
|
|
||||||
}
|
|
||||||
|
|
||||||
int MemoryManager::GetDirectMemoryType(PAddr addr, int* directMemoryTypeOut,
|
int MemoryManager::GetDirectMemoryType(PAddr addr, int* directMemoryTypeOut,
|
||||||
void** directMemoryStartOut, void** directMemoryEndOut) {
|
void** directMemoryStartOut, void** directMemoryEndOut) {
|
||||||
std::scoped_lock lk{mutex};
|
std::scoped_lock lk{mutex};
|
||||||
|
|
|
@ -3,20 +3,17 @@
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <functional>
|
#include <map>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <vector>
|
|
||||||
#include <boost/icl/split_interval_map.hpp>
|
|
||||||
#include "common/enum.h"
|
#include "common/enum.h"
|
||||||
#include "common/singleton.h"
|
#include "common/singleton.h"
|
||||||
#include "common/types.h"
|
#include "common/types.h"
|
||||||
#include "core/address_space.h"
|
#include "core/address_space.h"
|
||||||
#include "core/libraries/kernel/memory_management.h"
|
#include "core/libraries/kernel/memory_management.h"
|
||||||
#include "video_core/renderer_vulkan/vk_common.h"
|
|
||||||
|
|
||||||
namespace Vulkan {
|
namespace Vulkan {
|
||||||
class Instance;
|
class Rasterizer;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace Libraries::Kernel {
|
namespace Libraries::Kernel {
|
||||||
|
@ -128,8 +125,8 @@ public:
|
||||||
explicit MemoryManager();
|
explicit MemoryManager();
|
||||||
~MemoryManager();
|
~MemoryManager();
|
||||||
|
|
||||||
void SetInstance(const Vulkan::Instance* instance_) {
|
void SetRasterizer(Vulkan::Rasterizer* rasterizer_) {
|
||||||
instance = instance_;
|
rasterizer = rasterizer_;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SetTotalFlexibleSize(u64 size) {
|
void SetTotalFlexibleSize(u64 size) {
|
||||||
|
@ -140,9 +137,7 @@ public:
|
||||||
return total_flexible_size - flexible_usage;
|
return total_flexible_size - flexible_usage;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the offset of the mapped virtual system managed memory base from where it usually
|
VAddr SystemReservedVirtualBase() noexcept {
|
||||||
/// would be mapped.
|
|
||||||
[[nodiscard]] VAddr SystemReservedVirtualBase() noexcept {
|
|
||||||
return impl.SystemReservedVirtualBase();
|
return impl.SystemReservedVirtualBase();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -172,8 +167,6 @@ public:
|
||||||
int DirectQueryAvailable(PAddr search_start, PAddr search_end, size_t alignment,
|
int DirectQueryAvailable(PAddr search_start, PAddr search_end, size_t alignment,
|
||||||
PAddr* phys_addr_out, size_t* size_out);
|
PAddr* phys_addr_out, size_t* size_out);
|
||||||
|
|
||||||
std::pair<vk::Buffer, size_t> GetVulkanBuffer(VAddr addr);
|
|
||||||
|
|
||||||
int GetDirectMemoryType(PAddr addr, int* directMemoryTypeOut, void** directMemoryStartOut,
|
int GetDirectMemoryType(PAddr addr, int* directMemoryTypeOut, void** directMemoryStartOut,
|
||||||
void** directMemoryEndOut);
|
void** directMemoryEndOut);
|
||||||
|
|
||||||
|
@ -218,10 +211,6 @@ private:
|
||||||
|
|
||||||
DMemHandle Split(DMemHandle dmem_handle, size_t offset_in_area);
|
DMemHandle Split(DMemHandle dmem_handle, size_t offset_in_area);
|
||||||
|
|
||||||
void MapVulkanMemory(VAddr addr, size_t size);
|
|
||||||
|
|
||||||
void UnmapVulkanMemory(VAddr addr, size_t size);
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
AddressSpace impl;
|
AddressSpace impl;
|
||||||
DMemMap dmem_map;
|
DMemMap dmem_map;
|
||||||
|
@ -229,14 +218,7 @@ private:
|
||||||
std::recursive_mutex mutex;
|
std::recursive_mutex mutex;
|
||||||
size_t total_flexible_size = 448_MB;
|
size_t total_flexible_size = 448_MB;
|
||||||
size_t flexible_usage{};
|
size_t flexible_usage{};
|
||||||
|
Vulkan::Rasterizer* rasterizer{};
|
||||||
struct MappedMemory {
|
|
||||||
vk::UniqueBuffer buffer;
|
|
||||||
vk::UniqueDeviceMemory backing;
|
|
||||||
size_t buffer_size;
|
|
||||||
};
|
|
||||||
std::map<VAddr, MappedMemory> mapped_memories;
|
|
||||||
const Vulkan::Instance* instance{};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
using Memory = Common::Singleton<MemoryManager>;
|
using Memory = Common::Singleton<MemoryManager>;
|
||||||
|
|
|
@ -88,6 +88,7 @@ void Module::LoadModuleToMemory(u32& max_tls_index) {
|
||||||
aligned_base_size + TrampolineSize, MemoryProt::CpuReadWrite,
|
aligned_base_size + TrampolineSize, MemoryProt::CpuReadWrite,
|
||||||
MemoryMapFlags::Fixed, VMAType::Code, name, true);
|
MemoryMapFlags::Fixed, VMAType::Code, name, true);
|
||||||
LoadOffset += CODE_BASE_INCR * (1 + aligned_base_size / CODE_BASE_INCR);
|
LoadOffset += CODE_BASE_INCR * (1 + aligned_base_size / CODE_BASE_INCR);
|
||||||
|
LOG_INFO(Core_Linker, "Loading module {} to {}", name, fmt::ptr(*out_addr));
|
||||||
|
|
||||||
// Initialize trampoline generator.
|
// Initialize trampoline generator.
|
||||||
void* trampoline_addr = std::bit_cast<void*>(base_virtual_addr + aligned_base_size);
|
void* trampoline_addr = std::bit_cast<void*>(base_virtual_addr + aligned_base_size);
|
||||||
|
|
|
@ -21,6 +21,7 @@ Id VsOutputAttrPointer(EmitContext& ctx, VsOutput output) {
|
||||||
case VsOutput::ClipDist7: {
|
case VsOutput::ClipDist7: {
|
||||||
const u32 index = u32(output) - u32(VsOutput::ClipDist0);
|
const u32 index = u32(output) - u32(VsOutput::ClipDist0);
|
||||||
const Id clip_num{ctx.ConstU32(index)};
|
const Id clip_num{ctx.ConstU32(index)};
|
||||||
|
ASSERT_MSG(Sirit::ValidId(ctx.clip_distances), "Clip distance used but not defined");
|
||||||
return ctx.OpAccessChain(ctx.output_f32, ctx.clip_distances, clip_num);
|
return ctx.OpAccessChain(ctx.output_f32, ctx.clip_distances, clip_num);
|
||||||
}
|
}
|
||||||
case VsOutput::CullDist0:
|
case VsOutput::CullDist0:
|
||||||
|
@ -33,6 +34,7 @@ Id VsOutputAttrPointer(EmitContext& ctx, VsOutput output) {
|
||||||
case VsOutput::CullDist7: {
|
case VsOutput::CullDist7: {
|
||||||
const u32 index = u32(output) - u32(VsOutput::CullDist0);
|
const u32 index = u32(output) - u32(VsOutput::CullDist0);
|
||||||
const Id cull_num{ctx.ConstU32(index)};
|
const Id cull_num{ctx.ConstU32(index)};
|
||||||
|
ASSERT_MSG(Sirit::ValidId(ctx.cull_distances), "Cull distance used but not defined");
|
||||||
return ctx.OpAccessChain(ctx.output_f32, ctx.cull_distances, cull_num);
|
return ctx.OpAccessChain(ctx.output_f32, ctx.cull_distances, cull_num);
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
|
@ -125,7 +127,12 @@ Id EmitReadConst(EmitContext& ctx) {
|
||||||
}
|
}
|
||||||
|
|
||||||
Id EmitReadConstBuffer(EmitContext& ctx, u32 handle, Id index) {
|
Id EmitReadConstBuffer(EmitContext& ctx, u32 handle, Id index) {
|
||||||
const auto& buffer = ctx.buffers[handle];
|
auto& buffer = ctx.buffers[handle];
|
||||||
|
if (!Sirit::ValidId(buffer.offset)) {
|
||||||
|
buffer.offset = ctx.GetBufferOffset(handle);
|
||||||
|
}
|
||||||
|
const Id offset_dwords{ctx.OpShiftRightLogical(ctx.U32[1], buffer.offset, ctx.ConstU32(2U))};
|
||||||
|
index = ctx.OpIAdd(ctx.U32[1], index, offset_dwords);
|
||||||
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};
|
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};
|
||||||
return ctx.OpLoad(buffer.data_types->Get(1), ptr);
|
return ctx.OpLoad(buffer.data_types->Get(1), ptr);
|
||||||
}
|
}
|
||||||
|
@ -137,7 +144,7 @@ Id EmitReadConstBufferU32(EmitContext& ctx, u32 handle, Id index) {
|
||||||
Id EmitReadStepRate(EmitContext& ctx, int rate_idx) {
|
Id EmitReadStepRate(EmitContext& ctx, int rate_idx) {
|
||||||
return ctx.OpLoad(
|
return ctx.OpLoad(
|
||||||
ctx.U32[1], ctx.OpAccessChain(ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U32[1]),
|
ctx.U32[1], ctx.OpAccessChain(ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U32[1]),
|
||||||
ctx.instance_step_rates,
|
ctx.push_data_block,
|
||||||
rate_idx == 0 ? ctx.u32_zero_value : ctx.u32_one_value));
|
rate_idx == 0 ? ctx.u32_zero_value : ctx.u32_one_value));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -221,7 +228,11 @@ Id EmitLoadBufferU32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address) {
|
||||||
|
|
||||||
template <u32 N>
|
template <u32 N>
|
||||||
static Id EmitLoadBufferF32xN(EmitContext& ctx, u32 handle, Id address) {
|
static Id EmitLoadBufferF32xN(EmitContext& ctx, u32 handle, Id address) {
|
||||||
const auto& buffer = ctx.buffers[handle];
|
auto& buffer = ctx.buffers[handle];
|
||||||
|
if (!Sirit::ValidId(buffer.offset)) {
|
||||||
|
buffer.offset = ctx.GetBufferOffset(handle);
|
||||||
|
}
|
||||||
|
address = ctx.OpIAdd(ctx.U32[1], address, buffer.offset);
|
||||||
const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u));
|
const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u));
|
||||||
if constexpr (N == 1) {
|
if constexpr (N == 1) {
|
||||||
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};
|
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};
|
||||||
|
@ -314,7 +325,7 @@ static Id ComponentOffset(EmitContext& ctx, Id address, u32 stride, u32 bit_offs
|
||||||
}
|
}
|
||||||
|
|
||||||
static Id GetBufferFormatValue(EmitContext& ctx, u32 handle, Id address, u32 comp) {
|
static Id GetBufferFormatValue(EmitContext& ctx, u32 handle, Id address, u32 comp) {
|
||||||
const auto& buffer = ctx.buffers[handle];
|
auto& buffer = ctx.buffers[handle];
|
||||||
const auto format = buffer.buffer.GetDataFmt();
|
const auto format = buffer.buffer.GetDataFmt();
|
||||||
switch (format) {
|
switch (format) {
|
||||||
case AmdGpu::DataFormat::FormatInvalid:
|
case AmdGpu::DataFormat::FormatInvalid:
|
||||||
|
@ -399,6 +410,11 @@ static Id GetBufferFormatValue(EmitContext& ctx, u32 handle, Id address, u32 com
|
||||||
|
|
||||||
template <u32 N>
|
template <u32 N>
|
||||||
static Id EmitLoadBufferFormatF32xN(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address) {
|
static Id EmitLoadBufferFormatF32xN(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address) {
|
||||||
|
auto& buffer = ctx.buffers[handle];
|
||||||
|
if (!Sirit::ValidId(buffer.offset)) {
|
||||||
|
buffer.offset = ctx.GetBufferOffset(handle);
|
||||||
|
}
|
||||||
|
address = ctx.OpIAdd(ctx.U32[1], address, buffer.offset);
|
||||||
if constexpr (N == 1) {
|
if constexpr (N == 1) {
|
||||||
return GetBufferFormatValue(ctx, handle, address, 0);
|
return GetBufferFormatValue(ctx, handle, address, 0);
|
||||||
} else {
|
} else {
|
||||||
|
@ -428,7 +444,11 @@ Id EmitLoadBufferFormatF32x4(EmitContext& ctx, IR::Inst* inst, u32 handle, Id ad
|
||||||
|
|
||||||
template <u32 N>
|
template <u32 N>
|
||||||
static void EmitStoreBufferF32xN(EmitContext& ctx, u32 handle, Id address, Id value) {
|
static void EmitStoreBufferF32xN(EmitContext& ctx, u32 handle, Id address, Id value) {
|
||||||
const auto& buffer = ctx.buffers[handle];
|
auto& buffer = ctx.buffers[handle];
|
||||||
|
if (!Sirit::ValidId(buffer.offset)) {
|
||||||
|
buffer.offset = ctx.GetBufferOffset(handle);
|
||||||
|
}
|
||||||
|
address = ctx.OpIAdd(ctx.U32[1], address, buffer.offset);
|
||||||
const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u));
|
const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u));
|
||||||
if constexpr (N == 1) {
|
if constexpr (N == 1) {
|
||||||
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};
|
const Id ptr{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index)};
|
||||||
|
|
|
@ -46,9 +46,9 @@ EmitContext::EmitContext(const Profile& profile_, IR::Program& program, u32& bin
|
||||||
stage{program.info.stage}, binding{binding_} {
|
stage{program.info.stage}, binding{binding_} {
|
||||||
AddCapability(spv::Capability::Shader);
|
AddCapability(spv::Capability::Shader);
|
||||||
DefineArithmeticTypes();
|
DefineArithmeticTypes();
|
||||||
DefineInterfaces(program);
|
DefineInterfaces();
|
||||||
DefineBuffers(info);
|
DefineBuffers();
|
||||||
DefineImagesAndSamplers(info);
|
DefineImagesAndSamplers();
|
||||||
DefineSharedMemory();
|
DefineSharedMemory();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -117,9 +117,10 @@ void EmitContext::DefineArithmeticTypes() {
|
||||||
full_result_u32x2 = Name(TypeStruct(U32[1], U32[1]), "full_result_u32x2");
|
full_result_u32x2 = Name(TypeStruct(U32[1], U32[1]), "full_result_u32x2");
|
||||||
}
|
}
|
||||||
|
|
||||||
void EmitContext::DefineInterfaces(const IR::Program& program) {
|
void EmitContext::DefineInterfaces() {
|
||||||
DefineInputs(program.info);
|
DefinePushDataBlock();
|
||||||
DefineOutputs(program.info);
|
DefineInputs();
|
||||||
|
DefineOutputs();
|
||||||
}
|
}
|
||||||
|
|
||||||
Id GetAttributeType(EmitContext& ctx, AmdGpu::NumberFormat fmt) {
|
Id GetAttributeType(EmitContext& ctx, AmdGpu::NumberFormat fmt) {
|
||||||
|
@ -164,6 +165,16 @@ EmitContext::SpirvAttribute EmitContext::GetAttributeInfo(AmdGpu::NumberFormat f
|
||||||
throw InvalidArgument("Invalid attribute type {}", fmt);
|
throw InvalidArgument("Invalid attribute type {}", fmt);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Id EmitContext::GetBufferOffset(u32 binding) {
|
||||||
|
const u32 half = Shader::PushData::BufOffsetIndex + (binding >> 4);
|
||||||
|
const u32 comp = (binding & 0xf) >> 2;
|
||||||
|
const u32 offset = (binding & 0x3) << 3;
|
||||||
|
const Id ptr{OpAccessChain(TypePointer(spv::StorageClass::PushConstant, U32[1]),
|
||||||
|
push_data_block, ConstU32(half), ConstU32(comp))};
|
||||||
|
const Id value{OpLoad(U32[1], ptr)};
|
||||||
|
return OpBitFieldUExtract(U32[1], value, ConstU32(offset), ConstU32(8U));
|
||||||
|
}
|
||||||
|
|
||||||
Id MakeDefaultValue(EmitContext& ctx, u32 default_value) {
|
Id MakeDefaultValue(EmitContext& ctx, u32 default_value) {
|
||||||
switch (default_value) {
|
switch (default_value) {
|
||||||
case 0:
|
case 0:
|
||||||
|
@ -179,24 +190,13 @@ Id MakeDefaultValue(EmitContext& ctx, u32 default_value) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void EmitContext::DefineInputs(const Info& info) {
|
void EmitContext::DefineInputs() {
|
||||||
switch (stage) {
|
switch (stage) {
|
||||||
case Stage::Vertex: {
|
case Stage::Vertex: {
|
||||||
vertex_index = DefineVariable(U32[1], spv::BuiltIn::VertexIndex, spv::StorageClass::Input);
|
vertex_index = DefineVariable(U32[1], spv::BuiltIn::VertexIndex, spv::StorageClass::Input);
|
||||||
base_vertex = DefineVariable(U32[1], spv::BuiltIn::BaseVertex, spv::StorageClass::Input);
|
base_vertex = DefineVariable(U32[1], spv::BuiltIn::BaseVertex, spv::StorageClass::Input);
|
||||||
instance_id = DefineVariable(U32[1], spv::BuiltIn::InstanceIndex, spv::StorageClass::Input);
|
instance_id = DefineVariable(U32[1], spv::BuiltIn::InstanceIndex, spv::StorageClass::Input);
|
||||||
|
|
||||||
// Create push constants block for instance steps rates
|
|
||||||
const Id struct_type{Name(TypeStruct(U32[1], U32[1]), "instance_step_rates")};
|
|
||||||
Decorate(struct_type, spv::Decoration::Block);
|
|
||||||
MemberName(struct_type, 0, "sr0");
|
|
||||||
MemberName(struct_type, 1, "sr1");
|
|
||||||
MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U);
|
|
||||||
MemberDecorate(struct_type, 1, spv::Decoration::Offset, 4U);
|
|
||||||
instance_step_rates = DefineVar(struct_type, spv::StorageClass::PushConstant);
|
|
||||||
Name(instance_step_rates, "step_rates");
|
|
||||||
interfaces.push_back(instance_step_rates);
|
|
||||||
|
|
||||||
for (const auto& input : info.vs_inputs) {
|
for (const auto& input : info.vs_inputs) {
|
||||||
const Id type{GetAttributeType(*this, input.fmt)};
|
const Id type{GetAttributeType(*this, input.fmt)};
|
||||||
if (input.instance_step_rate == Info::VsInput::InstanceIdType::OverStepRate0 ||
|
if (input.instance_step_rate == Info::VsInput::InstanceIdType::OverStepRate0 ||
|
||||||
|
@ -260,19 +260,20 @@ void EmitContext::DefineInputs(const Info& info) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void EmitContext::DefineOutputs(const Info& info) {
|
void EmitContext::DefineOutputs() {
|
||||||
switch (stage) {
|
switch (stage) {
|
||||||
case Stage::Vertex: {
|
case Stage::Vertex: {
|
||||||
output_position = DefineVariable(F32[4], spv::BuiltIn::Position, spv::StorageClass::Output);
|
output_position = DefineVariable(F32[4], spv::BuiltIn::Position, spv::StorageClass::Output);
|
||||||
const std::array<Id, 8> zero{f32_zero_value, f32_zero_value, f32_zero_value,
|
const bool has_extra_pos_stores = info.stores.Get(IR::Attribute::Position1) ||
|
||||||
f32_zero_value, f32_zero_value, f32_zero_value,
|
info.stores.Get(IR::Attribute::Position2) ||
|
||||||
f32_zero_value, f32_zero_value};
|
info.stores.Get(IR::Attribute::Position3);
|
||||||
|
if (has_extra_pos_stores) {
|
||||||
const Id type{TypeArray(F32[1], ConstU32(8U))};
|
const Id type{TypeArray(F32[1], ConstU32(8U))};
|
||||||
const Id initializer{ConstantComposite(type, zero)};
|
clip_distances =
|
||||||
clip_distances = DefineVariable(type, spv::BuiltIn::ClipDistance, spv::StorageClass::Output,
|
DefineVariable(type, spv::BuiltIn::ClipDistance, spv::StorageClass::Output);
|
||||||
initializer);
|
cull_distances =
|
||||||
cull_distances = DefineVariable(type, spv::BuiltIn::CullDistance, spv::StorageClass::Output,
|
DefineVariable(type, spv::BuiltIn::CullDistance, spv::StorageClass::Output);
|
||||||
initializer);
|
}
|
||||||
for (u32 i = 0; i < IR::NumParams; i++) {
|
for (u32 i = 0; i < IR::NumParams; i++) {
|
||||||
const IR::Attribute param{IR::Attribute::Param0 + i};
|
const IR::Attribute param{IR::Attribute::Param0 + i};
|
||||||
if (!info.stores.GetAny(param)) {
|
if (!info.stores.GetAny(param)) {
|
||||||
|
@ -304,7 +305,24 @@ void EmitContext::DefineOutputs(const Info& info) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void EmitContext::DefineBuffers(const Info& info) {
|
void EmitContext::DefinePushDataBlock() {
|
||||||
|
// Create push constants block for instance steps rates
|
||||||
|
const Id struct_type{Name(TypeStruct(U32[1], U32[1], U32[4], U32[4]), "AuxData")};
|
||||||
|
Decorate(struct_type, spv::Decoration::Block);
|
||||||
|
MemberName(struct_type, 0, "sr0");
|
||||||
|
MemberName(struct_type, 1, "sr1");
|
||||||
|
MemberName(struct_type, 2, "buf_offsets0");
|
||||||
|
MemberName(struct_type, 3, "buf_offsets1");
|
||||||
|
MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U);
|
||||||
|
MemberDecorate(struct_type, 1, spv::Decoration::Offset, 4U);
|
||||||
|
MemberDecorate(struct_type, 2, spv::Decoration::Offset, 8U);
|
||||||
|
MemberDecorate(struct_type, 3, spv::Decoration::Offset, 24U);
|
||||||
|
push_data_block = DefineVar(struct_type, spv::StorageClass::PushConstant);
|
||||||
|
Name(push_data_block, "push_data");
|
||||||
|
interfaces.push_back(push_data_block);
|
||||||
|
}
|
||||||
|
|
||||||
|
void EmitContext::DefineBuffers() {
|
||||||
boost::container::small_vector<Id, 8> type_ids;
|
boost::container::small_vector<Id, 8> type_ids;
|
||||||
for (u32 i = 0; const auto& buffer : info.buffers) {
|
for (u32 i = 0; const auto& buffer : info.buffers) {
|
||||||
const auto* data_types = True(buffer.used_types & IR::Type::F32) ? &F32 : &U32;
|
const auto* data_types = True(buffer.used_types & IR::Type::F32) ? &F32 : &U32;
|
||||||
|
@ -322,8 +340,8 @@ void EmitContext::DefineBuffers(const Info& info) {
|
||||||
Decorate(struct_type, spv::Decoration::Block);
|
Decorate(struct_type, spv::Decoration::Block);
|
||||||
MemberName(struct_type, 0, "data");
|
MemberName(struct_type, 0, "data");
|
||||||
MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U);
|
MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U);
|
||||||
}
|
|
||||||
type_ids.push_back(record_array_type);
|
type_ids.push_back(record_array_type);
|
||||||
|
}
|
||||||
|
|
||||||
const auto storage_class =
|
const auto storage_class =
|
||||||
buffer.is_storage ? spv::StorageClass::StorageBuffer : spv::StorageClass::Uniform;
|
buffer.is_storage ? spv::StorageClass::StorageBuffer : spv::StorageClass::Uniform;
|
||||||
|
@ -430,7 +448,7 @@ Id ImageType(EmitContext& ctx, const ImageResource& desc, Id sampled_type) {
|
||||||
throw InvalidArgument("Invalid texture type {}", desc.type);
|
throw InvalidArgument("Invalid texture type {}", desc.type);
|
||||||
}
|
}
|
||||||
|
|
||||||
void EmitContext::DefineImagesAndSamplers(const Info& info) {
|
void EmitContext::DefineImagesAndSamplers() {
|
||||||
for (const auto& image_desc : info.images) {
|
for (const auto& image_desc : info.images) {
|
||||||
const VectorIds* data_types = [&] {
|
const VectorIds* data_types = [&] {
|
||||||
switch (image_desc.nfmt) {
|
switch (image_desc.nfmt) {
|
||||||
|
|
|
@ -40,6 +40,7 @@ public:
|
||||||
~EmitContext();
|
~EmitContext();
|
||||||
|
|
||||||
Id Def(const IR::Value& value);
|
Id Def(const IR::Value& value);
|
||||||
|
Id GetBufferOffset(u32 binding);
|
||||||
|
|
||||||
[[nodiscard]] Id DefineInput(Id type, u32 location) {
|
[[nodiscard]] Id DefineInput(Id type, u32 location) {
|
||||||
const Id input_id{DefineVar(type, spv::StorageClass::Input)};
|
const Id input_id{DefineVar(type, spv::StorageClass::Input)};
|
||||||
|
@ -168,7 +169,7 @@ public:
|
||||||
Id output_position{};
|
Id output_position{};
|
||||||
Id vertex_index{};
|
Id vertex_index{};
|
||||||
Id instance_id{};
|
Id instance_id{};
|
||||||
Id instance_step_rates{};
|
Id push_data_block{};
|
||||||
Id base_vertex{};
|
Id base_vertex{};
|
||||||
Id frag_coord{};
|
Id frag_coord{};
|
||||||
Id front_facing{};
|
Id front_facing{};
|
||||||
|
@ -201,14 +202,15 @@ public:
|
||||||
|
|
||||||
struct BufferDefinition {
|
struct BufferDefinition {
|
||||||
Id id;
|
Id id;
|
||||||
|
Id offset;
|
||||||
const VectorIds* data_types;
|
const VectorIds* data_types;
|
||||||
Id pointer_type;
|
Id pointer_type;
|
||||||
AmdGpu::Buffer buffer;
|
AmdGpu::Buffer buffer;
|
||||||
};
|
};
|
||||||
|
|
||||||
u32& binding;
|
u32& binding;
|
||||||
boost::container::small_vector<BufferDefinition, 4> buffers;
|
boost::container::small_vector<BufferDefinition, 16> buffers;
|
||||||
boost::container::small_vector<TextureDefinition, 4> images;
|
boost::container::small_vector<TextureDefinition, 8> images;
|
||||||
boost::container::small_vector<Id, 4> samplers;
|
boost::container::small_vector<Id, 4> samplers;
|
||||||
|
|
||||||
Id sampler_type{};
|
Id sampler_type{};
|
||||||
|
@ -227,11 +229,12 @@ public:
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void DefineArithmeticTypes();
|
void DefineArithmeticTypes();
|
||||||
void DefineInterfaces(const IR::Program& program);
|
void DefineInterfaces();
|
||||||
void DefineInputs(const Info& info);
|
void DefineInputs();
|
||||||
void DefineOutputs(const Info& info);
|
void DefineOutputs();
|
||||||
void DefineBuffers(const Info& info);
|
void DefinePushDataBlock();
|
||||||
void DefineImagesAndSamplers(const Info& info);
|
void DefineBuffers();
|
||||||
|
void DefineImagesAndSamplers();
|
||||||
void DefineSharedMemory();
|
void DefineSharedMemory();
|
||||||
|
|
||||||
SpirvAttribute GetAttributeInfo(AmdGpu::NumberFormat fmt, Id id);
|
SpirvAttribute GetAttributeInfo(AmdGpu::NumberFormat fmt, Id id);
|
||||||
|
|
|
@ -40,7 +40,7 @@ static IR::Condition MakeCondition(Opcode opcode) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
CFG::CFG(ObjectPool<Block>& block_pool_, std::span<const GcnInst> inst_list_)
|
CFG::CFG(Common::ObjectPool<Block>& block_pool_, std::span<const GcnInst> inst_list_)
|
||||||
: block_pool{block_pool_}, inst_list{inst_list_} {
|
: block_pool{block_pool_}, inst_list{inst_list_} {
|
||||||
index_to_pc.resize(inst_list.size() + 1);
|
index_to_pc.resize(inst_list.size() + 1);
|
||||||
EmitLabels();
|
EmitLabels();
|
||||||
|
|
|
@ -8,10 +8,10 @@
|
||||||
#include <boost/container/small_vector.hpp>
|
#include <boost/container/small_vector.hpp>
|
||||||
#include <boost/intrusive/set.hpp>
|
#include <boost/intrusive/set.hpp>
|
||||||
|
|
||||||
|
#include "common/object_pool.h"
|
||||||
#include "common/types.h"
|
#include "common/types.h"
|
||||||
#include "shader_recompiler/frontend/instruction.h"
|
#include "shader_recompiler/frontend/instruction.h"
|
||||||
#include "shader_recompiler/ir/condition.h"
|
#include "shader_recompiler/ir/condition.h"
|
||||||
#include "shader_recompiler/object_pool.h"
|
|
||||||
|
|
||||||
namespace Shader::Gcn {
|
namespace Shader::Gcn {
|
||||||
|
|
||||||
|
@ -49,7 +49,7 @@ class CFG {
|
||||||
using Label = u32;
|
using Label = u32;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit CFG(ObjectPool<Block>& block_pool, std::span<const GcnInst> inst_list);
|
explicit CFG(Common::ObjectPool<Block>& block_pool, std::span<const GcnInst> inst_list);
|
||||||
|
|
||||||
[[nodiscard]] std::string Dot() const;
|
[[nodiscard]] std::string Dot() const;
|
||||||
|
|
||||||
|
@ -59,7 +59,7 @@ private:
|
||||||
void LinkBlocks();
|
void LinkBlocks();
|
||||||
|
|
||||||
public:
|
public:
|
||||||
ObjectPool<Block>& block_pool;
|
Common::ObjectPool<Block>& block_pool;
|
||||||
std::span<const GcnInst> inst_list;
|
std::span<const GcnInst> inst_list;
|
||||||
std::vector<u32> index_to_pc;
|
std::vector<u32> index_to_pc;
|
||||||
boost::container::small_vector<Label, 16> labels;
|
boost::container::small_vector<Label, 16> labels;
|
||||||
|
|
|
@ -1,10 +0,0 @@
|
||||||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
|
||||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
namespace Shader::Gcn {
|
|
||||||
|
|
||||||
void Translate();
|
|
||||||
|
|
||||||
} // namespace Shader::Gcn
|
|
|
@ -287,7 +287,7 @@ bool NeedsLift(Node goto_stmt, Node label_stmt) noexcept {
|
||||||
*/
|
*/
|
||||||
class GotoPass {
|
class GotoPass {
|
||||||
public:
|
public:
|
||||||
explicit GotoPass(CFG& cfg, ObjectPool<Statement>& stmt_pool) : pool{stmt_pool} {
|
explicit GotoPass(CFG& cfg, Common::ObjectPool<Statement>& stmt_pool) : pool{stmt_pool} {
|
||||||
std::vector gotos{BuildTree(cfg)};
|
std::vector gotos{BuildTree(cfg)};
|
||||||
const auto end{gotos.rend()};
|
const auto end{gotos.rend()};
|
||||||
for (auto goto_stmt = gotos.rbegin(); goto_stmt != end; ++goto_stmt) {
|
for (auto goto_stmt = gotos.rbegin(); goto_stmt != end; ++goto_stmt) {
|
||||||
|
@ -563,7 +563,7 @@ private:
|
||||||
return parent_tree.insert(std::next(loop), *new_goto);
|
return parent_tree.insert(std::next(loop), *new_goto);
|
||||||
}
|
}
|
||||||
|
|
||||||
ObjectPool<Statement>& pool;
|
Common::ObjectPool<Statement>& pool;
|
||||||
Statement root_stmt{FunctionTag{}};
|
Statement root_stmt{FunctionTag{}};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -597,8 +597,9 @@ private:
|
||||||
|
|
||||||
class TranslatePass {
|
class TranslatePass {
|
||||||
public:
|
public:
|
||||||
TranslatePass(ObjectPool<IR::Inst>& inst_pool_, ObjectPool<IR::Block>& block_pool_,
|
TranslatePass(Common::ObjectPool<IR::Inst>& inst_pool_,
|
||||||
ObjectPool<Statement>& stmt_pool_, Statement& root_stmt,
|
Common::ObjectPool<IR::Block>& block_pool_,
|
||||||
|
Common::ObjectPool<Statement>& stmt_pool_, Statement& root_stmt,
|
||||||
IR::AbstractSyntaxList& syntax_list_, std::span<const GcnInst> inst_list_,
|
IR::AbstractSyntaxList& syntax_list_, std::span<const GcnInst> inst_list_,
|
||||||
Info& info_, const Profile& profile_)
|
Info& info_, const Profile& profile_)
|
||||||
: stmt_pool{stmt_pool_}, inst_pool{inst_pool_}, block_pool{block_pool_},
|
: stmt_pool{stmt_pool_}, inst_pool{inst_pool_}, block_pool{block_pool_},
|
||||||
|
@ -808,9 +809,9 @@ private:
|
||||||
return block_pool.Create(inst_pool);
|
return block_pool.Create(inst_pool);
|
||||||
}
|
}
|
||||||
|
|
||||||
ObjectPool<Statement>& stmt_pool;
|
Common::ObjectPool<Statement>& stmt_pool;
|
||||||
ObjectPool<IR::Inst>& inst_pool;
|
Common::ObjectPool<IR::Inst>& inst_pool;
|
||||||
ObjectPool<IR::Block>& block_pool;
|
Common::ObjectPool<IR::Block>& block_pool;
|
||||||
IR::AbstractSyntaxList& syntax_list;
|
IR::AbstractSyntaxList& syntax_list;
|
||||||
const Block dummy_flow_block{.is_dummy = true};
|
const Block dummy_flow_block{.is_dummy = true};
|
||||||
std::span<const GcnInst> inst_list;
|
std::span<const GcnInst> inst_list;
|
||||||
|
@ -819,9 +820,10 @@ private:
|
||||||
};
|
};
|
||||||
} // Anonymous namespace
|
} // Anonymous namespace
|
||||||
|
|
||||||
IR::AbstractSyntaxList BuildASL(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Block>& block_pool,
|
IR::AbstractSyntaxList BuildASL(Common::ObjectPool<IR::Inst>& inst_pool,
|
||||||
CFG& cfg, Info& info, const Profile& profile) {
|
Common::ObjectPool<IR::Block>& block_pool, CFG& cfg, Info& info,
|
||||||
ObjectPool<Statement> stmt_pool{64};
|
const Profile& profile) {
|
||||||
|
Common::ObjectPool<Statement> stmt_pool{64};
|
||||||
GotoPass goto_pass{cfg, stmt_pool};
|
GotoPass goto_pass{cfg, stmt_pool};
|
||||||
Statement& root{goto_pass.RootStatement()};
|
Statement& root{goto_pass.RootStatement()};
|
||||||
IR::AbstractSyntaxList syntax_list;
|
IR::AbstractSyntaxList syntax_list;
|
||||||
|
|
|
@ -7,7 +7,6 @@
|
||||||
#include "shader_recompiler/ir/abstract_syntax_list.h"
|
#include "shader_recompiler/ir/abstract_syntax_list.h"
|
||||||
#include "shader_recompiler/ir/basic_block.h"
|
#include "shader_recompiler/ir/basic_block.h"
|
||||||
#include "shader_recompiler/ir/value.h"
|
#include "shader_recompiler/ir/value.h"
|
||||||
#include "shader_recompiler/object_pool.h"
|
|
||||||
|
|
||||||
namespace Shader {
|
namespace Shader {
|
||||||
struct Info;
|
struct Info;
|
||||||
|
@ -16,8 +15,8 @@ struct Profile;
|
||||||
|
|
||||||
namespace Shader::Gcn {
|
namespace Shader::Gcn {
|
||||||
|
|
||||||
[[nodiscard]] IR::AbstractSyntaxList BuildASL(ObjectPool<IR::Inst>& inst_pool,
|
[[nodiscard]] IR::AbstractSyntaxList BuildASL(Common::ObjectPool<IR::Inst>& inst_pool,
|
||||||
ObjectPool<IR::Block>& block_pool, CFG& cfg,
|
Common::ObjectPool<IR::Block>& block_pool, CFG& cfg,
|
||||||
Info& info, const Profile& profile);
|
Info& info, const Profile& profile);
|
||||||
|
|
||||||
} // namespace Shader::Gcn
|
} // namespace Shader::Gcn
|
||||||
|
|
|
@ -447,6 +447,7 @@ void Translator::EmitFetch(const GcnInst& inst) {
|
||||||
.is_instance_data = true,
|
.is_instance_data = true,
|
||||||
});
|
});
|
||||||
instance_buf_handle = s32(info.buffers.size() - 1);
|
instance_buf_handle = s32(info.buffers.size() - 1);
|
||||||
|
info.uses_step_rates = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
const u32 num_components = AmdGpu::NumComponents(buffer.GetDataFmt());
|
const u32 num_components = AmdGpu::NumComponents(buffer.GetDataFmt());
|
||||||
|
|
|
@ -338,6 +338,11 @@ void Translator::BUFFER_LOAD_FORMAT(u32 num_dwords, bool is_typed, bool is_forma
|
||||||
if (is_typed) {
|
if (is_typed) {
|
||||||
info.dmft.Assign(static_cast<AmdGpu::DataFormat>(mtbuf.dfmt));
|
info.dmft.Assign(static_cast<AmdGpu::DataFormat>(mtbuf.dfmt));
|
||||||
info.nfmt.Assign(static_cast<AmdGpu::NumberFormat>(mtbuf.nfmt));
|
info.nfmt.Assign(static_cast<AmdGpu::NumberFormat>(mtbuf.nfmt));
|
||||||
|
ASSERT(info.nfmt == AmdGpu::NumberFormat::Float &&
|
||||||
|
(info.dmft == AmdGpu::DataFormat::Format32_32_32_32 ||
|
||||||
|
info.dmft == AmdGpu::DataFormat::Format32_32_32 ||
|
||||||
|
info.dmft == AmdGpu::DataFormat::Format32_32 ||
|
||||||
|
info.dmft == AmdGpu::DataFormat::Format32));
|
||||||
}
|
}
|
||||||
|
|
||||||
const IR::Value handle =
|
const IR::Value handle =
|
||||||
|
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
namespace Shader::IR {
|
namespace Shader::IR {
|
||||||
|
|
||||||
Block::Block(ObjectPool<Inst>& inst_pool_) : inst_pool{&inst_pool_} {}
|
Block::Block(Common::ObjectPool<Inst>& inst_pool_) : inst_pool{&inst_pool_} {}
|
||||||
|
|
||||||
Block::~Block() = default;
|
Block::~Block() = default;
|
||||||
|
|
||||||
|
|
|
@ -9,10 +9,10 @@
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <boost/intrusive/list.hpp>
|
#include <boost/intrusive/list.hpp>
|
||||||
|
|
||||||
|
#include "common/object_pool.h"
|
||||||
#include "common/types.h"
|
#include "common/types.h"
|
||||||
#include "shader_recompiler/ir/reg.h"
|
#include "shader_recompiler/ir/reg.h"
|
||||||
#include "shader_recompiler/ir/value.h"
|
#include "shader_recompiler/ir/value.h"
|
||||||
#include "shader_recompiler/object_pool.h"
|
|
||||||
|
|
||||||
namespace Shader::IR {
|
namespace Shader::IR {
|
||||||
|
|
||||||
|
@ -25,7 +25,7 @@ public:
|
||||||
using reverse_iterator = InstructionList::reverse_iterator;
|
using reverse_iterator = InstructionList::reverse_iterator;
|
||||||
using const_reverse_iterator = InstructionList::const_reverse_iterator;
|
using const_reverse_iterator = InstructionList::const_reverse_iterator;
|
||||||
|
|
||||||
explicit Block(ObjectPool<Inst>& inst_pool_);
|
explicit Block(Common::ObjectPool<Inst>& inst_pool_);
|
||||||
~Block();
|
~Block();
|
||||||
|
|
||||||
Block(const Block&) = delete;
|
Block(const Block&) = delete;
|
||||||
|
@ -153,7 +153,7 @@ public:
|
||||||
|
|
||||||
private:
|
private:
|
||||||
/// Memory pool for instruction list
|
/// Memory pool for instruction list
|
||||||
ObjectPool<Inst>* inst_pool;
|
Common::ObjectPool<Inst>* inst_pool;
|
||||||
|
|
||||||
/// List of instructions in this block
|
/// List of instructions in this block
|
||||||
InstructionList instructions;
|
InstructionList instructions;
|
||||||
|
|
|
@ -173,10 +173,9 @@ bool IsImageStorageInstruction(const IR::Inst& inst) {
|
||||||
|
|
||||||
class Descriptors {
|
class Descriptors {
|
||||||
public:
|
public:
|
||||||
explicit Descriptors(BufferResourceList& buffer_resources_, ImageResourceList& image_resources_,
|
explicit Descriptors(Info& info_)
|
||||||
SamplerResourceList& sampler_resources_)
|
: info{info_}, buffer_resources{info_.buffers}, image_resources{info_.images},
|
||||||
: buffer_resources{buffer_resources_}, image_resources{image_resources_},
|
sampler_resources{info_.samplers} {}
|
||||||
sampler_resources{sampler_resources_} {}
|
|
||||||
|
|
||||||
u32 Add(const BufferResource& desc) {
|
u32 Add(const BufferResource& desc) {
|
||||||
const u32 index{Add(buffer_resources, desc, [&desc](const auto& existing) {
|
const u32 index{Add(buffer_resources, desc, [&desc](const auto& existing) {
|
||||||
|
@ -188,6 +187,7 @@ public:
|
||||||
ASSERT(buffer.length == desc.length);
|
ASSERT(buffer.length == desc.length);
|
||||||
buffer.is_storage |= desc.is_storage;
|
buffer.is_storage |= desc.is_storage;
|
||||||
buffer.used_types |= desc.used_types;
|
buffer.used_types |= desc.used_types;
|
||||||
|
buffer.is_written |= desc.is_written;
|
||||||
return index;
|
return index;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -201,9 +201,16 @@ public:
|
||||||
}
|
}
|
||||||
|
|
||||||
u32 Add(const SamplerResource& desc) {
|
u32 Add(const SamplerResource& desc) {
|
||||||
const u32 index{Add(sampler_resources, desc, [&desc](const auto& existing) {
|
const u32 index{Add(sampler_resources, desc, [this, &desc](const auto& existing) {
|
||||||
return desc.sgpr_base == existing.sgpr_base &&
|
if (desc.sgpr_base == existing.sgpr_base &&
|
||||||
desc.dword_offset == existing.dword_offset;
|
desc.dword_offset == existing.dword_offset) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Samplers with different bindings might still be the same.
|
||||||
|
const auto old_sharp =
|
||||||
|
info.ReadUd<AmdGpu::Sampler>(existing.sgpr_base, existing.dword_offset);
|
||||||
|
const auto new_sharp = info.ReadUd<AmdGpu::Sampler>(desc.sgpr_base, desc.dword_offset);
|
||||||
|
return old_sharp == new_sharp;
|
||||||
})};
|
})};
|
||||||
return index;
|
return index;
|
||||||
}
|
}
|
||||||
|
@ -219,6 +226,7 @@ private:
|
||||||
return static_cast<u32>(descriptors.size()) - 1;
|
return static_cast<u32>(descriptors.size()) - 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const Info& info;
|
||||||
BufferResourceList& buffer_resources;
|
BufferResourceList& buffer_resources;
|
||||||
ImageResourceList& image_resources;
|
ImageResourceList& image_resources;
|
||||||
SamplerResourceList& sampler_resources;
|
SamplerResourceList& sampler_resources;
|
||||||
|
@ -328,16 +336,6 @@ static bool IsLoadBufferFormat(const IR::Inst& inst) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool IsReadConstBuffer(const IR::Inst& inst) {
|
|
||||||
switch (inst.GetOpcode()) {
|
|
||||||
case IR::Opcode::ReadConstBuffer:
|
|
||||||
case IR::Opcode::ReadConstBufferU32:
|
|
||||||
return true;
|
|
||||||
default:
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static u32 BufferLength(const AmdGpu::Buffer& buffer) {
|
static u32 BufferLength(const AmdGpu::Buffer& buffer) {
|
||||||
const auto stride = buffer.GetStride();
|
const auto stride = buffer.GetStride();
|
||||||
if (stride < sizeof(f32)) {
|
if (stride < sizeof(f32)) {
|
||||||
|
@ -401,30 +399,37 @@ void PatchBufferInstruction(IR::Block& block, IR::Inst& inst, Info& info,
|
||||||
IR::Inst* handle = inst.Arg(0).InstRecursive();
|
IR::Inst* handle = inst.Arg(0).InstRecursive();
|
||||||
IR::Inst* producer = handle->Arg(0).InstRecursive();
|
IR::Inst* producer = handle->Arg(0).InstRecursive();
|
||||||
const auto sharp = TrackSharp(producer);
|
const auto sharp = TrackSharp(producer);
|
||||||
|
const bool is_store = IsBufferStore(inst);
|
||||||
buffer = info.ReadUd<AmdGpu::Buffer>(sharp.sgpr_base, sharp.dword_offset);
|
buffer = info.ReadUd<AmdGpu::Buffer>(sharp.sgpr_base, sharp.dword_offset);
|
||||||
binding = descriptors.Add(BufferResource{
|
binding = descriptors.Add(BufferResource{
|
||||||
.sgpr_base = sharp.sgpr_base,
|
.sgpr_base = sharp.sgpr_base,
|
||||||
.dword_offset = sharp.dword_offset,
|
.dword_offset = sharp.dword_offset,
|
||||||
.length = BufferLength(buffer),
|
.length = BufferLength(buffer),
|
||||||
.used_types = BufferDataType(inst, buffer.GetNumberFmt()),
|
.used_types = BufferDataType(inst, buffer.GetNumberFmt()),
|
||||||
.is_storage = IsBufferStore(inst) || buffer.GetSize() > MaxUboSize,
|
.is_storage = is_store || buffer.GetSize() > MaxUboSize,
|
||||||
|
.is_written = is_store,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update buffer descriptor format.
|
||||||
const auto inst_info = inst.Flags<IR::BufferInstInfo>();
|
const auto inst_info = inst.Flags<IR::BufferInstInfo>();
|
||||||
IR::IREmitter ir{block, IR::Block::InstructionList::s_iterator_to(inst)};
|
auto& buffer_desc = info.buffers[binding];
|
||||||
// Replace handle with binding index in buffer resource list.
|
|
||||||
inst.SetArg(0, ir.Imm32(binding));
|
|
||||||
ASSERT(!buffer.swizzle_enable && !buffer.add_tid_enable);
|
|
||||||
if (inst_info.is_typed) {
|
if (inst_info.is_typed) {
|
||||||
ASSERT(inst_info.nfmt == AmdGpu::NumberFormat::Float &&
|
buffer_desc.dfmt = inst_info.dmft;
|
||||||
(inst_info.dmft == AmdGpu::DataFormat::Format32_32_32_32 ||
|
buffer_desc.nfmt = inst_info.nfmt;
|
||||||
inst_info.dmft == AmdGpu::DataFormat::Format32_32_32 ||
|
} else {
|
||||||
inst_info.dmft == AmdGpu::DataFormat::Format32_32 ||
|
buffer_desc.dfmt = buffer.GetDataFmt();
|
||||||
inst_info.dmft == AmdGpu::DataFormat::Format32));
|
buffer_desc.nfmt = buffer.GetNumberFmt();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (IsReadConstBuffer(inst)) {
|
// Replace handle with binding index in buffer resource list.
|
||||||
|
IR::IREmitter ir{block, IR::Block::InstructionList::s_iterator_to(inst)};
|
||||||
|
inst.SetArg(0, ir.Imm32(binding));
|
||||||
|
ASSERT(!buffer.swizzle_enable && !buffer.add_tid_enable);
|
||||||
|
|
||||||
|
// Address of constant buffer reads can be calculated at IR emittion time.
|
||||||
|
if (inst.GetOpcode() == IR::Opcode::ReadConstBuffer ||
|
||||||
|
inst.GetOpcode() == IR::Opcode::ReadConstBufferU32) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -434,10 +439,14 @@ void PatchBufferInstruction(IR::Block& block, IR::Inst& inst, Info& info,
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const u32 stride = buffer.GetStride();
|
const u32 stride = buffer.GetStride();
|
||||||
ASSERT_MSG(stride >= 4, "non-formatting load_buffer_* is not implemented for stride {}",
|
if (stride < 4) {
|
||||||
stride);
|
LOG_WARNING(Render_Vulkan,
|
||||||
|
"non-formatting load_buffer_* is not implemented for stride {}", stride);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Compute address of the buffer using the stride.
|
||||||
|
// Todo: What if buffer is rebound with different stride?
|
||||||
IR::U32 address = ir.Imm32(inst_info.inst_offset.Value());
|
IR::U32 address = ir.Imm32(inst_info.inst_offset.Value());
|
||||||
if (inst_info.index_enable) {
|
if (inst_info.index_enable) {
|
||||||
const IR::U32 index = inst_info.offset_enable ? IR::U32{ir.CompositeExtract(inst.Arg(1), 0)}
|
const IR::U32 index = inst_info.offset_enable ? IR::U32{ir.CompositeExtract(inst.Arg(1), 0)}
|
||||||
|
@ -587,39 +596,9 @@ void PatchImageInstruction(IR::Block& block, IR::Inst& inst, Info& info, Descrip
|
||||||
}
|
}
|
||||||
|
|
||||||
void ResourceTrackingPass(IR::Program& program) {
|
void ResourceTrackingPass(IR::Program& program) {
|
||||||
// When loading data from untyped buffer we don't have if it is float or integer.
|
|
||||||
// Most of the time it is float so that is the default. This pass detects float buffer loads
|
|
||||||
// combined with bitcasts and patches them to be integer loads.
|
|
||||||
for (IR::Block* const block : program.post_order_blocks) {
|
|
||||||
break;
|
|
||||||
for (IR::Inst& inst : block->Instructions()) {
|
|
||||||
if (inst.GetOpcode() != IR::Opcode::BitCastU32F32) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Replace the bitcast with a typed buffer read
|
|
||||||
IR::Inst* const arg_inst{inst.Arg(0).TryInstRecursive()};
|
|
||||||
if (!arg_inst) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const auto replace{[&](IR::Opcode new_opcode) {
|
|
||||||
inst.ReplaceOpcode(new_opcode);
|
|
||||||
inst.SetArg(0, arg_inst->Arg(0));
|
|
||||||
inst.SetArg(1, arg_inst->Arg(1));
|
|
||||||
inst.SetFlags(arg_inst->Flags<u32>());
|
|
||||||
arg_inst->Invalidate();
|
|
||||||
}};
|
|
||||||
if (arg_inst->GetOpcode() == IR::Opcode::ReadConstBuffer) {
|
|
||||||
replace(IR::Opcode::ReadConstBufferU32);
|
|
||||||
}
|
|
||||||
if (arg_inst->GetOpcode() == IR::Opcode::LoadBufferF32) {
|
|
||||||
replace(IR::Opcode::LoadBufferU32);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Iterate resource instructions and patch them after finding the sharp.
|
// Iterate resource instructions and patch them after finding the sharp.
|
||||||
auto& info = program.info;
|
auto& info = program.info;
|
||||||
Descriptors descriptors{info.buffers, info.images, info.samplers};
|
Descriptors descriptors{info};
|
||||||
for (IR::Block* const block : program.blocks) {
|
for (IR::Block* const block : program.blocks) {
|
||||||
for (IR::Inst& inst : block->Instructions()) {
|
for (IR::Inst& inst : block->Instructions()) {
|
||||||
if (IsBufferInstruction(inst)) {
|
if (IsBufferInstruction(inst)) {
|
||||||
|
|
|
@ -27,9 +27,9 @@ IR::BlockList GenerateBlocks(const IR::AbstractSyntaxList& syntax_list) {
|
||||||
return blocks;
|
return blocks;
|
||||||
}
|
}
|
||||||
|
|
||||||
IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Block>& block_pool,
|
IR::Program TranslateProgram(Common::ObjectPool<IR::Inst>& inst_pool,
|
||||||
std::span<const u32> token, const Info&& info,
|
Common::ObjectPool<IR::Block>& block_pool, std::span<const u32> token,
|
||||||
const Profile& profile) {
|
const Info&& info, const Profile& profile) {
|
||||||
// Ensure first instruction is expected.
|
// Ensure first instruction is expected.
|
||||||
constexpr u32 token_mov_vcchi = 0xBEEB03FF;
|
constexpr u32 token_mov_vcchi = 0xBEEB03FF;
|
||||||
ASSERT_MSG(token[0] == token_mov_vcchi, "First instruction is not s_mov_b32 vcc_hi, #imm");
|
ASSERT_MSG(token[0] == token_mov_vcchi, "First instruction is not s_mov_b32 vcc_hi, #imm");
|
||||||
|
@ -45,7 +45,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create control flow graph
|
// Create control flow graph
|
||||||
ObjectPool<Gcn::Block> gcn_block_pool{64};
|
Common::ObjectPool<Gcn::Block> gcn_block_pool{64};
|
||||||
Gcn::CFG cfg{gcn_block_pool, program.ins_list};
|
Gcn::CFG cfg{gcn_block_pool, program.ins_list};
|
||||||
|
|
||||||
// Structurize control flow graph and create program.
|
// Structurize control flow graph and create program.
|
||||||
|
@ -61,7 +61,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo
|
||||||
Shader::Optimization::IdentityRemovalPass(program.blocks);
|
Shader::Optimization::IdentityRemovalPass(program.blocks);
|
||||||
Shader::Optimization::DeadCodeEliminationPass(program);
|
Shader::Optimization::DeadCodeEliminationPass(program);
|
||||||
Shader::Optimization::CollectShaderInfoPass(program);
|
Shader::Optimization::CollectShaderInfoPass(program);
|
||||||
LOG_INFO(Render_Vulkan, "{}", Shader::IR::DumpProgram(program));
|
LOG_DEBUG(Render_Vulkan, "{}", Shader::IR::DumpProgram(program));
|
||||||
|
|
||||||
return program;
|
return program;
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,16 +3,16 @@
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include "common/object_pool.h"
|
||||||
#include "shader_recompiler/ir/basic_block.h"
|
#include "shader_recompiler/ir/basic_block.h"
|
||||||
#include "shader_recompiler/ir/program.h"
|
#include "shader_recompiler/ir/program.h"
|
||||||
#include "shader_recompiler/object_pool.h"
|
|
||||||
|
|
||||||
namespace Shader {
|
namespace Shader {
|
||||||
|
|
||||||
struct Profile;
|
struct Profile;
|
||||||
|
|
||||||
[[nodiscard]] IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool,
|
[[nodiscard]] IR::Program TranslateProgram(Common::ObjectPool<IR::Inst>& inst_pool,
|
||||||
ObjectPool<IR::Block>& block_pool,
|
Common::ObjectPool<IR::Block>& block_pool,
|
||||||
std::span<const u32> code, const Info&& info,
|
std::span<const u32> code, const Info&& info,
|
||||||
const Profile& profile);
|
const Profile& profile);
|
||||||
|
|
||||||
|
|
|
@ -77,8 +77,11 @@ struct BufferResource {
|
||||||
u32 length;
|
u32 length;
|
||||||
IR::Type used_types;
|
IR::Type used_types;
|
||||||
AmdGpu::Buffer inline_cbuf;
|
AmdGpu::Buffer inline_cbuf;
|
||||||
bool is_storage{false};
|
AmdGpu::DataFormat dfmt;
|
||||||
bool is_instance_data{false};
|
AmdGpu::NumberFormat nfmt;
|
||||||
|
bool is_storage{};
|
||||||
|
bool is_instance_data{};
|
||||||
|
bool is_written{};
|
||||||
|
|
||||||
constexpr AmdGpu::Buffer GetVsharp(const Info& info) const noexcept;
|
constexpr AmdGpu::Buffer GetVsharp(const Info& info) const noexcept;
|
||||||
};
|
};
|
||||||
|
@ -105,6 +108,19 @@ struct SamplerResource {
|
||||||
};
|
};
|
||||||
using SamplerResourceList = boost::container::static_vector<SamplerResource, 16>;
|
using SamplerResourceList = boost::container::static_vector<SamplerResource, 16>;
|
||||||
|
|
||||||
|
struct PushData {
|
||||||
|
static constexpr size_t BufOffsetIndex = 2;
|
||||||
|
|
||||||
|
u32 step0;
|
||||||
|
u32 step1;
|
||||||
|
std::array<u8, 32> buf_offsets;
|
||||||
|
|
||||||
|
void AddOffset(u32 binding, u32 offset) {
|
||||||
|
ASSERT(offset < 64 && binding < 32);
|
||||||
|
buf_offsets[binding] = offset;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
struct Info {
|
struct Info {
|
||||||
struct VsInput {
|
struct VsInput {
|
||||||
enum InstanceIdType : u8 {
|
enum InstanceIdType : u8 {
|
||||||
|
@ -182,6 +198,7 @@ struct Info {
|
||||||
bool uses_shared_u8{};
|
bool uses_shared_u8{};
|
||||||
bool uses_shared_u16{};
|
bool uses_shared_u16{};
|
||||||
bool uses_fp16{};
|
bool uses_fp16{};
|
||||||
|
bool uses_step_rates{};
|
||||||
bool translation_failed{}; // indicates that shader has unsupported instructions
|
bool translation_failed{}; // indicates that shader has unsupported instructions
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
|
|
|
@ -496,7 +496,7 @@ struct Liverpool {
|
||||||
|
|
||||||
template <typename T = VAddr>
|
template <typename T = VAddr>
|
||||||
T Address() const {
|
T Address() const {
|
||||||
return reinterpret_cast<T>((base_addr_lo & ~1U) | u64(base_addr_hi) << 32);
|
return std::bit_cast<T>((base_addr_lo & ~1U) | u64(base_addr_hi) << 32);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -363,6 +363,10 @@ struct Sampler {
|
||||||
return raw0 != 0 || raw1 != 0;
|
return raw0 != 0 || raw1 != 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool operator==(const Sampler& other) const noexcept {
|
||||||
|
return std::memcmp(this, &other, sizeof(Sampler)) == 0;
|
||||||
|
}
|
||||||
|
|
||||||
float LodBias() const noexcept {
|
float LodBias() const noexcept {
|
||||||
return static_cast<float>(static_cast<int16_t>((lod_bias.Value() ^ 0x2000u) - 0x2000u)) /
|
return static_cast<float>(static_cast<int16_t>((lod_bias.Value() ^ 0x2000u) - 0x2000u)) /
|
||||||
256.0f;
|
256.0f;
|
||||||
|
|
|
@ -0,0 +1,227 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#include "common/alignment.h"
|
||||||
|
#include "common/assert.h"
|
||||||
|
#include "video_core/buffer_cache/buffer.h"
|
||||||
|
#include "video_core/renderer_vulkan/liverpool_to_vk.h"
|
||||||
|
#include "video_core/renderer_vulkan/vk_instance.h"
|
||||||
|
#include "video_core/renderer_vulkan/vk_platform.h"
|
||||||
|
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
||||||
|
|
||||||
|
#include <vk_mem_alloc.h>
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
constexpr vk::BufferUsageFlags AllFlags =
|
||||||
|
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst |
|
||||||
|
vk::BufferUsageFlagBits::eUniformTexelBuffer | vk::BufferUsageFlagBits::eStorageTexelBuffer |
|
||||||
|
vk::BufferUsageFlagBits::eUniformBuffer | vk::BufferUsageFlagBits::eStorageBuffer |
|
||||||
|
vk::BufferUsageFlagBits::eIndexBuffer | vk::BufferUsageFlagBits::eVertexBuffer;
|
||||||
|
|
||||||
|
std::string_view BufferTypeName(MemoryUsage type) {
|
||||||
|
switch (type) {
|
||||||
|
case MemoryUsage::Upload:
|
||||||
|
return "Upload";
|
||||||
|
case MemoryUsage::Download:
|
||||||
|
return "Download";
|
||||||
|
case MemoryUsage::Stream:
|
||||||
|
return "Stream";
|
||||||
|
case MemoryUsage::DeviceLocal:
|
||||||
|
return "DeviceLocal";
|
||||||
|
default:
|
||||||
|
return "Invalid";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] VkMemoryPropertyFlags MemoryUsagePreferredVmaFlags(MemoryUsage usage) {
|
||||||
|
return usage != MemoryUsage::DeviceLocal ? VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
|
||||||
|
: VkMemoryPropertyFlagBits{};
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] VmaAllocationCreateFlags MemoryUsageVmaFlags(MemoryUsage usage) {
|
||||||
|
switch (usage) {
|
||||||
|
case MemoryUsage::Upload:
|
||||||
|
case MemoryUsage::Stream:
|
||||||
|
return VMA_ALLOCATION_CREATE_MAPPED_BIT |
|
||||||
|
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT;
|
||||||
|
case MemoryUsage::Download:
|
||||||
|
return VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT;
|
||||||
|
case MemoryUsage::DeviceLocal:
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] VmaMemoryUsage MemoryUsageVma(MemoryUsage usage) {
|
||||||
|
switch (usage) {
|
||||||
|
case MemoryUsage::DeviceLocal:
|
||||||
|
case MemoryUsage::Stream:
|
||||||
|
return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
|
||||||
|
case MemoryUsage::Upload:
|
||||||
|
case MemoryUsage::Download:
|
||||||
|
return VMA_MEMORY_USAGE_AUTO_PREFER_HOST;
|
||||||
|
}
|
||||||
|
return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
|
||||||
|
}
|
||||||
|
|
||||||
|
UniqueBuffer::UniqueBuffer(vk::Device device_, VmaAllocator allocator_)
|
||||||
|
: device{device_}, allocator{allocator_} {}
|
||||||
|
|
||||||
|
UniqueBuffer::~UniqueBuffer() {
|
||||||
|
if (buffer) {
|
||||||
|
vmaDestroyBuffer(allocator, buffer, allocation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UniqueBuffer::Create(const vk::BufferCreateInfo& buffer_ci, MemoryUsage usage,
|
||||||
|
VmaAllocationInfo* out_alloc_info) {
|
||||||
|
const VmaAllocationCreateInfo alloc_ci = {
|
||||||
|
.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage),
|
||||||
|
.usage = MemoryUsageVma(usage),
|
||||||
|
.requiredFlags = 0,
|
||||||
|
.preferredFlags = MemoryUsagePreferredVmaFlags(usage),
|
||||||
|
.pool = VK_NULL_HANDLE,
|
||||||
|
.pUserData = nullptr,
|
||||||
|
};
|
||||||
|
|
||||||
|
const VkBufferCreateInfo buffer_ci_unsafe = static_cast<VkBufferCreateInfo>(buffer_ci);
|
||||||
|
VkBuffer unsafe_buffer{};
|
||||||
|
VkResult result = vmaCreateBuffer(allocator, &buffer_ci_unsafe, &alloc_ci, &unsafe_buffer,
|
||||||
|
&allocation, out_alloc_info);
|
||||||
|
ASSERT_MSG(result == VK_SUCCESS, "Failed allocating buffer with error {}",
|
||||||
|
vk::to_string(vk::Result{result}));
|
||||||
|
buffer = vk::Buffer{unsafe_buffer};
|
||||||
|
}
|
||||||
|
|
||||||
|
Buffer::Buffer(const Vulkan::Instance& instance_, MemoryUsage usage_, VAddr cpu_addr_,
|
||||||
|
u64 size_bytes_)
|
||||||
|
: cpu_addr{cpu_addr_}, size_bytes{size_bytes_}, instance{&instance_}, usage{usage_},
|
||||||
|
buffer{instance->GetDevice(), instance->GetAllocator()} {
|
||||||
|
// Create buffer object.
|
||||||
|
const vk::BufferCreateInfo buffer_ci = {
|
||||||
|
.size = size_bytes,
|
||||||
|
.usage = AllFlags,
|
||||||
|
};
|
||||||
|
VmaAllocationInfo alloc_info{};
|
||||||
|
buffer.Create(buffer_ci, usage, &alloc_info);
|
||||||
|
|
||||||
|
if (instance->HasDebuggingToolAttached()) {
|
||||||
|
const auto device = instance->GetDevice();
|
||||||
|
Vulkan::SetObjectName(device, Handle(), "Buffer {:#x} {} KiB", cpu_addr, size_bytes / 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Map it if it is host visible.
|
||||||
|
VkMemoryPropertyFlags property_flags{};
|
||||||
|
vmaGetAllocationMemoryProperties(instance->GetAllocator(), buffer.allocation, &property_flags);
|
||||||
|
if (alloc_info.pMappedData) {
|
||||||
|
mapped_data = std::span<u8>{std::bit_cast<u8*>(alloc_info.pMappedData), size_bytes};
|
||||||
|
}
|
||||||
|
is_coherent = property_flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
|
||||||
|
}
|
||||||
|
|
||||||
|
vk::BufferView Buffer::View(u32 offset, u32 size, AmdGpu::DataFormat dfmt,
|
||||||
|
AmdGpu::NumberFormat nfmt) {
|
||||||
|
const auto it{std::ranges::find_if(views, [offset, size, dfmt, nfmt](const BufferView& view) {
|
||||||
|
return offset == view.offset && size == view.size && dfmt == view.dfmt && nfmt == view.nfmt;
|
||||||
|
})};
|
||||||
|
if (it != views.end()) {
|
||||||
|
return it->handle;
|
||||||
|
}
|
||||||
|
views.push_back({
|
||||||
|
.offset = offset,
|
||||||
|
.size = size,
|
||||||
|
.dfmt = dfmt,
|
||||||
|
.nfmt = nfmt,
|
||||||
|
.handle = instance->GetDevice().createBufferView({
|
||||||
|
.buffer = buffer.buffer,
|
||||||
|
.format = Vulkan::LiverpoolToVK::SurfaceFormat(dfmt, nfmt),
|
||||||
|
.offset = offset,
|
||||||
|
.range = size,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
return views.back().handle;
|
||||||
|
}
|
||||||
|
|
||||||
|
constexpr u64 WATCHES_INITIAL_RESERVE = 0x4000;
|
||||||
|
constexpr u64 WATCHES_RESERVE_CHUNK = 0x1000;
|
||||||
|
|
||||||
|
StreamBuffer::StreamBuffer(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler_,
|
||||||
|
MemoryUsage usage, u64 size_bytes)
|
||||||
|
: Buffer{instance, usage, 0, size_bytes}, scheduler{scheduler_} {
|
||||||
|
ReserveWatches(current_watches, WATCHES_INITIAL_RESERVE);
|
||||||
|
ReserveWatches(previous_watches, WATCHES_INITIAL_RESERVE);
|
||||||
|
const auto device = instance.GetDevice();
|
||||||
|
if (instance.HasDebuggingToolAttached()) {
|
||||||
|
Vulkan::SetObjectName(device, Handle(), "StreamBuffer({}): {} KiB", BufferTypeName(usage),
|
||||||
|
size_bytes / 1024);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::pair<u8*, u64> StreamBuffer::Map(u64 size, u64 alignment) {
|
||||||
|
if (!is_coherent && usage == MemoryUsage::Stream) {
|
||||||
|
size = Common::AlignUp(size, instance->NonCoherentAtomSize());
|
||||||
|
}
|
||||||
|
|
||||||
|
ASSERT(size <= this->size_bytes);
|
||||||
|
mapped_size = size;
|
||||||
|
|
||||||
|
if (alignment > 0) {
|
||||||
|
offset = Common::AlignUp(offset, alignment);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (offset + size > this->size_bytes) {
|
||||||
|
// The buffer would overflow, save the amount of used watches and reset the state.
|
||||||
|
invalidation_mark = current_watch_cursor;
|
||||||
|
current_watch_cursor = 0;
|
||||||
|
offset = 0;
|
||||||
|
|
||||||
|
// Swap watches and reset waiting cursors.
|
||||||
|
std::swap(previous_watches, current_watches);
|
||||||
|
wait_cursor = 0;
|
||||||
|
wait_bound = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const u64 mapped_upper_bound = offset + size;
|
||||||
|
WaitPendingOperations(mapped_upper_bound);
|
||||||
|
return std::make_pair(mapped_data.data() + offset, offset);
|
||||||
|
}
|
||||||
|
|
||||||
|
void StreamBuffer::Commit() {
|
||||||
|
if (!is_coherent) {
|
||||||
|
if (usage == MemoryUsage::Download) {
|
||||||
|
vmaInvalidateAllocation(instance->GetAllocator(), buffer.allocation, offset,
|
||||||
|
mapped_size);
|
||||||
|
} else {
|
||||||
|
vmaFlushAllocation(instance->GetAllocator(), buffer.allocation, offset, mapped_size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
offset += mapped_size;
|
||||||
|
if (current_watch_cursor + 1 >= current_watches.size()) {
|
||||||
|
// Ensure that there are enough watches.
|
||||||
|
ReserveWatches(current_watches, WATCHES_RESERVE_CHUNK);
|
||||||
|
}
|
||||||
|
|
||||||
|
auto& watch = current_watches[current_watch_cursor++];
|
||||||
|
watch.upper_bound = offset;
|
||||||
|
watch.tick = scheduler.CurrentTick();
|
||||||
|
}
|
||||||
|
|
||||||
|
void StreamBuffer::ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size) {
|
||||||
|
watches.resize(watches.size() + grow_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
void StreamBuffer::WaitPendingOperations(u64 requested_upper_bound) {
|
||||||
|
if (!invalidation_mark) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
while (requested_upper_bound > wait_bound && wait_cursor < *invalidation_mark) {
|
||||||
|
auto& watch = previous_watches[wait_cursor];
|
||||||
|
wait_bound = watch.upper_bound;
|
||||||
|
scheduler.Wait(watch.tick);
|
||||||
|
++wait_cursor;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,173 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <cstddef>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
#include "common/types.h"
|
||||||
|
#include "video_core/amdgpu/resource.h"
|
||||||
|
#include "video_core/renderer_vulkan/vk_common.h"
|
||||||
|
|
||||||
|
namespace Vulkan {
|
||||||
|
class Instance;
|
||||||
|
class Scheduler;
|
||||||
|
} // namespace Vulkan
|
||||||
|
|
||||||
|
VK_DEFINE_HANDLE(VmaAllocation)
|
||||||
|
VK_DEFINE_HANDLE(VmaAllocator)
|
||||||
|
|
||||||
|
struct VmaAllocationInfo;
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
/// Hints and requirements for the backing memory type of a commit
|
||||||
|
enum class MemoryUsage {
|
||||||
|
DeviceLocal, ///< Requests device local buffer.
|
||||||
|
Upload, ///< Requires a host visible memory type optimized for CPU to GPU uploads
|
||||||
|
Download, ///< Requires a host visible memory type optimized for GPU to CPU readbacks
|
||||||
|
Stream, ///< Requests device local host visible buffer, falling back host memory.
|
||||||
|
};
|
||||||
|
|
||||||
|
struct UniqueBuffer {
|
||||||
|
explicit UniqueBuffer(vk::Device device, VmaAllocator allocator);
|
||||||
|
~UniqueBuffer();
|
||||||
|
|
||||||
|
UniqueBuffer(const UniqueBuffer&) = delete;
|
||||||
|
UniqueBuffer& operator=(const UniqueBuffer&) = delete;
|
||||||
|
|
||||||
|
UniqueBuffer(UniqueBuffer&& other)
|
||||||
|
: buffer{std::exchange(other.buffer, VK_NULL_HANDLE)},
|
||||||
|
allocator{std::exchange(other.allocator, VK_NULL_HANDLE)},
|
||||||
|
allocation{std::exchange(other.allocation, VK_NULL_HANDLE)} {}
|
||||||
|
UniqueBuffer& operator=(UniqueBuffer&& other) {
|
||||||
|
buffer = std::exchange(other.buffer, VK_NULL_HANDLE);
|
||||||
|
allocator = std::exchange(other.allocator, VK_NULL_HANDLE);
|
||||||
|
allocation = std::exchange(other.allocation, VK_NULL_HANDLE);
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
void Create(const vk::BufferCreateInfo& image_ci, MemoryUsage usage,
|
||||||
|
VmaAllocationInfo* out_alloc_info);
|
||||||
|
|
||||||
|
operator vk::Buffer() const {
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
vk::Device device;
|
||||||
|
VmaAllocator allocator;
|
||||||
|
VmaAllocation allocation;
|
||||||
|
vk::Buffer buffer{};
|
||||||
|
};
|
||||||
|
|
||||||
|
class Buffer {
|
||||||
|
public:
|
||||||
|
explicit Buffer(const Vulkan::Instance& instance, MemoryUsage usage, VAddr cpu_addr_,
|
||||||
|
u64 size_bytes_);
|
||||||
|
|
||||||
|
Buffer& operator=(const Buffer&) = delete;
|
||||||
|
Buffer(const Buffer&) = delete;
|
||||||
|
|
||||||
|
Buffer& operator=(Buffer&&) = default;
|
||||||
|
Buffer(Buffer&&) = default;
|
||||||
|
|
||||||
|
vk::BufferView View(u32 offset, u32 size, AmdGpu::DataFormat dfmt, AmdGpu::NumberFormat nfmt);
|
||||||
|
|
||||||
|
/// Increases the likeliness of this being a stream buffer
|
||||||
|
void IncreaseStreamScore(int score) noexcept {
|
||||||
|
stream_score += score;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the likeliness of this being a stream buffer
|
||||||
|
[[nodiscard]] int StreamScore() const noexcept {
|
||||||
|
return stream_score;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns true when vaddr -> vaddr+size is fully contained in the buffer
|
||||||
|
[[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept {
|
||||||
|
return addr >= cpu_addr && addr + size <= cpu_addr + SizeBytes();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the base CPU address of the buffer
|
||||||
|
[[nodiscard]] VAddr CpuAddr() const noexcept {
|
||||||
|
return cpu_addr;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the offset relative to the given CPU address
|
||||||
|
[[nodiscard]] u32 Offset(VAddr other_cpu_addr) const noexcept {
|
||||||
|
return static_cast<u32>(other_cpu_addr - cpu_addr);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t SizeBytes() const {
|
||||||
|
return size_bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
vk::Buffer Handle() const noexcept {
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public:
|
||||||
|
VAddr cpu_addr = 0;
|
||||||
|
bool is_picked{};
|
||||||
|
bool is_coherent{};
|
||||||
|
int stream_score = 0;
|
||||||
|
size_t size_bytes = 0;
|
||||||
|
std::span<u8> mapped_data;
|
||||||
|
const Vulkan::Instance* instance{};
|
||||||
|
MemoryUsage usage;
|
||||||
|
UniqueBuffer buffer;
|
||||||
|
struct BufferView {
|
||||||
|
u32 offset;
|
||||||
|
u32 size;
|
||||||
|
AmdGpu::DataFormat dfmt;
|
||||||
|
AmdGpu::NumberFormat nfmt;
|
||||||
|
vk::BufferView handle;
|
||||||
|
};
|
||||||
|
std::vector<BufferView> views;
|
||||||
|
};
|
||||||
|
|
||||||
|
class StreamBuffer : public Buffer {
|
||||||
|
public:
|
||||||
|
explicit StreamBuffer(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||||
|
MemoryUsage usage, u64 size_bytes_);
|
||||||
|
|
||||||
|
/// Reserves a region of memory from the stream buffer.
|
||||||
|
std::pair<u8*, u64> Map(u64 size, u64 alignment = 0);
|
||||||
|
|
||||||
|
/// Ensures that reserved bytes of memory are available to the GPU.
|
||||||
|
void Commit();
|
||||||
|
|
||||||
|
/// Maps and commits a memory region with user provided data
|
||||||
|
u64 Copy(VAddr src, size_t size, size_t alignment = 0) {
|
||||||
|
const auto [data, offset] = Map(size, alignment);
|
||||||
|
std::memcpy(data, reinterpret_cast<const void*>(src), size);
|
||||||
|
Commit();
|
||||||
|
return offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
struct Watch {
|
||||||
|
u64 tick{};
|
||||||
|
u64 upper_bound{};
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Increases the amount of watches available.
|
||||||
|
void ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size);
|
||||||
|
|
||||||
|
/// Waits pending watches until requested upper bound.
|
||||||
|
void WaitPendingOperations(u64 requested_upper_bound);
|
||||||
|
|
||||||
|
private:
|
||||||
|
Vulkan::Scheduler& scheduler;
|
||||||
|
u64 offset{};
|
||||||
|
u64 mapped_size{};
|
||||||
|
std::vector<Watch> current_watches;
|
||||||
|
std::size_t current_watch_cursor{};
|
||||||
|
std::optional<size_t> invalidation_mark;
|
||||||
|
std::vector<Watch> previous_watches;
|
||||||
|
std::size_t wait_cursor{};
|
||||||
|
u64 wait_bound{};
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,497 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include "common/alignment.h"
|
||||||
|
#include "common/scope_exit.h"
|
||||||
|
#include "shader_recompiler/runtime_info.h"
|
||||||
|
#include "video_core/amdgpu/liverpool.h"
|
||||||
|
#include "video_core/buffer_cache/buffer_cache.h"
|
||||||
|
#include "video_core/renderer_vulkan/liverpool_to_vk.h"
|
||||||
|
#include "video_core/renderer_vulkan/vk_instance.h"
|
||||||
|
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
static constexpr size_t StagingBufferSize = 256_MB;
|
||||||
|
static constexpr size_t UboStreamBufferSize = 64_MB;
|
||||||
|
|
||||||
|
BufferCache::BufferCache(const Vulkan::Instance& instance_, Vulkan::Scheduler& scheduler_,
|
||||||
|
const AmdGpu::Liverpool* liverpool_, PageManager& tracker_)
|
||||||
|
: instance{instance_}, scheduler{scheduler_}, liverpool{liverpool_}, tracker{tracker_},
|
||||||
|
staging_buffer{instance, scheduler, MemoryUsage::Upload, StagingBufferSize},
|
||||||
|
stream_buffer{instance, scheduler, MemoryUsage::Stream, UboStreamBufferSize},
|
||||||
|
memory_tracker{&tracker} {
|
||||||
|
// Ensure the first slot is used for the null buffer
|
||||||
|
void(slot_buffers.insert(instance, MemoryUsage::DeviceLocal, 0, 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
BufferCache::~BufferCache() = default;
|
||||||
|
|
||||||
|
void BufferCache::InvalidateMemory(VAddr device_addr, u64 size) {
|
||||||
|
std::scoped_lock lk{mutex};
|
||||||
|
const bool is_tracked = IsRegionRegistered(device_addr, size);
|
||||||
|
if (!is_tracked) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Mark the page as CPU modified to stop tracking writes.
|
||||||
|
SCOPE_EXIT {
|
||||||
|
memory_tracker.MarkRegionAsCpuModified(device_addr, size);
|
||||||
|
};
|
||||||
|
if (!memory_tracker.IsRegionGpuModified(device_addr, size)) {
|
||||||
|
// Page has not been modified by the GPU, nothing to do.
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void BufferCache::DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 size) {
|
||||||
|
boost::container::small_vector<vk::BufferCopy, 1> copies;
|
||||||
|
u64 total_size_bytes = 0;
|
||||||
|
u64 largest_copy = 0;
|
||||||
|
memory_tracker.ForEachDownloadRange<true>(
|
||||||
|
device_addr, size, [&](u64 device_addr_out, u64 range_size) {
|
||||||
|
const VAddr buffer_addr = buffer.CpuAddr();
|
||||||
|
const auto add_download = [&](VAddr start, VAddr end, u64) {
|
||||||
|
const u64 new_offset = start - buffer_addr;
|
||||||
|
const u64 new_size = end - start;
|
||||||
|
copies.push_back(vk::BufferCopy{
|
||||||
|
.srcOffset = new_offset,
|
||||||
|
.dstOffset = total_size_bytes,
|
||||||
|
.size = new_size,
|
||||||
|
});
|
||||||
|
// Align up to avoid cache conflicts
|
||||||
|
constexpr u64 align = 64ULL;
|
||||||
|
constexpr u64 mask = ~(align - 1ULL);
|
||||||
|
total_size_bytes += (new_size + align - 1) & mask;
|
||||||
|
largest_copy = std::max(largest_copy, new_size);
|
||||||
|
};
|
||||||
|
});
|
||||||
|
if (total_size_bytes == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const auto [staging, offset] = staging_buffer.Map(total_size_bytes);
|
||||||
|
for (auto& copy : copies) {
|
||||||
|
// Modify copies to have the staging offset in mind
|
||||||
|
copy.dstOffset += offset;
|
||||||
|
}
|
||||||
|
staging_buffer.Commit();
|
||||||
|
scheduler.EndRendering();
|
||||||
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
cmdbuf.copyBuffer(buffer.buffer, staging_buffer.Handle(), copies);
|
||||||
|
scheduler.Finish();
|
||||||
|
for (const auto& copy : copies) {
|
||||||
|
const VAddr copy_device_addr = buffer.CpuAddr() + copy.srcOffset;
|
||||||
|
const u64 dst_offset = copy.dstOffset - offset;
|
||||||
|
std::memcpy(std::bit_cast<u8*>(copy_device_addr), staging + dst_offset, copy.size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool BufferCache::BindVertexBuffers(const Shader::Info& vs_info) {
|
||||||
|
if (vs_info.vs_inputs.empty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::array<vk::Buffer, NUM_VERTEX_BUFFERS> host_buffers;
|
||||||
|
std::array<vk::DeviceSize, NUM_VERTEX_BUFFERS> host_offsets;
|
||||||
|
boost::container::static_vector<AmdGpu::Buffer, NUM_VERTEX_BUFFERS> guest_buffers;
|
||||||
|
|
||||||
|
struct BufferRange {
|
||||||
|
VAddr base_address;
|
||||||
|
VAddr end_address;
|
||||||
|
vk::Buffer vk_buffer;
|
||||||
|
u64 offset;
|
||||||
|
|
||||||
|
size_t GetSize() const {
|
||||||
|
return end_address - base_address;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Calculate buffers memory overlaps
|
||||||
|
bool has_step_rate = false;
|
||||||
|
boost::container::static_vector<BufferRange, NUM_VERTEX_BUFFERS> ranges{};
|
||||||
|
for (const auto& input : vs_info.vs_inputs) {
|
||||||
|
if (input.instance_step_rate == Shader::Info::VsInput::InstanceIdType::OverStepRate0 ||
|
||||||
|
input.instance_step_rate == Shader::Info::VsInput::InstanceIdType::OverStepRate1) {
|
||||||
|
has_step_rate = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto& buffer = vs_info.ReadUd<AmdGpu::Buffer>(input.sgpr_base, input.dword_offset);
|
||||||
|
if (buffer.GetSize() == 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
guest_buffers.emplace_back(buffer);
|
||||||
|
ranges.emplace_back(buffer.base_address, buffer.base_address + buffer.GetSize());
|
||||||
|
}
|
||||||
|
|
||||||
|
std::ranges::sort(ranges, [](const BufferRange& lhv, const BufferRange& rhv) {
|
||||||
|
return lhv.base_address < rhv.base_address;
|
||||||
|
});
|
||||||
|
|
||||||
|
boost::container::static_vector<BufferRange, NUM_VERTEX_BUFFERS> ranges_merged{ranges[0]};
|
||||||
|
for (auto range : ranges) {
|
||||||
|
auto& prev_range = ranges_merged.back();
|
||||||
|
if (prev_range.end_address < range.base_address) {
|
||||||
|
ranges_merged.emplace_back(range);
|
||||||
|
} else {
|
||||||
|
prev_range.end_address = std::max(prev_range.end_address, range.end_address);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Map buffers
|
||||||
|
for (auto& range : ranges_merged) {
|
||||||
|
const auto [buffer, offset] = ObtainBuffer(range.base_address, range.GetSize(), false);
|
||||||
|
range.vk_buffer = buffer->buffer;
|
||||||
|
range.offset = offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bind vertex buffers
|
||||||
|
const size_t num_buffers = guest_buffers.size();
|
||||||
|
for (u32 i = 0; i < num_buffers; ++i) {
|
||||||
|
const auto& buffer = guest_buffers[i];
|
||||||
|
const auto host_buffer = std::ranges::find_if(ranges_merged, [&](const BufferRange& range) {
|
||||||
|
return (buffer.base_address >= range.base_address &&
|
||||||
|
buffer.base_address < range.end_address);
|
||||||
|
});
|
||||||
|
ASSERT(host_buffer != ranges_merged.cend());
|
||||||
|
|
||||||
|
host_buffers[i] = host_buffer->vk_buffer;
|
||||||
|
host_offsets[i] = host_buffer->offset + buffer.base_address - host_buffer->base_address;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (num_buffers > 0) {
|
||||||
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
cmdbuf.bindVertexBuffers(0, num_buffers, host_buffers.data(), host_offsets.data());
|
||||||
|
}
|
||||||
|
|
||||||
|
return has_step_rate;
|
||||||
|
}
|
||||||
|
|
||||||
|
u32 BufferCache::BindIndexBuffer(bool& is_indexed, u32 index_offset) {
|
||||||
|
// Emulate QuadList primitive type with CPU made index buffer.
|
||||||
|
const auto& regs = liverpool->regs;
|
||||||
|
if (regs.primitive_type == AmdGpu::Liverpool::PrimitiveType::QuadList) {
|
||||||
|
is_indexed = true;
|
||||||
|
|
||||||
|
// Emit indices.
|
||||||
|
const u32 index_size = 3 * regs.num_indices;
|
||||||
|
const auto [data, offset] = stream_buffer.Map(index_size);
|
||||||
|
Vulkan::LiverpoolToVK::EmitQuadToTriangleListIndices(data, regs.num_indices);
|
||||||
|
stream_buffer.Commit();
|
||||||
|
|
||||||
|
// Bind index buffer.
|
||||||
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
cmdbuf.bindIndexBuffer(stream_buffer.Handle(), offset, vk::IndexType::eUint16);
|
||||||
|
return index_size / sizeof(u16);
|
||||||
|
}
|
||||||
|
if (!is_indexed) {
|
||||||
|
return regs.num_indices;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Figure out index type and size.
|
||||||
|
const bool is_index16 =
|
||||||
|
regs.index_buffer_type.index_type == AmdGpu::Liverpool::IndexType::Index16;
|
||||||
|
const vk::IndexType index_type = is_index16 ? vk::IndexType::eUint16 : vk::IndexType::eUint32;
|
||||||
|
const u32 index_size = is_index16 ? sizeof(u16) : sizeof(u32);
|
||||||
|
VAddr index_address = regs.index_base_address.Address<VAddr>();
|
||||||
|
index_address += index_offset * index_size;
|
||||||
|
|
||||||
|
// Bind index buffer.
|
||||||
|
const u32 index_buffer_size = regs.num_indices * index_size;
|
||||||
|
const auto [vk_buffer, offset] = ObtainBuffer(index_address, index_buffer_size, false);
|
||||||
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
cmdbuf.bindIndexBuffer(vk_buffer->Handle(), offset, index_type);
|
||||||
|
return regs.num_indices;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::pair<Buffer*, u32> BufferCache::ObtainBuffer(VAddr device_addr, u32 size, bool is_written) {
|
||||||
|
std::scoped_lock lk{mutex};
|
||||||
|
static constexpr u64 StreamThreshold = CACHING_PAGESIZE;
|
||||||
|
const bool is_gpu_dirty = memory_tracker.IsRegionGpuModified(device_addr, size);
|
||||||
|
if (!is_written && size < StreamThreshold && !is_gpu_dirty) {
|
||||||
|
// For small uniform buffers that have not been modified by gpu
|
||||||
|
// use device local stream buffer to reduce renderpass breaks.
|
||||||
|
const u64 offset = stream_buffer.Copy(device_addr, size, instance.UniformMinAlignment());
|
||||||
|
return {&stream_buffer, offset};
|
||||||
|
}
|
||||||
|
|
||||||
|
const BufferId buffer_id = FindBuffer(device_addr, size);
|
||||||
|
Buffer& buffer = slot_buffers[buffer_id];
|
||||||
|
SynchronizeBuffer(buffer, device_addr, size);
|
||||||
|
if (is_written) {
|
||||||
|
memory_tracker.MarkRegionAsGpuModified(device_addr, size);
|
||||||
|
}
|
||||||
|
return {&buffer, buffer.Offset(device_addr)};
|
||||||
|
}
|
||||||
|
|
||||||
|
bool BufferCache::IsRegionRegistered(VAddr addr, size_t size) {
|
||||||
|
const VAddr end_addr = addr + size;
|
||||||
|
const u64 page_end = Common::DivCeil(end_addr, CACHING_PAGESIZE);
|
||||||
|
for (u64 page = addr >> CACHING_PAGEBITS; page < page_end;) {
|
||||||
|
const BufferId buffer_id = page_table[page];
|
||||||
|
if (!buffer_id) {
|
||||||
|
++page;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Buffer& buffer = slot_buffers[buffer_id];
|
||||||
|
const VAddr buf_start_addr = buffer.CpuAddr();
|
||||||
|
const VAddr buf_end_addr = buf_start_addr + buffer.SizeBytes();
|
||||||
|
if (buf_start_addr < end_addr && addr < buf_end_addr) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool BufferCache::IsRegionCpuModified(VAddr addr, size_t size) {
|
||||||
|
return memory_tracker.IsRegionCpuModified(addr, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
BufferId BufferCache::FindBuffer(VAddr device_addr, u32 size) {
|
||||||
|
if (device_addr == 0) {
|
||||||
|
return NULL_BUFFER_ID;
|
||||||
|
}
|
||||||
|
const u64 page = device_addr >> CACHING_PAGEBITS;
|
||||||
|
const BufferId buffer_id = page_table[page];
|
||||||
|
if (!buffer_id) {
|
||||||
|
return CreateBuffer(device_addr, size);
|
||||||
|
}
|
||||||
|
const Buffer& buffer = slot_buffers[buffer_id];
|
||||||
|
if (buffer.IsInBounds(device_addr, size)) {
|
||||||
|
return buffer_id;
|
||||||
|
}
|
||||||
|
return CreateBuffer(device_addr, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
BufferCache::OverlapResult BufferCache::ResolveOverlaps(VAddr device_addr, u32 wanted_size) {
|
||||||
|
static constexpr int STREAM_LEAP_THRESHOLD = 16;
|
||||||
|
boost::container::small_vector<BufferId, 16> overlap_ids;
|
||||||
|
VAddr begin = device_addr;
|
||||||
|
VAddr end = device_addr + wanted_size;
|
||||||
|
int stream_score = 0;
|
||||||
|
bool has_stream_leap = false;
|
||||||
|
const auto expand_begin = [&](VAddr add_value) {
|
||||||
|
static constexpr VAddr min_page = CACHING_PAGESIZE + DEVICE_PAGESIZE;
|
||||||
|
if (add_value > begin - min_page) {
|
||||||
|
begin = min_page;
|
||||||
|
device_addr = DEVICE_PAGESIZE;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
begin -= add_value;
|
||||||
|
device_addr = begin - CACHING_PAGESIZE;
|
||||||
|
};
|
||||||
|
const auto expand_end = [&](VAddr add_value) {
|
||||||
|
static constexpr VAddr max_page = 1ULL << MemoryTracker::MAX_CPU_PAGE_BITS;
|
||||||
|
if (add_value > max_page - end) {
|
||||||
|
end = max_page;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
end += add_value;
|
||||||
|
};
|
||||||
|
if (begin == 0) {
|
||||||
|
return OverlapResult{
|
||||||
|
.ids = std::move(overlap_ids),
|
||||||
|
.begin = begin,
|
||||||
|
.end = end,
|
||||||
|
.has_stream_leap = has_stream_leap,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
for (; device_addr >> CACHING_PAGEBITS < Common::DivCeil(end, CACHING_PAGESIZE);
|
||||||
|
device_addr += CACHING_PAGESIZE) {
|
||||||
|
const BufferId overlap_id = page_table[device_addr >> CACHING_PAGEBITS];
|
||||||
|
if (!overlap_id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Buffer& overlap = slot_buffers[overlap_id];
|
||||||
|
if (overlap.is_picked) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
overlap_ids.push_back(overlap_id);
|
||||||
|
overlap.is_picked = true;
|
||||||
|
const VAddr overlap_device_addr = overlap.CpuAddr();
|
||||||
|
const bool expands_left = overlap_device_addr < begin;
|
||||||
|
if (expands_left) {
|
||||||
|
begin = overlap_device_addr;
|
||||||
|
}
|
||||||
|
const VAddr overlap_end = overlap_device_addr + overlap.SizeBytes();
|
||||||
|
const bool expands_right = overlap_end > end;
|
||||||
|
if (overlap_end > end) {
|
||||||
|
end = overlap_end;
|
||||||
|
}
|
||||||
|
stream_score += overlap.StreamScore();
|
||||||
|
if (stream_score > STREAM_LEAP_THRESHOLD && !has_stream_leap) {
|
||||||
|
// When this memory region has been joined a bunch of times, we assume it's being used
|
||||||
|
// as a stream buffer. Increase the size to skip constantly recreating buffers.
|
||||||
|
has_stream_leap = true;
|
||||||
|
if (expands_right) {
|
||||||
|
expand_begin(CACHING_PAGESIZE * 128);
|
||||||
|
}
|
||||||
|
if (expands_left) {
|
||||||
|
expand_end(CACHING_PAGESIZE * 128);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return OverlapResult{
|
||||||
|
.ids = std::move(overlap_ids),
|
||||||
|
.begin = begin,
|
||||||
|
.end = end,
|
||||||
|
.has_stream_leap = has_stream_leap,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
void BufferCache::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id,
|
||||||
|
bool accumulate_stream_score) {
|
||||||
|
Buffer& new_buffer = slot_buffers[new_buffer_id];
|
||||||
|
Buffer& overlap = slot_buffers[overlap_id];
|
||||||
|
if (accumulate_stream_score) {
|
||||||
|
new_buffer.IncreaseStreamScore(overlap.StreamScore() + 1);
|
||||||
|
}
|
||||||
|
const size_t dst_base_offset = overlap.CpuAddr() - new_buffer.CpuAddr();
|
||||||
|
const vk::BufferCopy copy = {
|
||||||
|
.srcOffset = 0,
|
||||||
|
.dstOffset = dst_base_offset,
|
||||||
|
.size = overlap.SizeBytes(),
|
||||||
|
};
|
||||||
|
scheduler.EndRendering();
|
||||||
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
static constexpr vk::MemoryBarrier READ_BARRIER{
|
||||||
|
.srcAccessMask = vk::AccessFlagBits::eMemoryWrite,
|
||||||
|
.dstAccessMask = vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite,
|
||||||
|
};
|
||||||
|
static constexpr vk::MemoryBarrier WRITE_BARRIER{
|
||||||
|
.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
|
||||||
|
.dstAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
|
||||||
|
};
|
||||||
|
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
|
||||||
|
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
|
||||||
|
READ_BARRIER, {}, {});
|
||||||
|
cmdbuf.copyBuffer(overlap.buffer, new_buffer.buffer, copy);
|
||||||
|
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
|
||||||
|
vk::PipelineStageFlagBits::eAllCommands,
|
||||||
|
vk::DependencyFlagBits::eByRegion, WRITE_BARRIER, {}, {});
|
||||||
|
DeleteBuffer(overlap_id, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
BufferId BufferCache::CreateBuffer(VAddr device_addr, u32 wanted_size) {
|
||||||
|
const VAddr device_addr_end = Common::AlignUp(device_addr + wanted_size, CACHING_PAGESIZE);
|
||||||
|
device_addr = Common::AlignDown(device_addr, CACHING_PAGESIZE);
|
||||||
|
wanted_size = static_cast<u32>(device_addr_end - device_addr);
|
||||||
|
const OverlapResult overlap = ResolveOverlaps(device_addr, wanted_size);
|
||||||
|
const u32 size = static_cast<u32>(overlap.end - overlap.begin);
|
||||||
|
const BufferId new_buffer_id =
|
||||||
|
slot_buffers.insert(instance, MemoryUsage::DeviceLocal, overlap.begin, size);
|
||||||
|
auto& new_buffer = slot_buffers[new_buffer_id];
|
||||||
|
const size_t size_bytes = new_buffer.SizeBytes();
|
||||||
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
scheduler.EndRendering();
|
||||||
|
cmdbuf.fillBuffer(new_buffer.buffer, 0, size_bytes, 0);
|
||||||
|
for (const BufferId overlap_id : overlap.ids) {
|
||||||
|
JoinOverlap(new_buffer_id, overlap_id, !overlap.has_stream_leap);
|
||||||
|
}
|
||||||
|
Register(new_buffer_id);
|
||||||
|
return new_buffer_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
void BufferCache::Register(BufferId buffer_id) {
|
||||||
|
ChangeRegister<true>(buffer_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
void BufferCache::Unregister(BufferId buffer_id) {
|
||||||
|
ChangeRegister<false>(buffer_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <bool insert>
|
||||||
|
void BufferCache::ChangeRegister(BufferId buffer_id) {
|
||||||
|
Buffer& buffer = slot_buffers[buffer_id];
|
||||||
|
const auto size = buffer.SizeBytes();
|
||||||
|
const VAddr device_addr_begin = buffer.CpuAddr();
|
||||||
|
const VAddr device_addr_end = device_addr_begin + size;
|
||||||
|
const u64 page_begin = device_addr_begin / CACHING_PAGESIZE;
|
||||||
|
const u64 page_end = Common::DivCeil(device_addr_end, CACHING_PAGESIZE);
|
||||||
|
for (u64 page = page_begin; page != page_end; ++page) {
|
||||||
|
if constexpr (insert) {
|
||||||
|
page_table[page] = buffer_id;
|
||||||
|
} else {
|
||||||
|
page_table[page] = BufferId{};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool BufferCache::SynchronizeBuffer(Buffer& buffer, VAddr device_addr, u32 size) {
|
||||||
|
boost::container::small_vector<vk::BufferCopy, 4> copies;
|
||||||
|
u64 total_size_bytes = 0;
|
||||||
|
u64 largest_copy = 0;
|
||||||
|
VAddr buffer_start = buffer.CpuAddr();
|
||||||
|
const auto add_copy = [&](VAddr device_addr_out, u64 range_size) {
|
||||||
|
copies.push_back(vk::BufferCopy{
|
||||||
|
.srcOffset = total_size_bytes,
|
||||||
|
.dstOffset = device_addr_out - buffer_start,
|
||||||
|
.size = range_size,
|
||||||
|
});
|
||||||
|
total_size_bytes += range_size;
|
||||||
|
largest_copy = std::max(largest_copy, range_size);
|
||||||
|
};
|
||||||
|
memory_tracker.ForEachUploadRange(device_addr, size, [&](u64 device_addr_out, u64 range_size) {
|
||||||
|
add_copy(device_addr_out, range_size);
|
||||||
|
// Prevent uploading to gpu modified regions.
|
||||||
|
// gpu_modified_ranges.ForEachNotInRange(device_addr_out, range_size, add_copy);
|
||||||
|
});
|
||||||
|
if (total_size_bytes == 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
vk::Buffer src_buffer = staging_buffer.Handle();
|
||||||
|
if (total_size_bytes < StagingBufferSize) {
|
||||||
|
const auto [staging, offset] = staging_buffer.Map(total_size_bytes);
|
||||||
|
for (auto& copy : copies) {
|
||||||
|
u8* const src_pointer = staging + copy.srcOffset;
|
||||||
|
const VAddr device_addr = buffer.CpuAddr() + copy.dstOffset;
|
||||||
|
std::memcpy(src_pointer, std::bit_cast<const u8*>(device_addr), copy.size);
|
||||||
|
// Apply the staging offset
|
||||||
|
copy.srcOffset += offset;
|
||||||
|
}
|
||||||
|
staging_buffer.Commit();
|
||||||
|
} else {
|
||||||
|
// For large one time transfers use a temporary host buffer.
|
||||||
|
// RenderDoc can lag quite a bit if the stream buffer is too large.
|
||||||
|
Buffer temp_buffer{instance, MemoryUsage::Upload, 0, total_size_bytes};
|
||||||
|
src_buffer = temp_buffer.Handle();
|
||||||
|
u8* const staging = temp_buffer.mapped_data.data();
|
||||||
|
for (auto& copy : copies) {
|
||||||
|
u8* const src_pointer = staging + copy.srcOffset;
|
||||||
|
const VAddr device_addr = buffer.CpuAddr() + copy.dstOffset;
|
||||||
|
std::memcpy(src_pointer, std::bit_cast<const u8*>(device_addr), copy.size);
|
||||||
|
}
|
||||||
|
scheduler.DeferOperation([buffer = std::move(temp_buffer)]() mutable {});
|
||||||
|
}
|
||||||
|
scheduler.EndRendering();
|
||||||
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
static constexpr vk::MemoryBarrier READ_BARRIER{
|
||||||
|
.srcAccessMask = vk::AccessFlagBits::eMemoryWrite,
|
||||||
|
.dstAccessMask = vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite,
|
||||||
|
};
|
||||||
|
static constexpr vk::MemoryBarrier WRITE_BARRIER{
|
||||||
|
.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
|
||||||
|
.dstAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
|
||||||
|
};
|
||||||
|
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
|
||||||
|
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
|
||||||
|
READ_BARRIER, {}, {});
|
||||||
|
cmdbuf.copyBuffer(src_buffer, buffer.buffer, copies);
|
||||||
|
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
|
||||||
|
vk::PipelineStageFlagBits::eAllCommands,
|
||||||
|
vk::DependencyFlagBits::eByRegion, WRITE_BARRIER, {}, {});
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void BufferCache::DeleteBuffer(BufferId buffer_id, bool do_not_mark) {
|
||||||
|
// Mark the whole buffer as CPU written to stop tracking CPU writes
|
||||||
|
if (!do_not_mark) {
|
||||||
|
Buffer& buffer = slot_buffers[buffer_id];
|
||||||
|
memory_tracker.MarkRegionAsCpuModified(buffer.CpuAddr(), buffer.SizeBytes());
|
||||||
|
}
|
||||||
|
Unregister(buffer_id);
|
||||||
|
scheduler.DeferOperation([this, buffer_id] { slot_buffers.erase(buffer_id); });
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,129 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <mutex>
|
||||||
|
#include <boost/container/small_vector.hpp>
|
||||||
|
#include <boost/icl/interval_map.hpp>
|
||||||
|
#include <tsl/robin_map.h>
|
||||||
|
#include "common/div_ceil.h"
|
||||||
|
#include "common/slot_vector.h"
|
||||||
|
#include "common/types.h"
|
||||||
|
#include "video_core/buffer_cache/buffer.h"
|
||||||
|
#include "video_core/buffer_cache/memory_tracker_base.h"
|
||||||
|
#include "video_core/multi_level_page_table.h"
|
||||||
|
|
||||||
|
namespace AmdGpu {
|
||||||
|
struct Liverpool;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace Shader {
|
||||||
|
struct Info;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
using BufferId = Common::SlotId;
|
||||||
|
|
||||||
|
static constexpr BufferId NULL_BUFFER_ID{0};
|
||||||
|
|
||||||
|
static constexpr u32 NUM_VERTEX_BUFFERS = 32;
|
||||||
|
|
||||||
|
class BufferCache {
|
||||||
|
public:
|
||||||
|
static constexpr u32 CACHING_PAGEBITS = 12;
|
||||||
|
static constexpr u64 CACHING_PAGESIZE = u64{1} << CACHING_PAGEBITS;
|
||||||
|
static constexpr u64 DEVICE_PAGESIZE = 4_KB;
|
||||||
|
|
||||||
|
struct Traits {
|
||||||
|
using Entry = BufferId;
|
||||||
|
static constexpr size_t AddressSpaceBits = 39;
|
||||||
|
static constexpr size_t FirstLevelBits = 14;
|
||||||
|
static constexpr size_t PageBits = CACHING_PAGEBITS;
|
||||||
|
};
|
||||||
|
using PageTable = MultiLevelPageTable<Traits>;
|
||||||
|
|
||||||
|
struct OverlapResult {
|
||||||
|
boost::container::small_vector<BufferId, 16> ids;
|
||||||
|
VAddr begin;
|
||||||
|
VAddr end;
|
||||||
|
bool has_stream_leap = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
public:
|
||||||
|
explicit BufferCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||||
|
const AmdGpu::Liverpool* liverpool, PageManager& tracker);
|
||||||
|
~BufferCache();
|
||||||
|
|
||||||
|
/// Invalidates any buffer in the logical page range.
|
||||||
|
void InvalidateMemory(VAddr device_addr, u64 size);
|
||||||
|
|
||||||
|
/// Binds host vertex buffers for the current draw.
|
||||||
|
bool BindVertexBuffers(const Shader::Info& vs_info);
|
||||||
|
|
||||||
|
/// Bind host index buffer for the current draw.
|
||||||
|
u32 BindIndexBuffer(bool& is_indexed, u32 index_offset);
|
||||||
|
|
||||||
|
/// Obtains a buffer for the specified region.
|
||||||
|
[[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(VAddr gpu_addr, u32 size, bool is_written);
|
||||||
|
|
||||||
|
/// Return true when a region is registered on the cache
|
||||||
|
[[nodiscard]] bool IsRegionRegistered(VAddr addr, size_t size);
|
||||||
|
|
||||||
|
/// Return true when a CPU region is modified from the CPU
|
||||||
|
[[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
|
||||||
|
|
||||||
|
private:
|
||||||
|
template <typename Func>
|
||||||
|
void ForEachBufferInRange(VAddr device_addr, u64 size, Func&& func) {
|
||||||
|
const u64 page_end = Common::DivCeil(device_addr + size, CACHING_PAGESIZE);
|
||||||
|
for (u64 page = device_addr >> CACHING_PAGEBITS; page < page_end;) {
|
||||||
|
const BufferId buffer_id = page_table[page];
|
||||||
|
if (!buffer_id) {
|
||||||
|
++page;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Buffer& buffer = slot_buffers[buffer_id];
|
||||||
|
func(buffer_id, buffer);
|
||||||
|
|
||||||
|
const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
|
||||||
|
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 size);
|
||||||
|
|
||||||
|
[[nodiscard]] BufferId FindBuffer(VAddr device_addr, u32 size);
|
||||||
|
|
||||||
|
[[nodiscard]] OverlapResult ResolveOverlaps(VAddr device_addr, u32 wanted_size);
|
||||||
|
|
||||||
|
void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
|
||||||
|
|
||||||
|
[[nodiscard]] BufferId CreateBuffer(VAddr device_addr, u32 wanted_size);
|
||||||
|
|
||||||
|
void Register(BufferId buffer_id);
|
||||||
|
|
||||||
|
void Unregister(BufferId buffer_id);
|
||||||
|
|
||||||
|
template <bool insert>
|
||||||
|
void ChangeRegister(BufferId buffer_id);
|
||||||
|
|
||||||
|
bool SynchronizeBuffer(Buffer& buffer, VAddr device_addr, u32 size);
|
||||||
|
|
||||||
|
void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false);
|
||||||
|
|
||||||
|
const Vulkan::Instance& instance;
|
||||||
|
Vulkan::Scheduler& scheduler;
|
||||||
|
const AmdGpu::Liverpool* liverpool;
|
||||||
|
PageManager& tracker;
|
||||||
|
StreamBuffer staging_buffer;
|
||||||
|
StreamBuffer stream_buffer;
|
||||||
|
std::recursive_mutex mutex;
|
||||||
|
Common::SlotVector<Buffer> slot_buffers;
|
||||||
|
MemoryTracker memory_tracker;
|
||||||
|
PageTable page_table;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,175 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <deque>
|
||||||
|
#include <type_traits>
|
||||||
|
#include <vector>
|
||||||
|
#include "common/types.h"
|
||||||
|
#include "video_core/buffer_cache/word_manager.h"
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
class MemoryTracker {
|
||||||
|
public:
|
||||||
|
static constexpr size_t MAX_CPU_PAGE_BITS = 39;
|
||||||
|
static constexpr size_t HIGHER_PAGE_BITS = 22;
|
||||||
|
static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS;
|
||||||
|
static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL;
|
||||||
|
static constexpr size_t NUM_HIGH_PAGES = 1ULL << (MAX_CPU_PAGE_BITS - HIGHER_PAGE_BITS);
|
||||||
|
static constexpr size_t MANAGER_POOL_SIZE = 32;
|
||||||
|
static constexpr size_t WORDS_STACK_NEEDED = HIGHER_PAGE_SIZE / BYTES_PER_WORD;
|
||||||
|
using Manager = WordManager<WORDS_STACK_NEEDED>;
|
||||||
|
|
||||||
|
public:
|
||||||
|
explicit MemoryTracker(PageManager* tracker_) : tracker{tracker_} {}
|
||||||
|
~MemoryTracker() = default;
|
||||||
|
|
||||||
|
/// Returns true if a region has been modified from the CPU
|
||||||
|
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||||
|
return IteratePages<true>(
|
||||||
|
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||||
|
return manager->template IsRegionModified<Type::CPU>(offset, size);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns true if a region has been modified from the GPU
|
||||||
|
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||||
|
return IteratePages<false>(
|
||||||
|
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||||
|
return manager->template IsRegionModified<Type::GPU>(offset, size);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mark region as CPU modified, notifying the device_tracker about this change
|
||||||
|
void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
|
||||||
|
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||||
|
[](Manager* manager, u64 offset, size_t size) {
|
||||||
|
manager->template ChangeRegionState<Type::CPU, true>(
|
||||||
|
manager->GetCpuAddr() + offset, size);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unmark region as CPU modified, notifying the device_tracker about this change
|
||||||
|
void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
|
||||||
|
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||||
|
[](Manager* manager, u64 offset, size_t size) {
|
||||||
|
manager->template ChangeRegionState<Type::CPU, false>(
|
||||||
|
manager->GetCpuAddr() + offset, size);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mark region as modified from the host GPU
|
||||||
|
void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
|
||||||
|
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||||
|
[](Manager* manager, u64 offset, size_t size) {
|
||||||
|
manager->template ChangeRegionState<Type::GPU, true>(
|
||||||
|
manager->GetCpuAddr() + offset, size);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unmark region as modified from the host GPU
|
||||||
|
void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
|
||||||
|
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||||
|
[](Manager* manager, u64 offset, size_t size) {
|
||||||
|
manager->template ChangeRegionState<Type::GPU, false>(
|
||||||
|
manager->GetCpuAddr() + offset, size);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
|
||||||
|
template <typename Func>
|
||||||
|
void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
|
||||||
|
IteratePages<true>(query_cpu_range, query_size,
|
||||||
|
[&func](Manager* manager, u64 offset, size_t size) {
|
||||||
|
manager->template ForEachModifiedRange<Type::CPU, true>(
|
||||||
|
manager->GetCpuAddr() + offset, size, func);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
|
||||||
|
template <bool clear, typename Func>
|
||||||
|
void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
|
||||||
|
IteratePages<false>(query_cpu_range, query_size,
|
||||||
|
[&func](Manager* manager, u64 offset, size_t size) {
|
||||||
|
if constexpr (clear) {
|
||||||
|
manager->template ForEachModifiedRange<Type::GPU, true>(
|
||||||
|
manager->GetCpuAddr() + offset, size, func);
|
||||||
|
} else {
|
||||||
|
manager->template ForEachModifiedRange<Type::GPU, false>(
|
||||||
|
manager->GetCpuAddr() + offset, size, func);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
/**
|
||||||
|
* @brief IteratePages Iterates L2 word manager page table.
|
||||||
|
* @param cpu_address Start byte cpu address
|
||||||
|
* @param size Size in bytes of the region of iterate.
|
||||||
|
* @param func Callback for each word manager.
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
template <bool create_region_on_fail, typename Func>
|
||||||
|
bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
|
||||||
|
using FuncReturn = typename std::invoke_result<Func, Manager*, u64, size_t>::type;
|
||||||
|
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
|
||||||
|
std::size_t remaining_size{size};
|
||||||
|
std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
|
||||||
|
u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
|
||||||
|
while (remaining_size > 0) {
|
||||||
|
const std::size_t copy_amount{
|
||||||
|
std::min<std::size_t>(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
|
||||||
|
auto* manager{top_tier[page_index]};
|
||||||
|
if (manager) {
|
||||||
|
if constexpr (BOOL_BREAK) {
|
||||||
|
if (func(manager, page_offset, copy_amount)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
func(manager, page_offset, copy_amount);
|
||||||
|
}
|
||||||
|
} else if constexpr (create_region_on_fail) {
|
||||||
|
CreateRegion(page_index);
|
||||||
|
manager = top_tier[page_index];
|
||||||
|
if constexpr (BOOL_BREAK) {
|
||||||
|
if (func(manager, page_offset, copy_amount)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
func(manager, page_offset, copy_amount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
page_index++;
|
||||||
|
page_offset = 0;
|
||||||
|
remaining_size -= copy_amount;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void CreateRegion(std::size_t page_index) {
|
||||||
|
const VAddr base_cpu_addr = page_index << HIGHER_PAGE_BITS;
|
||||||
|
if (free_managers.empty()) {
|
||||||
|
manager_pool.emplace_back();
|
||||||
|
auto& last_pool = manager_pool.back();
|
||||||
|
for (size_t i = 0; i < MANAGER_POOL_SIZE; i++) {
|
||||||
|
std::construct_at(&last_pool[i], tracker, 0, HIGHER_PAGE_SIZE);
|
||||||
|
free_managers.push_back(&last_pool[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Each manager tracks a 4_MB virtual address space.
|
||||||
|
auto* new_manager = free_managers.back();
|
||||||
|
new_manager->SetCpuAddress(base_cpu_addr);
|
||||||
|
free_managers.pop_back();
|
||||||
|
top_tier[page_index] = new_manager;
|
||||||
|
}
|
||||||
|
|
||||||
|
PageManager* tracker;
|
||||||
|
std::deque<std::array<Manager, MANAGER_POOL_SIZE>> manager_pool;
|
||||||
|
std::vector<Manager*> free_managers;
|
||||||
|
std::array<Manager*, NUM_HIGH_PAGES> top_tier{};
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,159 @@
|
||||||
|
// SPDX-FileCopyrightText: 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <boost/icl/interval_map.hpp>
|
||||||
|
#include <boost/pool/pool.hpp>
|
||||||
|
#include <boost/pool/pool_alloc.hpp>
|
||||||
|
#include <boost/pool/poolfwd.hpp>
|
||||||
|
#include "common/types.h"
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
template <class T>
|
||||||
|
using RangeSetsAllocator =
|
||||||
|
boost::fast_pool_allocator<T, boost::default_user_allocator_new_delete,
|
||||||
|
boost::details::pool::default_mutex, 1024, 2048>;
|
||||||
|
|
||||||
|
struct RangeSet {
|
||||||
|
using IntervalSet =
|
||||||
|
boost::icl::interval_set<VAddr, std::less,
|
||||||
|
ICL_INTERVAL_INSTANCE(ICL_INTERVAL_DEFAULT, VAddr, std::less),
|
||||||
|
RangeSetsAllocator>;
|
||||||
|
using IntervalType = typename IntervalSet::interval_type;
|
||||||
|
|
||||||
|
explicit RangeSet() = default;
|
||||||
|
~RangeSet() = default;
|
||||||
|
|
||||||
|
void Add(VAddr base_address, size_t size) {
|
||||||
|
const VAddr end_address = base_address + size;
|
||||||
|
IntervalType interval{base_address, end_address};
|
||||||
|
m_ranges_set.add(interval);
|
||||||
|
}
|
||||||
|
|
||||||
|
void Subtract(VAddr base_address, size_t size) {
|
||||||
|
const VAddr end_address = base_address + size;
|
||||||
|
IntervalType interval{base_address, end_address};
|
||||||
|
m_ranges_set.subtract(interval);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Func>
|
||||||
|
void ForEach(Func&& func) const {
|
||||||
|
if (m_ranges_set.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
auto it = m_ranges_set.begin();
|
||||||
|
auto end_it = m_ranges_set.end();
|
||||||
|
for (; it != end_it; it++) {
|
||||||
|
const VAddr inter_addr_end = it->upper();
|
||||||
|
const VAddr inter_addr = it->lower();
|
||||||
|
func(inter_addr, inter_addr_end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Func>
|
||||||
|
void ForEachInRange(VAddr base_addr, size_t size, Func&& func) const {
|
||||||
|
if (m_ranges_set.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const VAddr start_address = base_addr;
|
||||||
|
const VAddr end_address = start_address + size;
|
||||||
|
const IntervalType search_interval{start_address, end_address};
|
||||||
|
auto it = m_ranges_set.lower_bound(search_interval);
|
||||||
|
if (it == m_ranges_set.end()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
auto end_it = m_ranges_set.upper_bound(search_interval);
|
||||||
|
for (; it != end_it; it++) {
|
||||||
|
VAddr inter_addr_end = it->upper();
|
||||||
|
VAddr inter_addr = it->lower();
|
||||||
|
if (inter_addr_end > end_address) {
|
||||||
|
inter_addr_end = end_address;
|
||||||
|
}
|
||||||
|
if (inter_addr < start_address) {
|
||||||
|
inter_addr = start_address;
|
||||||
|
}
|
||||||
|
func(inter_addr, inter_addr_end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
IntervalSet m_ranges_set;
|
||||||
|
};
|
||||||
|
|
||||||
|
class RangeMap {
|
||||||
|
public:
|
||||||
|
using IntervalMap =
|
||||||
|
boost::icl::interval_map<VAddr, u64, boost::icl::partial_absorber, std::less,
|
||||||
|
boost::icl::inplace_plus, boost::icl::inter_section,
|
||||||
|
ICL_INTERVAL_INSTANCE(ICL_INTERVAL_DEFAULT, VAddr, std::less),
|
||||||
|
RangeSetsAllocator>;
|
||||||
|
using IntervalType = typename IntervalMap::interval_type;
|
||||||
|
|
||||||
|
public:
|
||||||
|
RangeMap() = default;
|
||||||
|
~RangeMap() = default;
|
||||||
|
|
||||||
|
RangeMap(RangeMap const&) = delete;
|
||||||
|
RangeMap& operator=(RangeMap const&) = delete;
|
||||||
|
|
||||||
|
RangeMap(RangeMap&& other);
|
||||||
|
RangeMap& operator=(RangeMap&& other);
|
||||||
|
|
||||||
|
void Add(VAddr base_address, size_t size, u64 value) {
|
||||||
|
const VAddr end_address = base_address + size;
|
||||||
|
IntervalType interval{base_address, end_address};
|
||||||
|
m_ranges_map.add({interval, value});
|
||||||
|
}
|
||||||
|
|
||||||
|
void Subtract(VAddr base_address, size_t size) {
|
||||||
|
const VAddr end_address = base_address + size;
|
||||||
|
IntervalType interval{base_address, end_address};
|
||||||
|
m_ranges_map -= interval;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Func>
|
||||||
|
void ForEachInRange(VAddr base_addr, size_t size, Func&& func) const {
|
||||||
|
if (m_ranges_map.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const VAddr start_address = base_addr;
|
||||||
|
const VAddr end_address = start_address + size;
|
||||||
|
const IntervalType search_interval{start_address, end_address};
|
||||||
|
auto it = m_ranges_map.lower_bound(search_interval);
|
||||||
|
if (it == m_ranges_map.end()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
auto end_it = m_ranges_map.upper_bound(search_interval);
|
||||||
|
for (; it != end_it; it++) {
|
||||||
|
VAddr inter_addr_end = it->first.upper();
|
||||||
|
VAddr inter_addr = it->first.lower();
|
||||||
|
if (inter_addr_end > end_address) {
|
||||||
|
inter_addr_end = end_address;
|
||||||
|
}
|
||||||
|
if (inter_addr < start_address) {
|
||||||
|
inter_addr = start_address;
|
||||||
|
}
|
||||||
|
func(inter_addr, inter_addr_end, it->second);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Func>
|
||||||
|
void ForEachNotInRange(VAddr base_addr, size_t size, Func&& func) const {
|
||||||
|
const VAddr end_addr = base_addr + size;
|
||||||
|
ForEachInRange(base_addr, size, [&](VAddr range_addr, VAddr range_end, u64) {
|
||||||
|
if (size_t gap_size = range_addr - base_addr; gap_size != 0) {
|
||||||
|
func(base_addr, gap_size);
|
||||||
|
}
|
||||||
|
base_addr = range_end;
|
||||||
|
});
|
||||||
|
if (base_addr != end_addr) {
|
||||||
|
func(base_addr, end_addr - base_addr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
IntervalMap m_ranges_map;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,398 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <span>
|
||||||
|
#include <utility>
|
||||||
|
#include "common/div_ceil.h"
|
||||||
|
#include "common/types.h"
|
||||||
|
#include "video_core/page_manager.h"
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
constexpr u64 PAGES_PER_WORD = 64;
|
||||||
|
constexpr u64 BYTES_PER_PAGE = 4_KB;
|
||||||
|
constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
|
||||||
|
|
||||||
|
enum class Type {
|
||||||
|
CPU,
|
||||||
|
GPU,
|
||||||
|
Untracked,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Vector tracking modified pages tightly packed with small vector optimization
|
||||||
|
template <size_t stack_words = 1>
|
||||||
|
struct WordsArray {
|
||||||
|
/// Returns the pointer to the words state
|
||||||
|
[[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
|
||||||
|
return is_short ? stack.data() : heap;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the pointer to the words state
|
||||||
|
[[nodiscard]] u64* Pointer(bool is_short) noexcept {
|
||||||
|
return is_short ? stack.data() : heap;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::array<u64, stack_words> stack{}; ///< Small buffers storage
|
||||||
|
u64* heap; ///< Not-small buffers pointer to the storage
|
||||||
|
};
|
||||||
|
|
||||||
|
template <size_t stack_words = 1>
|
||||||
|
struct Words {
|
||||||
|
explicit Words() = default;
|
||||||
|
explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
|
||||||
|
num_words = Common::DivCeil(size_bytes, BYTES_PER_WORD);
|
||||||
|
if (IsShort()) {
|
||||||
|
cpu.stack.fill(~u64{0});
|
||||||
|
gpu.stack.fill(0);
|
||||||
|
untracked.stack.fill(~u64{0});
|
||||||
|
} else {
|
||||||
|
// Share allocation between CPU and GPU pages and set their default values
|
||||||
|
u64* const alloc = new u64[num_words * 3];
|
||||||
|
cpu.heap = alloc;
|
||||||
|
gpu.heap = alloc + num_words;
|
||||||
|
untracked.heap = alloc + num_words * 2;
|
||||||
|
std::fill_n(cpu.heap, num_words, ~u64{0});
|
||||||
|
std::fill_n(gpu.heap, num_words, 0);
|
||||||
|
std::fill_n(untracked.heap, num_words, ~u64{0});
|
||||||
|
}
|
||||||
|
// Clean up tailing bits
|
||||||
|
const u64 last_word_size = size_bytes % BYTES_PER_WORD;
|
||||||
|
const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
|
||||||
|
const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
|
||||||
|
const u64 last_word = (~u64{0} << shift) >> shift;
|
||||||
|
cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||||
|
untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||||
|
}
|
||||||
|
|
||||||
|
~Words() {
|
||||||
|
Release();
|
||||||
|
}
|
||||||
|
|
||||||
|
Words& operator=(Words&& rhs) noexcept {
|
||||||
|
Release();
|
||||||
|
size_bytes = rhs.size_bytes;
|
||||||
|
num_words = rhs.num_words;
|
||||||
|
cpu = rhs.cpu;
|
||||||
|
gpu = rhs.gpu;
|
||||||
|
untracked = rhs.untracked;
|
||||||
|
rhs.cpu.heap = nullptr;
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Words(Words&& rhs) noexcept
|
||||||
|
: size_bytes{rhs.size_bytes}, num_words{rhs.num_words}, cpu{rhs.cpu}, gpu{rhs.gpu},
|
||||||
|
untracked{rhs.untracked} {
|
||||||
|
rhs.cpu.heap = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
Words& operator=(const Words&) = delete;
|
||||||
|
Words(const Words&) = delete;
|
||||||
|
|
||||||
|
/// Returns true when the buffer fits in the small vector optimization
|
||||||
|
[[nodiscard]] bool IsShort() const noexcept {
|
||||||
|
return num_words <= stack_words;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of words of the buffer
|
||||||
|
[[nodiscard]] size_t NumWords() const noexcept {
|
||||||
|
return num_words;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Release buffer resources
|
||||||
|
void Release() {
|
||||||
|
if (!IsShort()) {
|
||||||
|
// CPU written words is the base for the heap allocation
|
||||||
|
delete[] cpu.heap;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <Type type>
|
||||||
|
std::span<u64> Span() noexcept {
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
return std::span<u64>(cpu.Pointer(IsShort()), num_words);
|
||||||
|
} else if constexpr (type == Type::GPU) {
|
||||||
|
return std::span<u64>(gpu.Pointer(IsShort()), num_words);
|
||||||
|
} else if constexpr (type == Type::Untracked) {
|
||||||
|
return std::span<u64>(untracked.Pointer(IsShort()), num_words);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <Type type>
|
||||||
|
std::span<const u64> Span() const noexcept {
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
return std::span<const u64>(cpu.Pointer(IsShort()), num_words);
|
||||||
|
} else if constexpr (type == Type::GPU) {
|
||||||
|
return std::span<const u64>(gpu.Pointer(IsShort()), num_words);
|
||||||
|
} else if constexpr (type == Type::Untracked) {
|
||||||
|
return std::span<const u64>(untracked.Pointer(IsShort()), num_words);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
u64 size_bytes = 0;
|
||||||
|
size_t num_words = 0;
|
||||||
|
WordsArray<stack_words> cpu;
|
||||||
|
WordsArray<stack_words> gpu;
|
||||||
|
WordsArray<stack_words> untracked;
|
||||||
|
};
|
||||||
|
|
||||||
|
template <size_t stack_words = 1>
|
||||||
|
class WordManager {
|
||||||
|
public:
|
||||||
|
explicit WordManager(PageManager* tracker_, VAddr cpu_addr_, u64 size_bytes)
|
||||||
|
: tracker{tracker_}, cpu_addr{cpu_addr_}, words{size_bytes} {}
|
||||||
|
|
||||||
|
explicit WordManager() = default;
|
||||||
|
|
||||||
|
void SetCpuAddress(VAddr new_cpu_addr) {
|
||||||
|
cpu_addr = new_cpu_addr;
|
||||||
|
}
|
||||||
|
|
||||||
|
VAddr GetCpuAddr() const {
|
||||||
|
return cpu_addr;
|
||||||
|
}
|
||||||
|
|
||||||
|
static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) {
|
||||||
|
constexpr size_t number_bits = sizeof(u64) * 8;
|
||||||
|
const size_t limit_page_end = number_bits - std::min(page_end, number_bits);
|
||||||
|
u64 bits = (word >> page_start) << page_start;
|
||||||
|
bits = (bits << limit_page_end) >> limit_page_end;
|
||||||
|
return bits;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::pair<size_t, size_t> GetWordPage(VAddr address) {
|
||||||
|
const size_t converted_address = static_cast<size_t>(address);
|
||||||
|
const size_t word_number = converted_address / BYTES_PER_WORD;
|
||||||
|
const size_t amount_pages = converted_address % BYTES_PER_WORD;
|
||||||
|
return std::make_pair(word_number, amount_pages / BYTES_PER_PAGE);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Func>
|
||||||
|
void IterateWords(size_t offset, size_t size, Func&& func) const {
|
||||||
|
using FuncReturn = std::invoke_result_t<Func, std::size_t, u64>;
|
||||||
|
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
|
||||||
|
const size_t start = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset), 0LL));
|
||||||
|
const size_t end = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset + size), 0LL));
|
||||||
|
if (start >= SizeBytes() || end <= start) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
auto [start_word, start_page] = GetWordPage(start);
|
||||||
|
auto [end_word, end_page] = GetWordPage(end + BYTES_PER_PAGE - 1ULL);
|
||||||
|
const size_t num_words = NumWords();
|
||||||
|
start_word = std::min(start_word, num_words);
|
||||||
|
end_word = std::min(end_word, num_words);
|
||||||
|
const size_t diff = end_word - start_word;
|
||||||
|
end_word += (end_page + PAGES_PER_WORD - 1ULL) / PAGES_PER_WORD;
|
||||||
|
end_word = std::min(end_word, num_words);
|
||||||
|
end_page += diff * PAGES_PER_WORD;
|
||||||
|
constexpr u64 base_mask{~0ULL};
|
||||||
|
for (size_t word_index = start_word; word_index < end_word; word_index++) {
|
||||||
|
const u64 mask = ExtractBits(base_mask, start_page, end_page);
|
||||||
|
start_page = 0;
|
||||||
|
end_page -= PAGES_PER_WORD;
|
||||||
|
if constexpr (BOOL_BREAK) {
|
||||||
|
if (func(word_index, mask)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
func(word_index, mask);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Func>
|
||||||
|
void IteratePages(u64 mask, Func&& func) const {
|
||||||
|
size_t offset = 0;
|
||||||
|
while (mask != 0) {
|
||||||
|
const size_t empty_bits = std::countr_zero(mask);
|
||||||
|
offset += empty_bits;
|
||||||
|
mask = mask >> empty_bits;
|
||||||
|
|
||||||
|
const size_t continuous_bits = std::countr_one(mask);
|
||||||
|
func(offset, continuous_bits);
|
||||||
|
mask = continuous_bits < PAGES_PER_WORD ? (mask >> continuous_bits) : 0;
|
||||||
|
offset += continuous_bits;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Change the state of a range of pages
|
||||||
|
*
|
||||||
|
* @param dirty_addr Base address to mark or unmark as modified
|
||||||
|
* @param size Size in bytes to mark or unmark as modified
|
||||||
|
*/
|
||||||
|
template <Type type, bool enable>
|
||||||
|
void ChangeRegionState(u64 dirty_addr, u64 size) noexcept(type == Type::GPU) {
|
||||||
|
std::span<u64> state_words = words.template Span<type>();
|
||||||
|
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
|
||||||
|
IterateWords(dirty_addr - cpu_addr, size, [&](size_t index, u64 mask) {
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
NotifyPageTracker<!enable>(index, untracked_words[index], mask);
|
||||||
|
}
|
||||||
|
if constexpr (enable) {
|
||||||
|
state_words[index] |= mask;
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
untracked_words[index] |= mask;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
state_words[index] &= ~mask;
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
untracked_words[index] &= ~mask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loop over each page in the given range, turn off those bits and notify the tracker if
|
||||||
|
* needed. Call the given function on each turned off range.
|
||||||
|
*
|
||||||
|
* @param query_cpu_range Base CPU address to loop over
|
||||||
|
* @param size Size in bytes of the CPU range to loop over
|
||||||
|
* @param func Function to call for each turned off region
|
||||||
|
*/
|
||||||
|
template <Type type, bool clear, typename Func>
|
||||||
|
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) {
|
||||||
|
static_assert(type != Type::Untracked);
|
||||||
|
|
||||||
|
std::span<u64> state_words = words.template Span<type>();
|
||||||
|
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
|
||||||
|
const size_t offset = query_cpu_range - cpu_addr;
|
||||||
|
bool pending = false;
|
||||||
|
size_t pending_offset{};
|
||||||
|
size_t pending_pointer{};
|
||||||
|
const auto release = [&]() {
|
||||||
|
func(cpu_addr + pending_offset * BYTES_PER_PAGE,
|
||||||
|
(pending_pointer - pending_offset) * BYTES_PER_PAGE);
|
||||||
|
};
|
||||||
|
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||||
|
if constexpr (type == Type::GPU) {
|
||||||
|
mask &= ~untracked_words[index];
|
||||||
|
}
|
||||||
|
const u64 word = state_words[index] & mask;
|
||||||
|
if constexpr (clear) {
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
NotifyPageTracker<true>(index, untracked_words[index], mask);
|
||||||
|
}
|
||||||
|
state_words[index] &= ~mask;
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
untracked_words[index] &= ~mask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const size_t base_offset = index * PAGES_PER_WORD;
|
||||||
|
IteratePages(word, [&](size_t pages_offset, size_t pages_size) {
|
||||||
|
const auto reset = [&]() {
|
||||||
|
pending_offset = base_offset + pages_offset;
|
||||||
|
pending_pointer = base_offset + pages_offset + pages_size;
|
||||||
|
};
|
||||||
|
if (!pending) {
|
||||||
|
reset();
|
||||||
|
pending = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (pending_pointer == base_offset + pages_offset) {
|
||||||
|
pending_pointer += pages_size;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
release();
|
||||||
|
reset();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
if (pending) {
|
||||||
|
release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns true when a region has been modified
|
||||||
|
*
|
||||||
|
* @param offset Offset in bytes from the start of the buffer
|
||||||
|
* @param size Size in bytes of the region to query for modifications
|
||||||
|
*/
|
||||||
|
template <Type type>
|
||||||
|
[[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
|
||||||
|
static_assert(type != Type::Untracked);
|
||||||
|
|
||||||
|
const std::span<const u64> state_words = words.template Span<type>();
|
||||||
|
[[maybe_unused]] const std::span<const u64> untracked_words =
|
||||||
|
words.template Span<Type::Untracked>();
|
||||||
|
bool result = false;
|
||||||
|
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||||
|
if constexpr (type == Type::GPU) {
|
||||||
|
mask &= ~untracked_words[index];
|
||||||
|
}
|
||||||
|
const u64 word = state_words[index] & mask;
|
||||||
|
if (word != 0) {
|
||||||
|
result = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of words of the manager
|
||||||
|
[[nodiscard]] size_t NumWords() const noexcept {
|
||||||
|
return words.NumWords();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the size in bytes of the manager
|
||||||
|
[[nodiscard]] u64 SizeBytes() const noexcept {
|
||||||
|
return words.size_bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns true when the buffer fits in the small vector optimization
|
||||||
|
[[nodiscard]] bool IsShort() const noexcept {
|
||||||
|
return words.IsShort();
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
template <Type type>
|
||||||
|
u64* Array() noexcept {
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
return words.cpu.Pointer(IsShort());
|
||||||
|
} else if constexpr (type == Type::GPU) {
|
||||||
|
return words.gpu.Pointer(IsShort());
|
||||||
|
} else if constexpr (type == Type::Untracked) {
|
||||||
|
return words.untracked.Pointer(IsShort());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <Type type>
|
||||||
|
const u64* Array() const noexcept {
|
||||||
|
if constexpr (type == Type::CPU) {
|
||||||
|
return words.cpu.Pointer(IsShort());
|
||||||
|
} else if constexpr (type == Type::GPU) {
|
||||||
|
return words.gpu.Pointer(IsShort());
|
||||||
|
} else if constexpr (type == Type::Untracked) {
|
||||||
|
return words.untracked.Pointer(IsShort());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Notify tracker about changes in the CPU tracking state of a word in the buffer
|
||||||
|
*
|
||||||
|
* @param word_index Index to the word to notify to the tracker
|
||||||
|
* @param current_bits Current state of the word
|
||||||
|
* @param new_bits New state of the word
|
||||||
|
*
|
||||||
|
* @tparam add_to_tracker True when the tracker should start tracking the new pages
|
||||||
|
*/
|
||||||
|
template <bool add_to_tracker>
|
||||||
|
void NotifyPageTracker(u64 word_index, u64 current_bits, u64 new_bits) const {
|
||||||
|
u64 changed_bits = (add_to_tracker ? current_bits : ~current_bits) & new_bits;
|
||||||
|
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
|
||||||
|
IteratePages(changed_bits, [&](size_t offset, size_t size) {
|
||||||
|
tracker->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE, size * BYTES_PER_PAGE,
|
||||||
|
add_to_tracker ? 1 : -1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
PageManager* tracker;
|
||||||
|
VAddr cpu_addr = 0;
|
||||||
|
Words<stack_words> words;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,65 @@
|
||||||
|
// SPDX-FileCopyrightText: 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <type_traits>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "common/object_pool.h"
|
||||||
|
#include "common/types.h"
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
template <class Traits>
|
||||||
|
class MultiLevelPageTable final {
|
||||||
|
using Entry = typename Traits::Entry;
|
||||||
|
|
||||||
|
static constexpr size_t AddressSpaceBits = Traits::AddressSpaceBits;
|
||||||
|
static constexpr size_t FirstLevelBits = Traits::FirstLevelBits;
|
||||||
|
static constexpr size_t PageBits = Traits::PageBits;
|
||||||
|
static constexpr size_t FirstLevelShift = AddressSpaceBits - FirstLevelBits;
|
||||||
|
static constexpr size_t SecondLevelBits = FirstLevelShift - PageBits;
|
||||||
|
static constexpr size_t NumEntriesPerL1Page = 1ULL << SecondLevelBits;
|
||||||
|
|
||||||
|
using L1Page = std::array<Entry, NumEntriesPerL1Page>;
|
||||||
|
|
||||||
|
public:
|
||||||
|
explicit MultiLevelPageTable() : first_level_map{1ULL << FirstLevelBits, nullptr} {}
|
||||||
|
|
||||||
|
~MultiLevelPageTable() noexcept = default;
|
||||||
|
|
||||||
|
[[nodiscard]] Entry* find(size_t page) {
|
||||||
|
const size_t l1_page = page >> SecondLevelBits;
|
||||||
|
const size_t l2_page = page & (NumEntriesPerL1Page - 1);
|
||||||
|
if (!first_level_map[l1_page]) {
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
return &(*first_level_map[l1_page])[l2_page];
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] const Entry& operator[](size_t page) const {
|
||||||
|
const size_t l1_page = page >> SecondLevelBits;
|
||||||
|
const size_t l2_page = page & (NumEntriesPerL1Page - 1);
|
||||||
|
if (!first_level_map[l1_page]) {
|
||||||
|
first_level_map[l1_page] = page_alloc.Create();
|
||||||
|
}
|
||||||
|
return (*first_level_map[l1_page])[l2_page];
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] Entry& operator[](size_t page) {
|
||||||
|
const size_t l1_page = page >> SecondLevelBits;
|
||||||
|
const size_t l2_page = page & (NumEntriesPerL1Page - 1);
|
||||||
|
if (!first_level_map[l1_page]) {
|
||||||
|
first_level_map[l1_page] = page_alloc.Create();
|
||||||
|
}
|
||||||
|
return (*first_level_map[l1_page])[l2_page];
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::vector<L1Page*> first_level_map{};
|
||||||
|
Common::ObjectPool<L1Page> page_alloc;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,260 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#include <thread>
|
||||||
|
#include "common/alignment.h"
|
||||||
|
#include "common/assert.h"
|
||||||
|
#include "common/error.h"
|
||||||
|
#include "video_core/page_manager.h"
|
||||||
|
#include "video_core/renderer_vulkan/vk_rasterizer.h"
|
||||||
|
|
||||||
|
#ifndef _WIN64
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <poll.h>
|
||||||
|
#include <signal.h>
|
||||||
|
#include <sys/ioctl.h>
|
||||||
|
#include <sys/mman.h>
|
||||||
|
#ifdef ENABLE_USERFAULTFD
|
||||||
|
#include <linux/userfaultfd.h>
|
||||||
|
#endif
|
||||||
|
#else
|
||||||
|
#include <windows.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
constexpr size_t PAGESIZE = 4_KB;
|
||||||
|
constexpr size_t PAGEBITS = 12;
|
||||||
|
|
||||||
|
#ifdef _WIN64
|
||||||
|
struct PageManager::Impl {
|
||||||
|
Impl(Vulkan::Rasterizer* rasterizer_) {
|
||||||
|
rasterizer = rasterizer_;
|
||||||
|
|
||||||
|
veh_handle = AddVectoredExceptionHandler(0, GuestFaultSignalHandler);
|
||||||
|
ASSERT_MSG(veh_handle, "Failed to register an exception handler");
|
||||||
|
}
|
||||||
|
|
||||||
|
void OnMap(VAddr address, size_t size) {}
|
||||||
|
|
||||||
|
void OnUnmap(VAddr address, size_t size) {}
|
||||||
|
|
||||||
|
void Protect(VAddr address, size_t size, bool allow_write) {
|
||||||
|
DWORD prot = allow_write ? PAGE_READWRITE : PAGE_READONLY;
|
||||||
|
DWORD old_prot{};
|
||||||
|
BOOL result = VirtualProtect(std::bit_cast<LPVOID>(address), size, prot, &old_prot);
|
||||||
|
ASSERT_MSG(result != 0, "Region protection failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
static LONG WINAPI GuestFaultSignalHandler(EXCEPTION_POINTERS* pExp) noexcept {
|
||||||
|
const u32 ec = pExp->ExceptionRecord->ExceptionCode;
|
||||||
|
if (ec == EXCEPTION_ACCESS_VIOLATION) {
|
||||||
|
const auto info = pExp->ExceptionRecord->ExceptionInformation;
|
||||||
|
if (info[0] == 1) { // Write violation
|
||||||
|
rasterizer->InvalidateMemory(info[1], sizeof(u64));
|
||||||
|
return EXCEPTION_CONTINUE_EXECUTION;
|
||||||
|
} /* else {
|
||||||
|
UNREACHABLE();
|
||||||
|
}*/
|
||||||
|
}
|
||||||
|
return EXCEPTION_CONTINUE_SEARCH; // pass further
|
||||||
|
}
|
||||||
|
|
||||||
|
inline static Vulkan::Rasterizer* rasterizer;
|
||||||
|
void* veh_handle{};
|
||||||
|
};
|
||||||
|
#elif ENABLE_USERFAULTFD
|
||||||
|
struct PageManager::Impl {
|
||||||
|
Impl(Vulkan::Rasterizer* rasterizer_) : rasterizer{rasterizer_} {
|
||||||
|
uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
|
||||||
|
ASSERT_MSG(uffd != -1, "{}", Common::GetLastErrorMsg());
|
||||||
|
|
||||||
|
// Request uffdio features from kernel.
|
||||||
|
uffdio_api api;
|
||||||
|
api.api = UFFD_API;
|
||||||
|
api.features = UFFD_FEATURE_THREAD_ID;
|
||||||
|
const int ret = ioctl(uffd, UFFDIO_API, &api);
|
||||||
|
ASSERT(ret == 0 && api.api == UFFD_API);
|
||||||
|
|
||||||
|
// Create uffd handler thread
|
||||||
|
ufd_thread = std::jthread([&](std::stop_token token) { UffdHandler(token); });
|
||||||
|
}
|
||||||
|
|
||||||
|
void OnMap(VAddr address, size_t size) {
|
||||||
|
uffdio_register reg;
|
||||||
|
reg.range.start = address;
|
||||||
|
reg.range.len = size;
|
||||||
|
reg.mode = UFFDIO_REGISTER_MODE_WP;
|
||||||
|
const int ret = ioctl(uffd, UFFDIO_REGISTER, ®);
|
||||||
|
ASSERT_MSG(ret != -1, "Uffdio register failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
void OnUnmap(VAddr address, size_t size) {
|
||||||
|
uffdio_range range;
|
||||||
|
range.start = address;
|
||||||
|
range.len = size;
|
||||||
|
const int ret = ioctl(uffd, UFFDIO_UNREGISTER, &range);
|
||||||
|
ASSERT_MSG(ret != -1, "Uffdio unregister failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
void Protect(VAddr address, size_t size, bool allow_write) {
|
||||||
|
uffdio_writeprotect wp;
|
||||||
|
wp.range.start = address;
|
||||||
|
wp.range.len = size;
|
||||||
|
wp.mode = allow_write ? 0 : UFFDIO_WRITEPROTECT_MODE_WP;
|
||||||
|
const int ret = ioctl(uffd, UFFDIO_WRITEPROTECT, &wp);
|
||||||
|
ASSERT_MSG(ret != -1, "Uffdio writeprotect failed with error: {}",
|
||||||
|
Common::GetLastErrorMsg());
|
||||||
|
}
|
||||||
|
|
||||||
|
void UffdHandler(std::stop_token token) {
|
||||||
|
while (!token.stop_requested()) {
|
||||||
|
pollfd pollfd;
|
||||||
|
pollfd.fd = uffd;
|
||||||
|
pollfd.events = POLLIN;
|
||||||
|
|
||||||
|
// Block until the descriptor is ready for data reads.
|
||||||
|
const int pollres = poll(&pollfd, 1, -1);
|
||||||
|
switch (pollres) {
|
||||||
|
case -1:
|
||||||
|
perror("Poll userfaultfd");
|
||||||
|
continue;
|
||||||
|
break;
|
||||||
|
case 0:
|
||||||
|
continue;
|
||||||
|
case 1:
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
UNREACHABLE_MSG("Unexpected number of descriptors {} out of poll", pollres);
|
||||||
|
}
|
||||||
|
|
||||||
|
// We don't want an error condition to have occured.
|
||||||
|
ASSERT_MSG(!(pollfd.revents & POLLERR), "POLLERR on userfaultfd");
|
||||||
|
|
||||||
|
// We waited until there is data to read, we don't care about anything else.
|
||||||
|
if (!(pollfd.revents & POLLIN)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read message from kernel.
|
||||||
|
uffd_msg msg;
|
||||||
|
const int readret = read(uffd, &msg, sizeof(msg));
|
||||||
|
ASSERT_MSG(readret != -1 || errno == EAGAIN, "Unexpected result of uffd read");
|
||||||
|
if (errno == EAGAIN) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ASSERT_MSG(readret == sizeof(msg), "Unexpected short read, exiting");
|
||||||
|
ASSERT(msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP);
|
||||||
|
|
||||||
|
// Notify rasterizer about the fault.
|
||||||
|
const VAddr addr = msg.arg.pagefault.address;
|
||||||
|
const VAddr addr_page = Common::AlignDown(addr, PAGESIZE);
|
||||||
|
rasterizer->InvalidateMemory(addr_page, PAGESIZE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Vulkan::Rasterizer* rasterizer;
|
||||||
|
std::jthread ufd_thread;
|
||||||
|
int uffd;
|
||||||
|
};
|
||||||
|
#else
|
||||||
|
struct PageManager::Impl {
|
||||||
|
Impl(Vulkan::Rasterizer* rasterizer_) {
|
||||||
|
rasterizer = rasterizer_;
|
||||||
|
|
||||||
|
#ifdef __APPLE__
|
||||||
|
// Read-only memory write results in SIGBUS on Apple.
|
||||||
|
static constexpr int SignalType = SIGBUS;
|
||||||
|
#else
|
||||||
|
static constexpr int SignalType = SIGSEGV;
|
||||||
|
#endif
|
||||||
|
sigset_t signal_mask;
|
||||||
|
sigemptyset(&signal_mask);
|
||||||
|
sigaddset(&signal_mask, SignalType);
|
||||||
|
|
||||||
|
using HandlerType = decltype(sigaction::sa_sigaction);
|
||||||
|
|
||||||
|
struct sigaction guest_access_fault {};
|
||||||
|
guest_access_fault.sa_flags = SA_SIGINFO | SA_ONSTACK;
|
||||||
|
guest_access_fault.sa_sigaction = &GuestFaultSignalHandler;
|
||||||
|
guest_access_fault.sa_mask = signal_mask;
|
||||||
|
sigaction(SignalType, &guest_access_fault, nullptr);
|
||||||
|
}
|
||||||
|
|
||||||
|
void OnMap(VAddr address, size_t size) {}
|
||||||
|
|
||||||
|
void OnUnmap(VAddr address, size_t size) {}
|
||||||
|
|
||||||
|
void Protect(VAddr address, size_t size, bool allow_write) {
|
||||||
|
mprotect(reinterpret_cast<void*>(address), size,
|
||||||
|
PROT_READ | (allow_write ? PROT_WRITE : 0));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void GuestFaultSignalHandler(int sig, siginfo_t* info, void* raw_context) {
|
||||||
|
ucontext_t* ctx = reinterpret_cast<ucontext_t*>(raw_context);
|
||||||
|
const VAddr address = reinterpret_cast<VAddr>(info->si_addr);
|
||||||
|
#ifdef __APPLE__
|
||||||
|
const u32 err = ctx->uc_mcontext->__es.__err;
|
||||||
|
#else
|
||||||
|
const greg_t err = ctx->uc_mcontext.gregs[REG_ERR];
|
||||||
|
#endif
|
||||||
|
if (err & 0x2) {
|
||||||
|
rasterizer->InvalidateMemory(address, sizeof(u64));
|
||||||
|
} else {
|
||||||
|
// Read not supported!
|
||||||
|
UNREACHABLE();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
inline static Vulkan::Rasterizer* rasterizer;
|
||||||
|
};
|
||||||
|
#endif
|
||||||
|
|
||||||
|
PageManager::PageManager(Vulkan::Rasterizer* rasterizer_)
|
||||||
|
: impl{std::make_unique<Impl>(rasterizer_)}, rasterizer{rasterizer_} {}
|
||||||
|
|
||||||
|
PageManager::~PageManager() = default;
|
||||||
|
|
||||||
|
void PageManager::OnGpuMap(VAddr address, size_t size) {
|
||||||
|
impl->OnMap(address, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
void PageManager::OnGpuUnmap(VAddr address, size_t size) {
|
||||||
|
impl->OnUnmap(address, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
void PageManager::UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta) {
|
||||||
|
static constexpr u64 PageShift = 12;
|
||||||
|
|
||||||
|
std::scoped_lock lk{mutex};
|
||||||
|
const u64 num_pages = ((addr + size - 1) >> PageShift) - (addr >> PageShift) + 1;
|
||||||
|
const u64 page_start = addr >> PageShift;
|
||||||
|
const u64 page_end = page_start + num_pages;
|
||||||
|
|
||||||
|
const auto pages_interval =
|
||||||
|
decltype(cached_pages)::interval_type::right_open(page_start, page_end);
|
||||||
|
if (delta > 0) {
|
||||||
|
cached_pages.add({pages_interval, delta});
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto& range = cached_pages.equal_range(pages_interval);
|
||||||
|
for (const auto& [range, count] : boost::make_iterator_range(range)) {
|
||||||
|
const auto interval = range & pages_interval;
|
||||||
|
const VAddr interval_start_addr = boost::icl::first(interval) << PageShift;
|
||||||
|
const VAddr interval_end_addr = boost::icl::last_next(interval) << PageShift;
|
||||||
|
const u32 interval_size = interval_end_addr - interval_start_addr;
|
||||||
|
if (delta > 0 && count == delta) {
|
||||||
|
impl->Protect(interval_start_addr, interval_size, false);
|
||||||
|
} else if (delta < 0 && count == -delta) {
|
||||||
|
impl->Protect(interval_start_addr, interval_size, true);
|
||||||
|
} else {
|
||||||
|
ASSERT(count >= 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (delta < 0) {
|
||||||
|
cached_pages.add({pages_interval, delta});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -0,0 +1,39 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <mutex>
|
||||||
|
#include <boost/icl/interval_map.hpp>
|
||||||
|
#include "common/types.h"
|
||||||
|
|
||||||
|
namespace Vulkan {
|
||||||
|
class Rasterizer;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace VideoCore {
|
||||||
|
|
||||||
|
class PageManager {
|
||||||
|
public:
|
||||||
|
explicit PageManager(Vulkan::Rasterizer* rasterizer);
|
||||||
|
~PageManager();
|
||||||
|
|
||||||
|
/// Register a range of mapped gpu memory.
|
||||||
|
void OnGpuMap(VAddr address, size_t size);
|
||||||
|
|
||||||
|
/// Unregister a range of gpu memory that was unmapped.
|
||||||
|
void OnGpuUnmap(VAddr address, size_t size);
|
||||||
|
|
||||||
|
/// Increase/decrease the number of surface in pages touching the specified region
|
||||||
|
void UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta);
|
||||||
|
|
||||||
|
private:
|
||||||
|
struct Impl;
|
||||||
|
std::unique_ptr<Impl> impl;
|
||||||
|
Vulkan::Rasterizer* rasterizer;
|
||||||
|
std::mutex mutex;
|
||||||
|
boost::icl::interval_map<VAddr, s32> cached_pages;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace VideoCore
|
|
@ -67,8 +67,8 @@ RendererVulkan::RendererVulkan(Frontend::WindowSDL& window_, AmdGpu::Liverpool*
|
||||||
: window{window_}, liverpool{liverpool_},
|
: window{window_}, liverpool{liverpool_},
|
||||||
instance{window, Config::getGpuId(), Config::vkValidationEnabled()}, draw_scheduler{instance},
|
instance{window, Config::getGpuId(), Config::vkValidationEnabled()}, draw_scheduler{instance},
|
||||||
present_scheduler{instance}, flip_scheduler{instance}, swapchain{instance, window},
|
present_scheduler{instance}, flip_scheduler{instance}, swapchain{instance, window},
|
||||||
texture_cache{instance, draw_scheduler} {
|
rasterizer{std::make_unique<Rasterizer>(instance, draw_scheduler, liverpool)},
|
||||||
rasterizer = std::make_unique<Rasterizer>(instance, draw_scheduler, texture_cache, liverpool);
|
texture_cache{rasterizer->GetTextureCache()} {
|
||||||
const u32 num_images = swapchain.GetImageCount();
|
const u32 num_images = swapchain.GetImageCount();
|
||||||
const vk::Device device = instance.GetDevice();
|
const vk::Device device = instance.GetDevice();
|
||||||
|
|
||||||
|
|
|
@ -47,7 +47,7 @@ public:
|
||||||
Frame* PrepareFrame(const Libraries::VideoOut::BufferAttributeGroup& attribute,
|
Frame* PrepareFrame(const Libraries::VideoOut::BufferAttributeGroup& attribute,
|
||||||
VAddr cpu_address, bool is_eop) {
|
VAddr cpu_address, bool is_eop) {
|
||||||
const auto info = VideoCore::ImageInfo{attribute, cpu_address};
|
const auto info = VideoCore::ImageInfo{attribute, cpu_address};
|
||||||
const auto image_id = texture_cache.FindImage(info, cpu_address);
|
const auto image_id = texture_cache.FindImage(info, false);
|
||||||
auto& image = texture_cache.GetImage(image_id);
|
auto& image = texture_cache.GetImage(image_id);
|
||||||
return PrepareFrameInternal(image, is_eop);
|
return PrepareFrameInternal(image, is_eop);
|
||||||
}
|
}
|
||||||
|
@ -61,7 +61,7 @@ public:
|
||||||
const Libraries::VideoOut::BufferAttributeGroup& attribute, VAddr cpu_address) {
|
const Libraries::VideoOut::BufferAttributeGroup& attribute, VAddr cpu_address) {
|
||||||
vo_buffers_addr.emplace_back(cpu_address);
|
vo_buffers_addr.emplace_back(cpu_address);
|
||||||
const auto info = VideoCore::ImageInfo{attribute, cpu_address};
|
const auto info = VideoCore::ImageInfo{attribute, cpu_address};
|
||||||
const auto image_id = texture_cache.FindImage(info, cpu_address);
|
const auto image_id = texture_cache.FindImage(info, false);
|
||||||
return texture_cache.GetImage(image_id);
|
return texture_cache.GetImage(image_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -88,7 +88,7 @@ private:
|
||||||
Scheduler flip_scheduler;
|
Scheduler flip_scheduler;
|
||||||
Swapchain swapchain;
|
Swapchain swapchain;
|
||||||
std::unique_ptr<Rasterizer> rasterizer;
|
std::unique_ptr<Rasterizer> rasterizer;
|
||||||
VideoCore::TextureCache texture_cache;
|
VideoCore::TextureCache& texture_cache;
|
||||||
vk::UniqueCommandPool command_pool;
|
vk::UniqueCommandPool command_pool;
|
||||||
std::vector<Frame> present_frames;
|
std::vector<Frame> present_frames;
|
||||||
std::queue<Frame*> free_queue;
|
std::queue<Frame*> free_queue;
|
||||||
|
|
|
@ -3,11 +3,10 @@
|
||||||
|
|
||||||
#include <boost/container/small_vector.hpp>
|
#include <boost/container/small_vector.hpp>
|
||||||
#include "common/alignment.h"
|
#include "common/alignment.h"
|
||||||
#include "core/memory.h"
|
#include "video_core/buffer_cache/buffer_cache.h"
|
||||||
#include "video_core/renderer_vulkan/vk_compute_pipeline.h"
|
#include "video_core/renderer_vulkan/vk_compute_pipeline.h"
|
||||||
#include "video_core/renderer_vulkan/vk_instance.h"
|
#include "video_core/renderer_vulkan/vk_instance.h"
|
||||||
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
||||||
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
|
|
||||||
#include "video_core/texture_cache/texture_cache.h"
|
#include "video_core/texture_cache/texture_cache.h"
|
||||||
|
|
||||||
namespace Vulkan {
|
namespace Vulkan {
|
||||||
|
@ -51,6 +50,12 @@ ComputePipeline::ComputePipeline(const Instance& instance_, Scheduler& scheduler
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const vk::PushConstantRange push_constants = {
|
||||||
|
.stageFlags = vk::ShaderStageFlagBits::eCompute,
|
||||||
|
.offset = 0,
|
||||||
|
.size = sizeof(Shader::PushData),
|
||||||
|
};
|
||||||
|
|
||||||
const vk::DescriptorSetLayoutCreateInfo desc_layout_ci = {
|
const vk::DescriptorSetLayoutCreateInfo desc_layout_ci = {
|
||||||
.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR,
|
.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR,
|
||||||
.bindingCount = static_cast<u32>(bindings.size()),
|
.bindingCount = static_cast<u32>(bindings.size()),
|
||||||
|
@ -62,8 +67,8 @@ ComputePipeline::ComputePipeline(const Instance& instance_, Scheduler& scheduler
|
||||||
const vk::PipelineLayoutCreateInfo layout_info = {
|
const vk::PipelineLayoutCreateInfo layout_info = {
|
||||||
.setLayoutCount = 1U,
|
.setLayoutCount = 1U,
|
||||||
.pSetLayouts = &set_layout,
|
.pSetLayouts = &set_layout,
|
||||||
.pushConstantRangeCount = 0,
|
.pushConstantRangeCount = 1U,
|
||||||
.pPushConstantRanges = nullptr,
|
.pPushConstantRanges = &push_constants,
|
||||||
};
|
};
|
||||||
pipeline_layout = instance.GetDevice().createPipelineLayoutUnique(layout_info);
|
pipeline_layout = instance.GetDevice().createPipelineLayoutUnique(layout_info);
|
||||||
|
|
||||||
|
@ -82,35 +87,18 @@ ComputePipeline::ComputePipeline(const Instance& instance_, Scheduler& scheduler
|
||||||
|
|
||||||
ComputePipeline::~ComputePipeline() = default;
|
ComputePipeline::~ComputePipeline() = default;
|
||||||
|
|
||||||
bool ComputePipeline::BindResources(Core::MemoryManager* memory, StreamBuffer& staging,
|
bool ComputePipeline::BindResources(VideoCore::BufferCache& buffer_cache,
|
||||||
VideoCore::TextureCache& texture_cache) const {
|
VideoCore::TextureCache& texture_cache) const {
|
||||||
// Bind resource buffers and textures.
|
// Bind resource buffers and textures.
|
||||||
boost::container::static_vector<vk::DescriptorBufferInfo, 16> buffer_infos;
|
boost::container::static_vector<vk::DescriptorBufferInfo, 16> buffer_infos;
|
||||||
boost::container::static_vector<vk::DescriptorImageInfo, 16> image_infos;
|
boost::container::static_vector<vk::DescriptorImageInfo, 16> image_infos;
|
||||||
boost::container::small_vector<vk::WriteDescriptorSet, 16> set_writes;
|
boost::container::small_vector<vk::WriteDescriptorSet, 16> set_writes;
|
||||||
|
Shader::PushData push_data{};
|
||||||
u32 binding{};
|
u32 binding{};
|
||||||
|
|
||||||
for (const auto& buffer : info.buffers) {
|
for (u32 i = 0; const auto& buffer : info.buffers) {
|
||||||
const auto vsharp = buffer.GetVsharp(info);
|
const auto vsharp = buffer.GetVsharp(info);
|
||||||
const u32 size = vsharp.GetSize();
|
|
||||||
const VAddr address = vsharp.base_address;
|
const VAddr address = vsharp.base_address;
|
||||||
if (buffer.is_storage) {
|
|
||||||
texture_cache.OnCpuWrite(address);
|
|
||||||
}
|
|
||||||
const u32 offset = staging.Copy(address, size,
|
|
||||||
buffer.is_storage ? instance.StorageMinAlignment()
|
|
||||||
: instance.UniformMinAlignment());
|
|
||||||
buffer_infos.emplace_back(staging.Handle(), offset, size);
|
|
||||||
set_writes.push_back({
|
|
||||||
.dstSet = VK_NULL_HANDLE,
|
|
||||||
.dstBinding = binding++,
|
|
||||||
.dstArrayElement = 0,
|
|
||||||
.descriptorCount = 1,
|
|
||||||
.descriptorType = buffer.is_storage ? vk::DescriptorType::eStorageBuffer
|
|
||||||
: vk::DescriptorType::eUniformBuffer,
|
|
||||||
.pBufferInfo = &buffer_infos.back(),
|
|
||||||
});
|
|
||||||
|
|
||||||
// Most of the time when a metadata is updated with a shader it gets cleared. It means we
|
// Most of the time when a metadata is updated with a shader it gets cleared. It means we
|
||||||
// can skip the whole dispatch and update the tracked state instead. Also, it is not
|
// can skip the whole dispatch and update the tracked state instead. Also, it is not
|
||||||
// intended to be consumed and in such rare cases (e.g. HTile introspection, CRAA) we will
|
// intended to be consumed and in such rare cases (e.g. HTile introspection, CRAA) we will
|
||||||
|
@ -125,6 +113,31 @@ bool ComputePipeline::BindResources(Core::MemoryManager* memory, StreamBuffer& s
|
||||||
LOG_WARNING(Render_Vulkan, "Unexpected metadata read by a CS shader (buffer)");
|
LOG_WARNING(Render_Vulkan, "Unexpected metadata read by a CS shader (buffer)");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
const u32 size = vsharp.GetSize();
|
||||||
|
if (buffer.is_written) {
|
||||||
|
texture_cache.InvalidateMemory(address, size);
|
||||||
|
}
|
||||||
|
const u32 alignment =
|
||||||
|
buffer.is_storage ? instance.StorageMinAlignment() : instance.UniformMinAlignment();
|
||||||
|
const auto [vk_buffer, offset] =
|
||||||
|
buffer_cache.ObtainBuffer(address, size, buffer.is_written);
|
||||||
|
const u32 offset_aligned = Common::AlignDown(offset, alignment);
|
||||||
|
const u32 adjust = offset - offset_aligned;
|
||||||
|
if (adjust != 0) {
|
||||||
|
ASSERT(adjust % 4 == 0);
|
||||||
|
push_data.AddOffset(i, adjust);
|
||||||
|
}
|
||||||
|
buffer_infos.emplace_back(vk_buffer->Handle(), offset_aligned, size + adjust);
|
||||||
|
set_writes.push_back({
|
||||||
|
.dstSet = VK_NULL_HANDLE,
|
||||||
|
.dstBinding = binding++,
|
||||||
|
.dstArrayElement = 0,
|
||||||
|
.descriptorCount = 1,
|
||||||
|
.descriptorType = buffer.is_storage ? vk::DescriptorType::eStorageBuffer
|
||||||
|
: vk::DescriptorType::eUniformBuffer,
|
||||||
|
.pBufferInfo = &buffer_infos.back(),
|
||||||
|
});
|
||||||
|
i++;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const auto& image_desc : info.images) {
|
for (const auto& image_desc : info.images) {
|
||||||
|
@ -168,6 +181,8 @@ bool ComputePipeline::BindResources(Core::MemoryManager* memory, StreamBuffer& s
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto cmdbuf = scheduler.CommandBuffer();
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
cmdbuf.pushConstants(*pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0u, sizeof(push_data),
|
||||||
|
&push_data);
|
||||||
cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, *pipeline_layout, 0, set_writes);
|
cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, *pipeline_layout, 0, set_writes);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,19 +6,15 @@
|
||||||
#include "shader_recompiler/runtime_info.h"
|
#include "shader_recompiler/runtime_info.h"
|
||||||
#include "video_core/renderer_vulkan/vk_common.h"
|
#include "video_core/renderer_vulkan/vk_common.h"
|
||||||
|
|
||||||
namespace Core {
|
|
||||||
class MemoryManager;
|
|
||||||
}
|
|
||||||
|
|
||||||
namespace VideoCore {
|
namespace VideoCore {
|
||||||
|
class BufferCache;
|
||||||
class TextureCache;
|
class TextureCache;
|
||||||
}
|
} // namespace VideoCore
|
||||||
|
|
||||||
namespace Vulkan {
|
namespace Vulkan {
|
||||||
|
|
||||||
class Instance;
|
class Instance;
|
||||||
class Scheduler;
|
class Scheduler;
|
||||||
class StreamBuffer;
|
|
||||||
|
|
||||||
class ComputePipeline {
|
class ComputePipeline {
|
||||||
public:
|
public:
|
||||||
|
@ -31,7 +27,7 @@ public:
|
||||||
return *pipeline;
|
return *pipeline;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool BindResources(Core::MemoryManager* memory, StreamBuffer& staging,
|
bool BindResources(VideoCore::BufferCache& buffer_cache,
|
||||||
VideoCore::TextureCache& texture_cache) const;
|
VideoCore::TextureCache& texture_cache) const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
|
@ -5,13 +5,13 @@
|
||||||
#include <boost/container/small_vector.hpp>
|
#include <boost/container/small_vector.hpp>
|
||||||
#include <boost/container/static_vector.hpp>
|
#include <boost/container/static_vector.hpp>
|
||||||
|
|
||||||
|
#include "common/alignment.h"
|
||||||
#include "common/assert.h"
|
#include "common/assert.h"
|
||||||
#include "core/memory.h"
|
|
||||||
#include "video_core/amdgpu/resource.h"
|
#include "video_core/amdgpu/resource.h"
|
||||||
|
#include "video_core/buffer_cache/buffer_cache.h"
|
||||||
#include "video_core/renderer_vulkan/vk_graphics_pipeline.h"
|
#include "video_core/renderer_vulkan/vk_graphics_pipeline.h"
|
||||||
#include "video_core/renderer_vulkan/vk_instance.h"
|
#include "video_core/renderer_vulkan/vk_instance.h"
|
||||||
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
||||||
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
|
|
||||||
#include "video_core/texture_cache/texture_cache.h"
|
#include "video_core/texture_cache/texture_cache.h"
|
||||||
|
|
||||||
namespace Vulkan {
|
namespace Vulkan {
|
||||||
|
@ -32,9 +32,9 @@ GraphicsPipeline::GraphicsPipeline(const Instance& instance_, Scheduler& schedul
|
||||||
BuildDescSetLayout();
|
BuildDescSetLayout();
|
||||||
|
|
||||||
const vk::PushConstantRange push_constants = {
|
const vk::PushConstantRange push_constants = {
|
||||||
.stageFlags = vk::ShaderStageFlagBits::eVertex,
|
.stageFlags = vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment,
|
||||||
.offset = 0,
|
.offset = 0,
|
||||||
.size = 2 * sizeof(u32),
|
.size = sizeof(Shader::PushData),
|
||||||
};
|
};
|
||||||
|
|
||||||
const vk::DescriptorSetLayout set_layout = *desc_layout;
|
const vk::DescriptorSetLayout set_layout = *desc_layout;
|
||||||
|
@ -328,25 +328,43 @@ void GraphicsPipeline::BuildDescSetLayout() {
|
||||||
desc_layout = instance.GetDevice().createDescriptorSetLayoutUnique(desc_layout_ci);
|
desc_layout = instance.GetDevice().createDescriptorSetLayoutUnique(desc_layout_ci);
|
||||||
}
|
}
|
||||||
|
|
||||||
void GraphicsPipeline::BindResources(Core::MemoryManager* memory, StreamBuffer& staging,
|
void GraphicsPipeline::BindResources(const Liverpool::Regs& regs,
|
||||||
|
VideoCore::BufferCache& buffer_cache,
|
||||||
VideoCore::TextureCache& texture_cache) const {
|
VideoCore::TextureCache& texture_cache) const {
|
||||||
BindVertexBuffers(staging);
|
|
||||||
|
|
||||||
// Bind resource buffers and textures.
|
// Bind resource buffers and textures.
|
||||||
boost::container::static_vector<vk::DescriptorBufferInfo, 16> buffer_infos;
|
boost::container::static_vector<vk::DescriptorBufferInfo, 16> buffer_infos;
|
||||||
boost::container::static_vector<vk::DescriptorImageInfo, 32> image_infos;
|
boost::container::static_vector<vk::DescriptorImageInfo, 32> image_infos;
|
||||||
boost::container::small_vector<vk::WriteDescriptorSet, 16> set_writes;
|
boost::container::small_vector<vk::WriteDescriptorSet, 16> set_writes;
|
||||||
|
Shader::PushData push_data{};
|
||||||
u32 binding{};
|
u32 binding{};
|
||||||
|
|
||||||
for (const auto& stage : stages) {
|
for (const auto& stage : stages) {
|
||||||
for (const auto& buffer : stage.buffers) {
|
if (stage.uses_step_rates) {
|
||||||
|
push_data.step0 = regs.vgt_instance_step_rate_0;
|
||||||
|
push_data.step1 = regs.vgt_instance_step_rate_1;
|
||||||
|
}
|
||||||
|
for (u32 i = 0; const auto& buffer : stage.buffers) {
|
||||||
const auto vsharp = buffer.GetVsharp(stage);
|
const auto vsharp = buffer.GetVsharp(stage);
|
||||||
|
if (vsharp) {
|
||||||
const VAddr address = vsharp.base_address;
|
const VAddr address = vsharp.base_address;
|
||||||
|
if (texture_cache.IsMeta(address)) {
|
||||||
|
LOG_WARNING(Render_Vulkan, "Unexpected metadata read by a PS shader (buffer)");
|
||||||
|
}
|
||||||
const u32 size = vsharp.GetSize();
|
const u32 size = vsharp.GetSize();
|
||||||
const u32 offset = staging.Copy(address, size,
|
const u32 alignment = buffer.is_storage ? instance.StorageMinAlignment()
|
||||||
buffer.is_storage ? instance.StorageMinAlignment()
|
: instance.UniformMinAlignment();
|
||||||
: instance.UniformMinAlignment());
|
const auto [vk_buffer, offset] =
|
||||||
buffer_infos.emplace_back(staging.Handle(), offset, size);
|
buffer_cache.ObtainBuffer(address, size, buffer.is_written);
|
||||||
|
const u32 offset_aligned = Common::AlignDown(offset, alignment);
|
||||||
|
const u32 adjust = offset - offset_aligned;
|
||||||
|
if (adjust != 0) {
|
||||||
|
ASSERT(adjust % 4 == 0);
|
||||||
|
push_data.AddOffset(i, adjust);
|
||||||
|
}
|
||||||
|
buffer_infos.emplace_back(vk_buffer->Handle(), offset_aligned, size + adjust);
|
||||||
|
} else {
|
||||||
|
buffer_infos.emplace_back(VK_NULL_HANDLE, 0, VK_WHOLE_SIZE);
|
||||||
|
}
|
||||||
set_writes.push_back({
|
set_writes.push_back({
|
||||||
.dstSet = VK_NULL_HANDLE,
|
.dstSet = VK_NULL_HANDLE,
|
||||||
.dstBinding = binding++,
|
.dstBinding = binding++,
|
||||||
|
@ -356,10 +374,7 @@ void GraphicsPipeline::BindResources(Core::MemoryManager* memory, StreamBuffer&
|
||||||
: vk::DescriptorType::eUniformBuffer,
|
: vk::DescriptorType::eUniformBuffer,
|
||||||
.pBufferInfo = &buffer_infos.back(),
|
.pBufferInfo = &buffer_infos.back(),
|
||||||
});
|
});
|
||||||
|
i++;
|
||||||
if (texture_cache.IsMeta(address)) {
|
|
||||||
LOG_WARNING(Render_Vulkan, "Unexpected metadata read by a PS shader (buffer)");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
boost::container::static_vector<AmdGpu::Image, 16> tsharps;
|
boost::container::static_vector<AmdGpu::Image, 16> tsharps;
|
||||||
|
@ -406,86 +421,15 @@ void GraphicsPipeline::BindResources(Core::MemoryManager* memory, StreamBuffer&
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!set_writes.empty()) {
|
|
||||||
const auto cmdbuf = scheduler.CommandBuffer();
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
|
if (!set_writes.empty()) {
|
||||||
cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eGraphics, *pipeline_layout, 0,
|
cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eGraphics, *pipeline_layout, 0,
|
||||||
set_writes);
|
set_writes);
|
||||||
}
|
}
|
||||||
}
|
cmdbuf.pushConstants(*pipeline_layout,
|
||||||
|
vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment, 0U,
|
||||||
void GraphicsPipeline::BindVertexBuffers(StreamBuffer& staging) const {
|
sizeof(push_data), &push_data);
|
||||||
const auto& vs_info = stages[u32(Shader::Stage::Vertex)];
|
cmdbuf.bindPipeline(vk::PipelineBindPoint::eGraphics, Handle());
|
||||||
if (vs_info.vs_inputs.empty()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::array<vk::Buffer, MaxVertexBufferCount> host_buffers;
|
|
||||||
std::array<vk::DeviceSize, MaxVertexBufferCount> host_offsets;
|
|
||||||
boost::container::static_vector<AmdGpu::Buffer, MaxVertexBufferCount> guest_buffers;
|
|
||||||
|
|
||||||
struct BufferRange {
|
|
||||||
VAddr base_address;
|
|
||||||
VAddr end_address;
|
|
||||||
u64 offset; // offset in the mapped memory
|
|
||||||
|
|
||||||
size_t GetSize() const {
|
|
||||||
return end_address - base_address;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Calculate buffers memory overlaps
|
|
||||||
boost::container::static_vector<BufferRange, MaxVertexBufferCount> ranges{};
|
|
||||||
for (const auto& input : vs_info.vs_inputs) {
|
|
||||||
if (input.instance_step_rate == Shader::Info::VsInput::InstanceIdType::OverStepRate0 ||
|
|
||||||
input.instance_step_rate == Shader::Info::VsInput::InstanceIdType::OverStepRate1) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const auto& buffer = vs_info.ReadUd<AmdGpu::Buffer>(input.sgpr_base, input.dword_offset);
|
|
||||||
if (buffer.GetSize() == 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
guest_buffers.emplace_back(buffer);
|
|
||||||
ranges.emplace_back(buffer.base_address, buffer.base_address + buffer.GetSize());
|
|
||||||
}
|
|
||||||
std::ranges::sort(ranges, [](const BufferRange& lhv, const BufferRange& rhv) {
|
|
||||||
return lhv.base_address < rhv.base_address;
|
|
||||||
});
|
|
||||||
|
|
||||||
boost::container::static_vector<BufferRange, MaxVertexBufferCount> ranges_merged{ranges[0]};
|
|
||||||
for (auto range : ranges) {
|
|
||||||
auto& prev_range = ranges_merged.back();
|
|
||||||
if (prev_range.end_address < range.base_address) {
|
|
||||||
ranges_merged.emplace_back(range);
|
|
||||||
} else {
|
|
||||||
prev_range.end_address = std::max(prev_range.end_address, range.end_address);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Map buffers
|
|
||||||
for (auto& range : ranges_merged) {
|
|
||||||
range.offset = staging.Copy(range.base_address, range.GetSize(), 4);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Bind vertex buffers
|
|
||||||
const size_t num_buffers = guest_buffers.size();
|
|
||||||
for (u32 i = 0; i < num_buffers; ++i) {
|
|
||||||
const auto& buffer = guest_buffers[i];
|
|
||||||
const auto& host_buffer = std::ranges::find_if(
|
|
||||||
ranges_merged.cbegin(), ranges_merged.cend(), [&](const BufferRange& range) {
|
|
||||||
return (buffer.base_address >= range.base_address &&
|
|
||||||
buffer.base_address < range.end_address);
|
|
||||||
});
|
|
||||||
assert(host_buffer != ranges_merged.cend());
|
|
||||||
|
|
||||||
host_buffers[i] = staging.Handle();
|
|
||||||
host_offsets[i] = host_buffer->offset + buffer.base_address - host_buffer->base_address;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (num_buffers > 0) {
|
|
||||||
const auto cmdbuf = scheduler.CommandBuffer();
|
|
||||||
cmdbuf.bindVertexBuffers(0, num_buffers, host_buffers.data(), host_offsets.data());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace Vulkan
|
} // namespace Vulkan
|
||||||
|
|
|
@ -7,13 +7,10 @@
|
||||||
#include "video_core/renderer_vulkan/liverpool_to_vk.h"
|
#include "video_core/renderer_vulkan/liverpool_to_vk.h"
|
||||||
#include "video_core/renderer_vulkan/vk_common.h"
|
#include "video_core/renderer_vulkan/vk_common.h"
|
||||||
|
|
||||||
namespace Core {
|
|
||||||
class MemoryManager;
|
|
||||||
}
|
|
||||||
|
|
||||||
namespace VideoCore {
|
namespace VideoCore {
|
||||||
|
class BufferCache;
|
||||||
class TextureCache;
|
class TextureCache;
|
||||||
}
|
} // namespace VideoCore
|
||||||
|
|
||||||
namespace Vulkan {
|
namespace Vulkan {
|
||||||
|
|
||||||
|
@ -22,7 +19,6 @@ static constexpr u32 MaxShaderStages = 5;
|
||||||
|
|
||||||
class Instance;
|
class Instance;
|
||||||
class Scheduler;
|
class Scheduler;
|
||||||
class StreamBuffer;
|
|
||||||
|
|
||||||
using Liverpool = AmdGpu::Liverpool;
|
using Liverpool = AmdGpu::Liverpool;
|
||||||
|
|
||||||
|
@ -64,7 +60,7 @@ public:
|
||||||
std::array<vk::ShaderModule, MaxShaderStages> modules);
|
std::array<vk::ShaderModule, MaxShaderStages> modules);
|
||||||
~GraphicsPipeline();
|
~GraphicsPipeline();
|
||||||
|
|
||||||
void BindResources(Core::MemoryManager* memory, StreamBuffer& staging,
|
void BindResources(const Liverpool::Regs& regs, VideoCore::BufferCache& buffer_cache,
|
||||||
VideoCore::TextureCache& texture_cache) const;
|
VideoCore::TextureCache& texture_cache) const;
|
||||||
|
|
||||||
vk::Pipeline Handle() const noexcept {
|
vk::Pipeline Handle() const noexcept {
|
||||||
|
@ -75,6 +71,10 @@ public:
|
||||||
return *pipeline_layout;
|
return *pipeline_layout;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const Shader::Info& GetStage(Shader::Stage stage) const noexcept {
|
||||||
|
return stages[u32(stage)];
|
||||||
|
}
|
||||||
|
|
||||||
bool IsEmbeddedVs() const noexcept {
|
bool IsEmbeddedVs() const noexcept {
|
||||||
static constexpr size_t EmbeddedVsHash = 0x9b2da5cf47f8c29f;
|
static constexpr size_t EmbeddedVsHash = 0x9b2da5cf47f8c29f;
|
||||||
return key.stage_hashes[u32(Shader::Stage::Vertex)] == EmbeddedVsHash;
|
return key.stage_hashes[u32(Shader::Stage::Vertex)] == EmbeddedVsHash;
|
||||||
|
@ -90,7 +90,6 @@ public:
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void BuildDescSetLayout();
|
void BuildDescSetLayout();
|
||||||
void BindVertexBuffers(StreamBuffer& staging) const;
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
const Instance& instance;
|
const Instance& instance;
|
||||||
|
|
|
@ -204,7 +204,8 @@ bool Instance::CreateDevice() {
|
||||||
// The next two extensions are required to be available together in order to support write masks
|
// The next two extensions are required to be available together in order to support write masks
|
||||||
color_write_en = add_extension(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
|
color_write_en = add_extension(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
|
||||||
color_write_en &= add_extension(VK_EXT_EXTENDED_DYNAMIC_STATE_3_EXTENSION_NAME);
|
color_write_en &= add_extension(VK_EXT_EXTENDED_DYNAMIC_STATE_3_EXTENSION_NAME);
|
||||||
const auto calibrated_timestamps = add_extension(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME);
|
const bool calibrated_timestamps = add_extension(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME);
|
||||||
|
const bool robustness = add_extension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
|
||||||
|
|
||||||
// These extensions are promoted by Vulkan 1.3, but for greater compatibility we use Vulkan 1.2
|
// These extensions are promoted by Vulkan 1.3, but for greater compatibility we use Vulkan 1.2
|
||||||
// with extensions.
|
// with extensions.
|
||||||
|
@ -303,12 +304,19 @@ bool Instance::CreateDevice() {
|
||||||
.workgroupMemoryExplicitLayoutScalarBlockLayout = true,
|
.workgroupMemoryExplicitLayoutScalarBlockLayout = true,
|
||||||
.workgroupMemoryExplicitLayout8BitAccess = true,
|
.workgroupMemoryExplicitLayout8BitAccess = true,
|
||||||
.workgroupMemoryExplicitLayout16BitAccess = true,
|
.workgroupMemoryExplicitLayout16BitAccess = true,
|
||||||
}};
|
},
|
||||||
|
vk::PhysicalDeviceRobustness2FeaturesEXT{
|
||||||
|
.nullDescriptor = true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
if (!color_write_en) {
|
if (!color_write_en) {
|
||||||
device_chain.unlink<vk::PhysicalDeviceColorWriteEnableFeaturesEXT>();
|
device_chain.unlink<vk::PhysicalDeviceColorWriteEnableFeaturesEXT>();
|
||||||
device_chain.unlink<vk::PhysicalDeviceExtendedDynamicState3FeaturesEXT>();
|
device_chain.unlink<vk::PhysicalDeviceExtendedDynamicState3FeaturesEXT>();
|
||||||
}
|
}
|
||||||
|
if (!robustness) {
|
||||||
|
device_chain.unlink<vk::PhysicalDeviceRobustness2FeaturesEXT>();
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
device = physical_device.createDeviceUnique(device_chain.get());
|
device = physical_device.createDeviceUnique(device_chain.get());
|
||||||
|
|
|
@ -5,7 +5,6 @@
|
||||||
|
|
||||||
#include <tsl/robin_map.h>
|
#include <tsl/robin_map.h>
|
||||||
#include "shader_recompiler/ir/basic_block.h"
|
#include "shader_recompiler/ir/basic_block.h"
|
||||||
#include "shader_recompiler/object_pool.h"
|
|
||||||
#include "shader_recompiler/profile.h"
|
#include "shader_recompiler/profile.h"
|
||||||
#include "video_core/renderer_vulkan/vk_compute_pipeline.h"
|
#include "video_core/renderer_vulkan/vk_compute_pipeline.h"
|
||||||
#include "video_core/renderer_vulkan/vk_graphics_pipeline.h"
|
#include "video_core/renderer_vulkan/vk_graphics_pipeline.h"
|
||||||
|
@ -51,8 +50,8 @@ private:
|
||||||
Shader::Profile profile{};
|
Shader::Profile profile{};
|
||||||
GraphicsPipelineKey graphics_key{};
|
GraphicsPipelineKey graphics_key{};
|
||||||
u64 compute_key{};
|
u64 compute_key{};
|
||||||
Shader::ObjectPool<Shader::IR::Inst> inst_pool;
|
Common::ObjectPool<Shader::IR::Inst> inst_pool;
|
||||||
Shader::ObjectPool<Shader::IR::Block> block_pool;
|
Common::ObjectPool<Shader::IR::Block> block_pool;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace Vulkan
|
} // namespace Vulkan
|
||||||
|
|
|
@ -13,22 +13,17 @@
|
||||||
|
|
||||||
namespace Vulkan {
|
namespace Vulkan {
|
||||||
|
|
||||||
static constexpr vk::BufferUsageFlags VertexIndexFlags =
|
|
||||||
vk::BufferUsageFlagBits::eVertexBuffer | vk::BufferUsageFlagBits::eIndexBuffer |
|
|
||||||
vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eUniformBuffer |
|
|
||||||
vk::BufferUsageFlagBits::eStorageBuffer;
|
|
||||||
|
|
||||||
Rasterizer::Rasterizer(const Instance& instance_, Scheduler& scheduler_,
|
Rasterizer::Rasterizer(const Instance& instance_, Scheduler& scheduler_,
|
||||||
VideoCore::TextureCache& texture_cache_, AmdGpu::Liverpool* liverpool_)
|
AmdGpu::Liverpool* liverpool_)
|
||||||
: instance{instance_}, scheduler{scheduler_}, texture_cache{texture_cache_},
|
: instance{instance_}, scheduler{scheduler_}, page_manager{this},
|
||||||
liverpool{liverpool_}, memory{Core::Memory::Instance()},
|
buffer_cache{instance, scheduler, liverpool_, page_manager},
|
||||||
pipeline_cache{instance, scheduler, liverpool},
|
texture_cache{instance, scheduler, buffer_cache, page_manager}, liverpool{liverpool_},
|
||||||
vertex_index_buffer{instance, scheduler, VertexIndexFlags, 2_GB, BufferType::Upload} {
|
memory{Core::Memory::Instance()}, pipeline_cache{instance, scheduler, liverpool} {
|
||||||
if (!Config::nullGpu()) {
|
if (!Config::nullGpu()) {
|
||||||
liverpool->BindRasterizer(this);
|
liverpool->BindRasterizer(this);
|
||||||
}
|
}
|
||||||
|
memory->SetRasterizer(this);
|
||||||
memory->SetInstance(&instance);
|
wfi_event = instance.GetDevice().createEventUnique({});
|
||||||
}
|
}
|
||||||
|
|
||||||
Rasterizer::~Rasterizer() = default;
|
Rasterizer::~Rasterizer() = default;
|
||||||
|
@ -38,29 +33,24 @@ void Rasterizer::Draw(bool is_indexed, u32 index_offset) {
|
||||||
|
|
||||||
const auto cmdbuf = scheduler.CommandBuffer();
|
const auto cmdbuf = scheduler.CommandBuffer();
|
||||||
const auto& regs = liverpool->regs;
|
const auto& regs = liverpool->regs;
|
||||||
const u32 num_indices = SetupIndexBuffer(is_indexed, index_offset);
|
|
||||||
const GraphicsPipeline* pipeline = pipeline_cache.GetGraphicsPipeline();
|
const GraphicsPipeline* pipeline = pipeline_cache.GetGraphicsPipeline();
|
||||||
if (!pipeline) {
|
if (!pipeline) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
pipeline->BindResources(memory, vertex_index_buffer, texture_cache);
|
pipeline->BindResources(regs, buffer_cache, texture_cache);
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
UNREACHABLE();
|
UNREACHABLE();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const auto& vs_info = pipeline->GetStage(Shader::Stage::Vertex);
|
||||||
|
buffer_cache.BindVertexBuffers(vs_info);
|
||||||
|
const u32 num_indices = buffer_cache.BindIndexBuffer(is_indexed, index_offset);
|
||||||
|
|
||||||
BeginRendering();
|
BeginRendering();
|
||||||
UpdateDynamicState(*pipeline);
|
UpdateDynamicState(*pipeline);
|
||||||
|
|
||||||
cmdbuf.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline->Handle());
|
|
||||||
|
|
||||||
const u32 step_rates[] = {
|
|
||||||
regs.vgt_instance_step_rate_0,
|
|
||||||
regs.vgt_instance_step_rate_1,
|
|
||||||
};
|
|
||||||
cmdbuf.pushConstants(pipeline->GetLayout(), vk::ShaderStageFlagBits::eVertex, 0u,
|
|
||||||
sizeof(step_rates), &step_rates);
|
|
||||||
if (is_indexed) {
|
if (is_indexed) {
|
||||||
cmdbuf.drawIndexed(num_indices, regs.num_instances.NumInstances(), 0, 0, 0);
|
cmdbuf.drawIndexed(num_indices, regs.num_instances.NumInstances(), 0, 0, 0);
|
||||||
} else {
|
} else {
|
||||||
|
@ -82,8 +72,7 @@ void Rasterizer::DispatchDirect() {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const auto has_resources =
|
const auto has_resources = pipeline->BindResources(buffer_cache, texture_cache);
|
||||||
pipeline->BindResources(memory, vertex_index_buffer, texture_cache);
|
|
||||||
if (!has_resources) {
|
if (!has_resources) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -131,7 +120,7 @@ void Rasterizer::BeginRendering() {
|
||||||
state.color_images[state.num_color_attachments] = image.image;
|
state.color_images[state.num_color_attachments] = image.image;
|
||||||
state.color_attachments[state.num_color_attachments++] = {
|
state.color_attachments[state.num_color_attachments++] = {
|
||||||
.imageView = *image_view.image_view,
|
.imageView = *image_view.image_view,
|
||||||
.imageLayout = vk::ImageLayout::eGeneral,
|
.imageLayout = vk::ImageLayout::eColorAttachmentOptimal,
|
||||||
.loadOp = is_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad,
|
.loadOp = is_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad,
|
||||||
.storeOp = vk::AttachmentStoreOp::eStore,
|
.storeOp = vk::AttachmentStoreOp::eStore,
|
||||||
.clearValue =
|
.clearValue =
|
||||||
|
@ -168,45 +157,19 @@ void Rasterizer::BeginRendering() {
|
||||||
scheduler.BeginRendering(state);
|
scheduler.BeginRendering(state);
|
||||||
}
|
}
|
||||||
|
|
||||||
u32 Rasterizer::SetupIndexBuffer(bool& is_indexed, u32 index_offset) {
|
void Rasterizer::InvalidateMemory(VAddr addr, u64 size) {
|
||||||
// Emulate QuadList primitive type with CPU made index buffer.
|
buffer_cache.InvalidateMemory(addr, size);
|
||||||
const auto& regs = liverpool->regs;
|
texture_cache.InvalidateMemory(addr, size);
|
||||||
if (liverpool->regs.primitive_type == Liverpool::PrimitiveType::QuadList) {
|
}
|
||||||
// ASSERT_MSG(!is_indexed, "Using QuadList primitive with indexed draw");
|
|
||||||
is_indexed = true;
|
|
||||||
|
|
||||||
// Emit indices.
|
void Rasterizer::MapMemory(VAddr addr, u64 size) {
|
||||||
const u32 index_size = 3 * regs.num_indices;
|
page_manager.OnGpuMap(addr, size);
|
||||||
const auto [data, offset, _] = vertex_index_buffer.Map(index_size);
|
}
|
||||||
LiverpoolToVK::EmitQuadToTriangleListIndices(data, regs.num_indices);
|
|
||||||
vertex_index_buffer.Commit(index_size);
|
|
||||||
|
|
||||||
// Bind index buffer.
|
void Rasterizer::UnmapMemory(VAddr addr, u64 size) {
|
||||||
const auto cmdbuf = scheduler.CommandBuffer();
|
buffer_cache.InvalidateMemory(addr, size);
|
||||||
cmdbuf.bindIndexBuffer(vertex_index_buffer.Handle(), offset, vk::IndexType::eUint16);
|
texture_cache.UnmapMemory(addr, size);
|
||||||
return index_size / sizeof(u16);
|
page_manager.OnGpuUnmap(addr, size);
|
||||||
}
|
|
||||||
if (!is_indexed) {
|
|
||||||
return regs.num_indices;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Figure out index type and size.
|
|
||||||
const bool is_index16 = regs.index_buffer_type.index_type == Liverpool::IndexType::Index16;
|
|
||||||
const vk::IndexType index_type = is_index16 ? vk::IndexType::eUint16 : vk::IndexType::eUint32;
|
|
||||||
const u32 index_size = is_index16 ? sizeof(u16) : sizeof(u32);
|
|
||||||
|
|
||||||
// Upload index data to stream buffer.
|
|
||||||
const auto index_address = regs.index_base_address.Address<const void*>();
|
|
||||||
const u32 index_buffer_size = (index_offset + regs.num_indices) * index_size;
|
|
||||||
const auto [data, offset, _] = vertex_index_buffer.Map(index_buffer_size);
|
|
||||||
std::memcpy(data, index_address, index_buffer_size);
|
|
||||||
vertex_index_buffer.Commit(index_buffer_size);
|
|
||||||
|
|
||||||
// Bind index buffer.
|
|
||||||
const auto cmdbuf = scheduler.CommandBuffer();
|
|
||||||
cmdbuf.bindIndexBuffer(vertex_index_buffer.Handle(), offset + index_offset * index_size,
|
|
||||||
index_type);
|
|
||||||
return regs.num_indices;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void Rasterizer::UpdateDynamicState(const GraphicsPipeline& pipeline) {
|
void Rasterizer::UpdateDynamicState(const GraphicsPipeline& pipeline) {
|
||||||
|
|
|
@ -3,8 +3,10 @@
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include "video_core/buffer_cache/buffer_cache.h"
|
||||||
|
#include "video_core/page_manager.h"
|
||||||
#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
|
#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
|
||||||
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
|
#include "video_core/texture_cache/texture_cache.h"
|
||||||
|
|
||||||
namespace AmdGpu {
|
namespace AmdGpu {
|
||||||
struct Liverpool;
|
struct Liverpool;
|
||||||
|
@ -14,10 +16,6 @@ namespace Core {
|
||||||
class MemoryManager;
|
class MemoryManager;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace VideoCore {
|
|
||||||
class TextureCache;
|
|
||||||
}
|
|
||||||
|
|
||||||
namespace Vulkan {
|
namespace Vulkan {
|
||||||
|
|
||||||
class Scheduler;
|
class Scheduler;
|
||||||
|
@ -26,9 +24,13 @@ class GraphicsPipeline;
|
||||||
class Rasterizer {
|
class Rasterizer {
|
||||||
public:
|
public:
|
||||||
explicit Rasterizer(const Instance& instance, Scheduler& scheduler,
|
explicit Rasterizer(const Instance& instance, Scheduler& scheduler,
|
||||||
VideoCore::TextureCache& texture_cache, AmdGpu::Liverpool* liverpool);
|
AmdGpu::Liverpool* liverpool);
|
||||||
~Rasterizer();
|
~Rasterizer();
|
||||||
|
|
||||||
|
[[nodiscard]] VideoCore::TextureCache& GetTextureCache() noexcept {
|
||||||
|
return texture_cache;
|
||||||
|
}
|
||||||
|
|
||||||
void Draw(bool is_indexed, u32 index_offset = 0);
|
void Draw(bool is_indexed, u32 index_offset = 0);
|
||||||
|
|
||||||
void DispatchDirect();
|
void DispatchDirect();
|
||||||
|
@ -36,12 +38,13 @@ public:
|
||||||
void ScopeMarkerBegin(const std::string& str);
|
void ScopeMarkerBegin(const std::string& str);
|
||||||
void ScopeMarkerEnd();
|
void ScopeMarkerEnd();
|
||||||
|
|
||||||
|
void InvalidateMemory(VAddr addr, u64 size);
|
||||||
|
void MapMemory(VAddr addr, u64 size);
|
||||||
|
void UnmapMemory(VAddr addr, u64 size);
|
||||||
|
|
||||||
u64 Flush();
|
u64 Flush();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
u32 SetupIndexBuffer(bool& is_indexed, u32 index_offset);
|
|
||||||
void MapMemory(VAddr addr, size_t size);
|
|
||||||
|
|
||||||
void BeginRendering();
|
void BeginRendering();
|
||||||
|
|
||||||
void UpdateDynamicState(const GraphicsPipeline& pipeline);
|
void UpdateDynamicState(const GraphicsPipeline& pipeline);
|
||||||
|
@ -51,11 +54,13 @@ private:
|
||||||
private:
|
private:
|
||||||
const Instance& instance;
|
const Instance& instance;
|
||||||
Scheduler& scheduler;
|
Scheduler& scheduler;
|
||||||
VideoCore::TextureCache& texture_cache;
|
VideoCore::PageManager page_manager;
|
||||||
|
VideoCore::BufferCache buffer_cache;
|
||||||
|
VideoCore::TextureCache texture_cache;
|
||||||
AmdGpu::Liverpool* liverpool;
|
AmdGpu::Liverpool* liverpool;
|
||||||
Core::MemoryManager* memory;
|
Core::MemoryManager* memory;
|
||||||
PipelineCache pipeline_cache;
|
PipelineCache pipeline_cache;
|
||||||
StreamBuffer vertex_index_buffer;
|
vk::UniqueEvent wfi_event;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace Vulkan
|
} // namespace Vulkan
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
#include <condition_variable>
|
#include <condition_variable>
|
||||||
#include <boost/container/static_vector.hpp>
|
#include <boost/container/static_vector.hpp>
|
||||||
#include "common/types.h"
|
#include "common/types.h"
|
||||||
|
#include "common/unique_function.h"
|
||||||
#include "video_core/renderer_vulkan/vk_master_semaphore.h"
|
#include "video_core/renderer_vulkan/vk_master_semaphore.h"
|
||||||
#include "video_core/renderer_vulkan/vk_resource_pool.h"
|
#include "video_core/renderer_vulkan/vk_resource_pool.h"
|
||||||
|
|
||||||
|
@ -97,8 +98,8 @@ public:
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Defers an operation until the gpu has reached the current cpu tick.
|
/// Defers an operation until the gpu has reached the current cpu tick.
|
||||||
void DeferOperation(auto&& func) {
|
void DeferOperation(Common::UniqueFunction<void>&& func) {
|
||||||
pending_ops.emplace(func, CurrentTick());
|
pending_ops.emplace(std::move(func), CurrentTick());
|
||||||
}
|
}
|
||||||
|
|
||||||
static std::mutex submit_mutex;
|
static std::mutex submit_mutex;
|
||||||
|
@ -115,7 +116,7 @@ private:
|
||||||
vk::CommandBuffer current_cmdbuf;
|
vk::CommandBuffer current_cmdbuf;
|
||||||
std::condition_variable_any event_cv;
|
std::condition_variable_any event_cv;
|
||||||
struct PendingOp {
|
struct PendingOp {
|
||||||
std::function<void()> callback;
|
Common::UniqueFunction<void> callback;
|
||||||
u64 gpu_tick;
|
u64 gpu_tick;
|
||||||
};
|
};
|
||||||
std::queue<PendingOp> pending_ops;
|
std::queue<PendingOp> pending_ops;
|
||||||
|
|
|
@ -1,241 +0,0 @@
|
||||||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
|
||||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
||||||
|
|
||||||
#include <algorithm>
|
|
||||||
#include "common/alignment.h"
|
|
||||||
#include "common/assert.h"
|
|
||||||
#include "video_core/renderer_vulkan/vk_instance.h"
|
|
||||||
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
|
||||||
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
|
|
||||||
|
|
||||||
namespace Vulkan {
|
|
||||||
|
|
||||||
namespace {
|
|
||||||
|
|
||||||
std::string_view BufferTypeName(BufferType type) {
|
|
||||||
switch (type) {
|
|
||||||
case BufferType::Upload:
|
|
||||||
return "Upload";
|
|
||||||
case BufferType::Download:
|
|
||||||
return "Download";
|
|
||||||
case BufferType::Stream:
|
|
||||||
return "Stream";
|
|
||||||
default:
|
|
||||||
return "Invalid";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
vk::MemoryPropertyFlags MakePropertyFlags(BufferType type) {
|
|
||||||
switch (type) {
|
|
||||||
case BufferType::Upload:
|
|
||||||
return vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent;
|
|
||||||
case BufferType::Download:
|
|
||||||
return vk::MemoryPropertyFlagBits::eHostVisible |
|
|
||||||
vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached;
|
|
||||||
case BufferType::Stream:
|
|
||||||
return vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible |
|
|
||||||
vk::MemoryPropertyFlagBits::eHostCoherent;
|
|
||||||
default:
|
|
||||||
UNREACHABLE_MSG("Unknown buffer type {}", static_cast<u32>(type));
|
|
||||||
return vk::MemoryPropertyFlagBits::eHostVisible;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static std::optional<u32> FindMemoryType(const vk::PhysicalDeviceMemoryProperties& properties,
|
|
||||||
vk::MemoryPropertyFlags wanted) {
|
|
||||||
for (u32 i = 0; i < properties.memoryTypeCount; ++i) {
|
|
||||||
const auto flags = properties.memoryTypes[i].propertyFlags;
|
|
||||||
if ((flags & wanted) == wanted) {
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return std::nullopt;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get the preferred host visible memory type.
|
|
||||||
u32 GetMemoryType(const vk::PhysicalDeviceMemoryProperties& properties, BufferType type) {
|
|
||||||
vk::MemoryPropertyFlags flags = MakePropertyFlags(type);
|
|
||||||
std::optional preferred_type = FindMemoryType(properties, flags);
|
|
||||||
|
|
||||||
constexpr std::array remove_flags = {
|
|
||||||
vk::MemoryPropertyFlagBits::eHostCached,
|
|
||||||
vk::MemoryPropertyFlagBits::eHostCoherent,
|
|
||||||
};
|
|
||||||
|
|
||||||
for (u32 i = 0; i < remove_flags.size() && !preferred_type; i++) {
|
|
||||||
flags &= ~remove_flags[i];
|
|
||||||
preferred_type = FindMemoryType(properties, flags);
|
|
||||||
}
|
|
||||||
ASSERT_MSG(preferred_type, "No suitable memory type found");
|
|
||||||
return preferred_type.value();
|
|
||||||
}
|
|
||||||
|
|
||||||
constexpr u64 WATCHES_INITIAL_RESERVE = 0x4000;
|
|
||||||
constexpr u64 WATCHES_RESERVE_CHUNK = 0x1000;
|
|
||||||
|
|
||||||
} // Anonymous namespace
|
|
||||||
|
|
||||||
StreamBuffer::StreamBuffer(const Instance& instance_, Scheduler& scheduler_,
|
|
||||||
vk::BufferUsageFlags usage_, u64 size, BufferType type_)
|
|
||||||
: instance{instance_}, scheduler{scheduler_}, device{instance.GetDevice()},
|
|
||||||
stream_buffer_size{size}, usage{usage_}, type{type_} {
|
|
||||||
CreateBuffers(size);
|
|
||||||
ReserveWatches(current_watches, WATCHES_INITIAL_RESERVE);
|
|
||||||
ReserveWatches(previous_watches, WATCHES_INITIAL_RESERVE);
|
|
||||||
}
|
|
||||||
|
|
||||||
StreamBuffer::~StreamBuffer() {
|
|
||||||
device.unmapMemory(memory);
|
|
||||||
device.destroyBuffer(buffer);
|
|
||||||
device.freeMemory(memory);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::tuple<u8*, u64, bool> StreamBuffer::Map(u64 size, u64 alignment) {
|
|
||||||
if (!is_coherent && type == BufferType::Stream) {
|
|
||||||
size = Common::AlignUp(size, instance.NonCoherentAtomSize());
|
|
||||||
}
|
|
||||||
|
|
||||||
ASSERT(size <= stream_buffer_size);
|
|
||||||
mapped_size = size;
|
|
||||||
|
|
||||||
if (alignment > 0) {
|
|
||||||
offset = Common::AlignUp(offset, alignment);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool invalidate{false};
|
|
||||||
if (offset + size > stream_buffer_size) {
|
|
||||||
// The buffer would overflow, save the amount of used watches and reset the state.
|
|
||||||
invalidate = true;
|
|
||||||
invalidation_mark = current_watch_cursor;
|
|
||||||
current_watch_cursor = 0;
|
|
||||||
offset = 0;
|
|
||||||
|
|
||||||
// Swap watches and reset waiting cursors.
|
|
||||||
std::swap(previous_watches, current_watches);
|
|
||||||
wait_cursor = 0;
|
|
||||||
wait_bound = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
const u64 mapped_upper_bound = offset + size;
|
|
||||||
WaitPendingOperations(mapped_upper_bound);
|
|
||||||
|
|
||||||
return std::make_tuple(mapped + offset, offset, invalidate);
|
|
||||||
}
|
|
||||||
|
|
||||||
void StreamBuffer::Commit(u64 size) {
|
|
||||||
if (!is_coherent && type == BufferType::Stream) {
|
|
||||||
size = Common::AlignUp(size, instance.NonCoherentAtomSize());
|
|
||||||
}
|
|
||||||
|
|
||||||
ASSERT_MSG(size <= mapped_size, "Reserved size {} is too small compared to {}", mapped_size,
|
|
||||||
size);
|
|
||||||
|
|
||||||
const vk::MappedMemoryRange range = {
|
|
||||||
.memory = memory,
|
|
||||||
.offset = offset,
|
|
||||||
.size = size,
|
|
||||||
};
|
|
||||||
|
|
||||||
if (!is_coherent && type == BufferType::Download) {
|
|
||||||
device.invalidateMappedMemoryRanges(range);
|
|
||||||
} else if (!is_coherent) {
|
|
||||||
device.flushMappedMemoryRanges(range);
|
|
||||||
}
|
|
||||||
|
|
||||||
offset += size;
|
|
||||||
|
|
||||||
if (current_watch_cursor + 1 >= current_watches.size()) {
|
|
||||||
// Ensure that there are enough watches.
|
|
||||||
ReserveWatches(current_watches, WATCHES_RESERVE_CHUNK);
|
|
||||||
}
|
|
||||||
auto& watch = current_watches[current_watch_cursor++];
|
|
||||||
watch.upper_bound = offset;
|
|
||||||
watch.tick = scheduler.CurrentTick();
|
|
||||||
}
|
|
||||||
|
|
||||||
void StreamBuffer::CreateBuffers(u64 prefered_size) {
|
|
||||||
const vk::Device device = instance.GetDevice();
|
|
||||||
const auto memory_properties = instance.GetPhysicalDevice().getMemoryProperties();
|
|
||||||
const u32 preferred_type = GetMemoryType(memory_properties, type);
|
|
||||||
const vk::MemoryType mem_type = memory_properties.memoryTypes[preferred_type];
|
|
||||||
const u32 preferred_heap = mem_type.heapIndex;
|
|
||||||
is_coherent =
|
|
||||||
static_cast<bool>(mem_type.propertyFlags & vk::MemoryPropertyFlagBits::eHostCoherent);
|
|
||||||
|
|
||||||
// Substract from the preferred heap size some bytes to avoid getting out of memory.
|
|
||||||
const vk::DeviceSize heap_size = memory_properties.memoryHeaps[preferred_heap].size;
|
|
||||||
// As per DXVK's example, using `heap_size / 2`
|
|
||||||
const vk::DeviceSize allocable_size = heap_size / 2;
|
|
||||||
buffer = device.createBuffer({
|
|
||||||
.size = std::min(prefered_size, allocable_size),
|
|
||||||
.usage = usage,
|
|
||||||
});
|
|
||||||
|
|
||||||
const auto requirements_chain =
|
|
||||||
device
|
|
||||||
.getBufferMemoryRequirements2<vk::MemoryRequirements2, vk::MemoryDedicatedRequirements>(
|
|
||||||
{.buffer = buffer});
|
|
||||||
|
|
||||||
const auto& requirements = requirements_chain.get<vk::MemoryRequirements2>();
|
|
||||||
const auto& dedicated_requirements = requirements_chain.get<vk::MemoryDedicatedRequirements>();
|
|
||||||
|
|
||||||
stream_buffer_size = static_cast<u64>(requirements.memoryRequirements.size);
|
|
||||||
|
|
||||||
LOG_INFO(Render_Vulkan, "Creating {} buffer with size {} KiB with flags {}",
|
|
||||||
BufferTypeName(type), stream_buffer_size / 1024,
|
|
||||||
vk::to_string(mem_type.propertyFlags));
|
|
||||||
|
|
||||||
if (dedicated_requirements.prefersDedicatedAllocation) {
|
|
||||||
vk::StructureChain<vk::MemoryAllocateInfo, vk::MemoryDedicatedAllocateInfo> alloc_chain =
|
|
||||||
{};
|
|
||||||
|
|
||||||
auto& alloc_info = alloc_chain.get<vk::MemoryAllocateInfo>();
|
|
||||||
alloc_info.allocationSize = requirements.memoryRequirements.size;
|
|
||||||
alloc_info.memoryTypeIndex = preferred_type;
|
|
||||||
|
|
||||||
auto& dedicated_alloc_info = alloc_chain.get<vk::MemoryDedicatedAllocateInfo>();
|
|
||||||
dedicated_alloc_info.buffer = buffer;
|
|
||||||
|
|
||||||
memory = device.allocateMemory(alloc_chain.get());
|
|
||||||
} else {
|
|
||||||
memory = device.allocateMemory({
|
|
||||||
.allocationSize = requirements.memoryRequirements.size,
|
|
||||||
.memoryTypeIndex = preferred_type,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
device.bindBufferMemory(buffer, memory, 0);
|
|
||||||
mapped = reinterpret_cast<u8*>(device.mapMemory(memory, 0, VK_WHOLE_SIZE));
|
|
||||||
|
|
||||||
if (instance.HasDebuggingToolAttached()) {
|
|
||||||
SetObjectName(device, buffer, "StreamBuffer({}): {} KiB {}", BufferTypeName(type),
|
|
||||||
stream_buffer_size / 1024, vk::to_string(mem_type.propertyFlags));
|
|
||||||
SetObjectName(device, memory, "StreamBufferMemory({}): {} Kib {}", BufferTypeName(type),
|
|
||||||
stream_buffer_size / 1024, vk::to_string(mem_type.propertyFlags));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void StreamBuffer::ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size) {
|
|
||||||
watches.resize(watches.size() + grow_size);
|
|
||||||
}
|
|
||||||
|
|
||||||
void StreamBuffer::WaitPendingOperations(u64 requested_upper_bound) {
|
|
||||||
if (!invalidation_mark) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
while (requested_upper_bound > wait_bound && wait_cursor < *invalidation_mark) {
|
|
||||||
auto& watch = previous_watches[wait_cursor];
|
|
||||||
wait_bound = watch.upper_bound;
|
|
||||||
scheduler.Wait(watch.tick);
|
|
||||||
++wait_cursor;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
u64 StreamBuffer::Copy(VAddr src, size_t size, size_t alignment /*= 0*/) {
|
|
||||||
const auto [data, offset, _] = Map(size, alignment);
|
|
||||||
std::memcpy(data, reinterpret_cast<const void*>(src), size);
|
|
||||||
Commit(size);
|
|
||||||
return offset;
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace Vulkan
|
|
|
@ -1,89 +0,0 @@
|
||||||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
|
||||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <optional>
|
|
||||||
#include <span>
|
|
||||||
#include <tuple>
|
|
||||||
#include <vector>
|
|
||||||
#include "common/types.h"
|
|
||||||
#include "video_core/renderer_vulkan/vk_common.h"
|
|
||||||
|
|
||||||
namespace Vulkan {
|
|
||||||
|
|
||||||
enum class BufferType : u32 {
|
|
||||||
Upload = 0,
|
|
||||||
Download = 1,
|
|
||||||
Stream = 2,
|
|
||||||
};
|
|
||||||
|
|
||||||
class Instance;
|
|
||||||
class Scheduler;
|
|
||||||
|
|
||||||
class StreamBuffer final {
|
|
||||||
static constexpr std::size_t MAX_BUFFER_VIEWS = 3;
|
|
||||||
|
|
||||||
public:
|
|
||||||
explicit StreamBuffer(const Instance& instance, Scheduler& scheduler,
|
|
||||||
vk::BufferUsageFlags usage, u64 size,
|
|
||||||
BufferType type = BufferType::Stream);
|
|
||||||
~StreamBuffer();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reserves a region of memory from the stream buffer.
|
|
||||||
* @param size Size to reserve.
|
|
||||||
* @returns A pair of a raw memory pointer (with offset added), and the buffer offset
|
|
||||||
*/
|
|
||||||
std::tuple<u8*, u64, bool> Map(u64 size, u64 alignment = 0);
|
|
||||||
|
|
||||||
/// Ensures that "size" bytes of memory are available to the GPU, potentially recording a copy.
|
|
||||||
void Commit(u64 size);
|
|
||||||
|
|
||||||
/// Maps and commits a memory region with user provided data
|
|
||||||
u64 Copy(VAddr src, size_t size, size_t alignment = 0);
|
|
||||||
|
|
||||||
vk::Buffer Handle() const noexcept {
|
|
||||||
return buffer;
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
struct Watch {
|
|
||||||
u64 tick{};
|
|
||||||
u64 upper_bound{};
|
|
||||||
};
|
|
||||||
|
|
||||||
/// Creates Vulkan buffer handles committing the required the required memory.
|
|
||||||
void CreateBuffers(u64 prefered_size);
|
|
||||||
|
|
||||||
/// Increases the amount of watches available.
|
|
||||||
void ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size);
|
|
||||||
|
|
||||||
void WaitPendingOperations(u64 requested_upper_bound);
|
|
||||||
|
|
||||||
private:
|
|
||||||
const Instance& instance; ///< Vulkan instance.
|
|
||||||
Scheduler& scheduler; ///< Command scheduler.
|
|
||||||
|
|
||||||
vk::Device device;
|
|
||||||
vk::Buffer buffer; ///< Mapped buffer.
|
|
||||||
vk::DeviceMemory memory; ///< Memory allocation.
|
|
||||||
u8* mapped{}; ///< Pointer to the mapped memory
|
|
||||||
u64 stream_buffer_size{}; ///< Stream buffer size.
|
|
||||||
vk::BufferUsageFlags usage{};
|
|
||||||
BufferType type;
|
|
||||||
|
|
||||||
u64 offset{}; ///< Buffer iterator.
|
|
||||||
u64 mapped_size{}; ///< Size reserved for the current copy.
|
|
||||||
bool is_coherent{}; ///< True if the buffer is coherent
|
|
||||||
|
|
||||||
std::vector<Watch> current_watches; ///< Watches recorded in the current iteration.
|
|
||||||
std::size_t current_watch_cursor{}; ///< Count of watches, reset on invalidation.
|
|
||||||
std::optional<std::size_t> invalidation_mark; ///< Number of watches used in the previous cycle.
|
|
||||||
|
|
||||||
std::vector<Watch> previous_watches; ///< Watches used in the previous iteration.
|
|
||||||
std::size_t wait_cursor{}; ///< Last watch being waited for completion.
|
|
||||||
u64 wait_bound{}; ///< Highest offset being watched for completion.
|
|
||||||
};
|
|
||||||
|
|
||||||
} // namespace Vulkan
|
|
|
@ -260,7 +260,7 @@ ImageInfo::ImageInfo(const AmdGpu::Image& image) noexcept {
|
||||||
case AmdGpu::TilingMode::Display_MacroTiled:
|
case AmdGpu::TilingMode::Display_MacroTiled:
|
||||||
case AmdGpu::TilingMode::Texture_MacroTiled:
|
case AmdGpu::TilingMode::Texture_MacroTiled:
|
||||||
case AmdGpu::TilingMode::Depth_MacroTiled: {
|
case AmdGpu::TilingMode::Depth_MacroTiled: {
|
||||||
ASSERT(!props.is_cube && !props.is_block);
|
// ASSERT(!props.is_cube && !props.is_block);
|
||||||
ASSERT(num_samples == 1);
|
ASSERT(num_samples == 1);
|
||||||
std::tie(mip_info.pitch, mip_info.size) =
|
std::tie(mip_info.pitch, mip_info.size) =
|
||||||
ImageSizeMacroTiled(mip_w, mip_h, bpp, num_samples, image.tiling_index);
|
ImageSizeMacroTiled(mip_w, mip_h, bpp, num_samples, image.tiling_index);
|
||||||
|
|
|
@ -61,23 +61,24 @@ vk::Format TrySwizzleFormat(vk::Format format, u32 dst_sel) {
|
||||||
return format;
|
return format;
|
||||||
}
|
}
|
||||||
|
|
||||||
ImageViewInfo::ImageViewInfo(const AmdGpu::Image& image, bool is_storage) noexcept
|
ImageViewInfo::ImageViewInfo(const AmdGpu::Image& image, bool is_storage_) noexcept
|
||||||
: is_storage{is_storage} {
|
: is_storage{is_storage_} {
|
||||||
type = ConvertImageViewType(image.GetType());
|
type = ConvertImageViewType(image.GetType());
|
||||||
format = Vulkan::LiverpoolToVK::SurfaceFormat(image.GetDataFmt(), image.GetNumberFmt());
|
format = Vulkan::LiverpoolToVK::SurfaceFormat(image.GetDataFmt(), image.GetNumberFmt());
|
||||||
range.base.level = image.base_level;
|
range.base.level = image.base_level;
|
||||||
range.base.layer = image.base_array;
|
range.base.layer = image.base_array;
|
||||||
range.extent.levels = image.last_level + 1;
|
range.extent.levels = image.last_level + 1;
|
||||||
range.extent.layers = image.last_array + 1;
|
range.extent.layers = image.last_array + 1;
|
||||||
|
if (!is_storage) {
|
||||||
mapping.r = ConvertComponentSwizzle(image.dst_sel_x);
|
mapping.r = ConvertComponentSwizzle(image.dst_sel_x);
|
||||||
mapping.g = ConvertComponentSwizzle(image.dst_sel_y);
|
mapping.g = ConvertComponentSwizzle(image.dst_sel_y);
|
||||||
mapping.b = ConvertComponentSwizzle(image.dst_sel_z);
|
mapping.b = ConvertComponentSwizzle(image.dst_sel_z);
|
||||||
mapping.a = ConvertComponentSwizzle(image.dst_sel_w);
|
mapping.a = ConvertComponentSwizzle(image.dst_sel_w);
|
||||||
|
}
|
||||||
// Check for unfortunate case of storage images being swizzled
|
// Check for unfortunate case of storage images being swizzled
|
||||||
const u32 num_comps = AmdGpu::NumComponents(image.GetDataFmt());
|
const u32 num_comps = AmdGpu::NumComponents(image.GetDataFmt());
|
||||||
const u32 dst_sel = image.DstSelect();
|
const u32 dst_sel = image.DstSelect();
|
||||||
if (is_storage && !IsIdentityMapping(dst_sel, num_comps)) {
|
if (is_storage && !IsIdentityMapping(dst_sel, num_comps)) {
|
||||||
mapping = vk::ComponentMapping{};
|
|
||||||
if (auto new_format = TrySwizzleFormat(format, dst_sel); new_format != format) {
|
if (auto new_format = TrySwizzleFormat(format, dst_sel); new_format != format) {
|
||||||
format = new_format;
|
format = new_format;
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -3,103 +3,22 @@
|
||||||
|
|
||||||
#include <xxhash.h>
|
#include <xxhash.h>
|
||||||
#include "common/assert.h"
|
#include "common/assert.h"
|
||||||
#include "common/config.h"
|
#include "video_core/page_manager.h"
|
||||||
#include "core/virtual_memory.h"
|
|
||||||
#include "video_core/renderer_vulkan/vk_instance.h"
|
#include "video_core/renderer_vulkan/vk_instance.h"
|
||||||
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
||||||
#include "video_core/texture_cache/texture_cache.h"
|
#include "video_core/texture_cache/texture_cache.h"
|
||||||
#include "video_core/texture_cache/tile_manager.h"
|
#include "video_core/texture_cache/tile_manager.h"
|
||||||
|
|
||||||
#ifndef _WIN64
|
|
||||||
#include <signal.h>
|
|
||||||
#include <sys/mman.h>
|
|
||||||
|
|
||||||
#define PAGE_NOACCESS PROT_NONE
|
|
||||||
#define PAGE_READWRITE (PROT_READ | PROT_WRITE)
|
|
||||||
#define PAGE_READONLY PROT_READ
|
|
||||||
#else
|
|
||||||
#include <windows.h>
|
|
||||||
|
|
||||||
void mprotect(void* addr, size_t len, int prot) {
|
|
||||||
DWORD old_prot{};
|
|
||||||
BOOL result = VirtualProtect(addr, len, prot, &old_prot);
|
|
||||||
ASSERT_MSG(result != 0, "Region protection failed");
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
namespace VideoCore {
|
namespace VideoCore {
|
||||||
|
|
||||||
static TextureCache* g_texture_cache = nullptr;
|
|
||||||
|
|
||||||
#ifndef _WIN64
|
|
||||||
void GuestFaultSignalHandler(int sig, siginfo_t* info, void* raw_context) {
|
|
||||||
ucontext_t* ctx = reinterpret_cast<ucontext_t*>(raw_context);
|
|
||||||
const VAddr address = reinterpret_cast<VAddr>(info->si_addr);
|
|
||||||
|
|
||||||
#ifdef __APPLE__
|
|
||||||
const u32 err = ctx->uc_mcontext->__es.__err;
|
|
||||||
#else
|
|
||||||
const greg_t err = ctx->uc_mcontext.gregs[REG_ERR];
|
|
||||||
#endif
|
|
||||||
|
|
||||||
if (err & 0x2) {
|
|
||||||
g_texture_cache->OnCpuWrite(address);
|
|
||||||
} else {
|
|
||||||
// Read not supported!
|
|
||||||
UNREACHABLE();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
LONG WINAPI GuestFaultSignalHandler(EXCEPTION_POINTERS* pExp) noexcept {
|
|
||||||
const u32 ec = pExp->ExceptionRecord->ExceptionCode;
|
|
||||||
if (ec == EXCEPTION_ACCESS_VIOLATION) {
|
|
||||||
const auto info = pExp->ExceptionRecord->ExceptionInformation;
|
|
||||||
if (info[0] == 1) { // Write violation
|
|
||||||
g_texture_cache->OnCpuWrite(info[1]);
|
|
||||||
return EXCEPTION_CONTINUE_EXECUTION;
|
|
||||||
} /* else {
|
|
||||||
UNREACHABLE();
|
|
||||||
}*/
|
|
||||||
}
|
|
||||||
return EXCEPTION_CONTINUE_SEARCH; // pass further
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static constexpr u64 StreamBufferSize = 512_MB;
|
static constexpr u64 StreamBufferSize = 512_MB;
|
||||||
static constexpr u64 PageShift = 12;
|
static constexpr u64 PageShift = 12;
|
||||||
|
|
||||||
TextureCache::TextureCache(const Vulkan::Instance& instance_, Vulkan::Scheduler& scheduler_)
|
TextureCache::TextureCache(const Vulkan::Instance& instance_, Vulkan::Scheduler& scheduler_,
|
||||||
: instance{instance_}, scheduler{scheduler_},
|
BufferCache& buffer_cache_, PageManager& tracker_)
|
||||||
staging{instance, scheduler, vk::BufferUsageFlagBits::eTransferSrc, StreamBufferSize,
|
: instance{instance_}, scheduler{scheduler_}, buffer_cache{buffer_cache_}, tracker{tracker_},
|
||||||
Vulkan::BufferType::Upload},
|
staging{instance, scheduler, MemoryUsage::Upload, StreamBufferSize},
|
||||||
tile_manager{instance, scheduler} {
|
tile_manager{instance, scheduler} {
|
||||||
|
|
||||||
#ifndef _WIN64
|
|
||||||
#ifdef __APPLE__
|
|
||||||
// Read-only memory write results in SIGBUS on Apple.
|
|
||||||
static constexpr int SignalType = SIGBUS;
|
|
||||||
#else
|
|
||||||
static constexpr int SignalType = SIGSEGV;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
sigset_t signal_mask;
|
|
||||||
sigemptyset(&signal_mask);
|
|
||||||
sigaddset(&signal_mask, SignalType);
|
|
||||||
|
|
||||||
using HandlerType = decltype(sigaction::sa_sigaction);
|
|
||||||
|
|
||||||
struct sigaction guest_access_fault {};
|
|
||||||
guest_access_fault.sa_flags = SA_SIGINFO | SA_ONSTACK;
|
|
||||||
guest_access_fault.sa_sigaction = &GuestFaultSignalHandler;
|
|
||||||
guest_access_fault.sa_mask = signal_mask;
|
|
||||||
sigaction(SignalType, &guest_access_fault, nullptr);
|
|
||||||
#else
|
|
||||||
veh_handle = AddVectoredExceptionHandler(0, GuestFaultSignalHandler);
|
|
||||||
ASSERT_MSG(veh_handle, "Failed to register an exception handler");
|
|
||||||
#endif
|
|
||||||
g_texture_cache = this;
|
|
||||||
|
|
||||||
ImageInfo info;
|
ImageInfo info;
|
||||||
info.pixel_format = vk::Format::eR8G8B8A8Unorm;
|
info.pixel_format = vk::Format::eR8G8B8A8Unorm;
|
||||||
info.type = vk::ImageType::e2D;
|
info.type = vk::ImageType::e2D;
|
||||||
|
@ -110,15 +29,11 @@ TextureCache::TextureCache(const Vulkan::Instance& instance_, Vulkan::Scheduler&
|
||||||
void(slot_image_views.insert(instance, view_info, slot_images[null_id], null_id));
|
void(slot_image_views.insert(instance, view_info, slot_images[null_id], null_id));
|
||||||
}
|
}
|
||||||
|
|
||||||
TextureCache::~TextureCache() {
|
TextureCache::~TextureCache() = default;
|
||||||
#if _WIN64
|
|
||||||
RemoveVectoredExceptionHandler(veh_handle);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
void TextureCache::OnCpuWrite(VAddr address) {
|
void TextureCache::InvalidateMemory(VAddr address, size_t size) {
|
||||||
std::unique_lock lock{m_page_table};
|
std::unique_lock lock{mutex};
|
||||||
ForEachImageInRegion(address, 1 << PageShift, [&](ImageId image_id, Image& image) {
|
ForEachImageInRegion(address, size, [&](ImageId image_id, Image& image) {
|
||||||
// Ensure image is reuploaded when accessed again.
|
// Ensure image is reuploaded when accessed again.
|
||||||
image.flags |= ImageFlagBits::CpuModified;
|
image.flags |= ImageFlagBits::CpuModified;
|
||||||
// Untrack image, so the range is unprotected and the guest can write freely.
|
// Untrack image, so the range is unprotected and the guest can write freely.
|
||||||
|
@ -126,8 +41,28 @@ void TextureCache::OnCpuWrite(VAddr address) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void TextureCache::UnmapMemory(VAddr cpu_addr, size_t size) {
|
||||||
|
std::scoped_lock lk{mutex};
|
||||||
|
|
||||||
|
boost::container::small_vector<ImageId, 16> deleted_images;
|
||||||
|
ForEachImageInRegion(cpu_addr, size, [&](ImageId id, Image&) { deleted_images.push_back(id); });
|
||||||
|
for (const ImageId id : deleted_images) {
|
||||||
|
Image& image = slot_images[id];
|
||||||
|
if (True(image.flags & ImageFlagBits::Tracked)) {
|
||||||
|
UntrackImage(image, id);
|
||||||
|
}
|
||||||
|
// TODO: Download image data back to host.
|
||||||
|
UnregisterImage(id);
|
||||||
|
DeleteImage(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ImageId TextureCache::FindImage(const ImageInfo& info, bool refresh_on_create) {
|
ImageId TextureCache::FindImage(const ImageInfo& info, bool refresh_on_create) {
|
||||||
std::unique_lock lock{m_page_table};
|
if (info.guest_address == 0) [[unlikely]] {
|
||||||
|
return NULL_IMAGE_VIEW_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::unique_lock lock{mutex};
|
||||||
boost::container::small_vector<ImageId, 2> image_ids;
|
boost::container::small_vector<ImageId, 2> image_ids;
|
||||||
ForEachImageInRegion(
|
ForEachImageInRegion(
|
||||||
info.guest_address, info.guest_size_bytes, [&](ImageId image_id, Image& image) {
|
info.guest_address, info.guest_size_bytes, [&](ImageId image_id, Image& image) {
|
||||||
|
@ -183,10 +118,6 @@ ImageView& TextureCache::RegisterImageView(ImageId image_id, const ImageViewInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
ImageView& TextureCache::FindTexture(const ImageInfo& info, const ImageViewInfo& view_info) {
|
ImageView& TextureCache::FindTexture(const ImageInfo& info, const ImageViewInfo& view_info) {
|
||||||
if (info.guest_address == 0) [[unlikely]] {
|
|
||||||
return slot_image_views[NULL_IMAGE_VIEW_ID];
|
|
||||||
}
|
|
||||||
|
|
||||||
const ImageId image_id = FindImage(info);
|
const ImageId image_id = FindImage(info);
|
||||||
Image& image = slot_images[image_id];
|
Image& image = slot_images[image_id];
|
||||||
auto& usage = image.info.usage;
|
auto& usage = image.info.usage;
|
||||||
|
@ -310,10 +241,7 @@ void TextureCache::RefreshImage(Image& image) {
|
||||||
buffer = *upload_buffer;
|
buffer = *upload_buffer;
|
||||||
} else {
|
} else {
|
||||||
// Upload data to the staging buffer.
|
// Upload data to the staging buffer.
|
||||||
const auto [data, offset_, _] = staging.Map(image.info.guest_size_bytes, 16);
|
offset = staging.Copy(image.info.guest_address, image.info.guest_size_bytes, 16);
|
||||||
std::memcpy(data, (void*)image.info.guest_address, image.info.guest_size_bytes);
|
|
||||||
staging.Commit(image.info.guest_size_bytes);
|
|
||||||
offset = offset_;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& num_layers = image.info.resources.layers;
|
const auto& num_layers = image.info.resources.layers;
|
||||||
|
@ -344,9 +272,6 @@ void TextureCache::RefreshImage(Image& image) {
|
||||||
}
|
}
|
||||||
|
|
||||||
cmdbuf.copyBufferToImage(buffer, image.image, vk::ImageLayout::eTransferDstOptimal, image_copy);
|
cmdbuf.copyBufferToImage(buffer, image.image, vk::ImageLayout::eTransferDstOptimal, image_copy);
|
||||||
|
|
||||||
image.Transit(vk::ImageLayout::eGeneral,
|
|
||||||
vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eMemoryRead);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
vk::Sampler TextureCache::GetSampler(const AmdGpu::Sampler& sampler) {
|
vk::Sampler TextureCache::GetSampler(const AmdGpu::Sampler& sampler) {
|
||||||
|
@ -362,8 +287,6 @@ void TextureCache::RegisterImage(ImageId image_id) {
|
||||||
image.flags |= ImageFlagBits::Registered;
|
image.flags |= ImageFlagBits::Registered;
|
||||||
ForEachPage(image.cpu_addr, image.info.guest_size_bytes,
|
ForEachPage(image.cpu_addr, image.info.guest_size_bytes,
|
||||||
[this, image_id](u64 page) { page_table[page].push_back(image_id); });
|
[this, image_id](u64 page) { page_table[page].push_back(image_id); });
|
||||||
|
|
||||||
image.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits::eNone);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void TextureCache::UnregisterImage(ImageId image_id) {
|
void TextureCache::UnregisterImage(ImageId image_id) {
|
||||||
|
@ -373,11 +296,11 @@ void TextureCache::UnregisterImage(ImageId image_id) {
|
||||||
image.flags &= ~ImageFlagBits::Registered;
|
image.flags &= ~ImageFlagBits::Registered;
|
||||||
ForEachPage(image.cpu_addr, image.info.guest_size_bytes, [this, image_id](u64 page) {
|
ForEachPage(image.cpu_addr, image.info.guest_size_bytes, [this, image_id](u64 page) {
|
||||||
const auto page_it = page_table.find(page);
|
const auto page_it = page_table.find(page);
|
||||||
if (page_it == page_table.end()) {
|
if (page_it == nullptr) {
|
||||||
ASSERT_MSG(false, "Unregistering unregistered page=0x{:x}", page << PageShift);
|
UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PageShift);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
auto& image_ids = page_it.value();
|
auto& image_ids = *page_it;
|
||||||
const auto vector_it = std::ranges::find(image_ids, image_id);
|
const auto vector_it = std::ranges::find(image_ids, image_id);
|
||||||
if (vector_it == image_ids.end()) {
|
if (vector_it == image_ids.end()) {
|
||||||
ASSERT_MSG(false, "Unregistering unregistered image in page=0x{:x}", page << PageShift);
|
ASSERT_MSG(false, "Unregistering unregistered image in page=0x{:x}", page << PageShift);
|
||||||
|
@ -393,7 +316,7 @@ void TextureCache::TrackImage(Image& image, ImageId image_id) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
image.flags |= ImageFlagBits::Tracked;
|
image.flags |= ImageFlagBits::Tracked;
|
||||||
UpdatePagesCachedCount(image.cpu_addr, image.info.guest_size_bytes, 1);
|
tracker.UpdatePagesCachedCount(image.cpu_addr, image.info.guest_size_bytes, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
void TextureCache::UntrackImage(Image& image, ImageId image_id) {
|
void TextureCache::UntrackImage(Image& image, ImageId image_id) {
|
||||||
|
@ -401,40 +324,34 @@ void TextureCache::UntrackImage(Image& image, ImageId image_id) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
image.flags &= ~ImageFlagBits::Tracked;
|
image.flags &= ~ImageFlagBits::Tracked;
|
||||||
UpdatePagesCachedCount(image.cpu_addr, image.info.guest_size_bytes, -1);
|
tracker.UpdatePagesCachedCount(image.cpu_addr, image.info.guest_size_bytes, -1);
|
||||||
}
|
}
|
||||||
|
|
||||||
void TextureCache::UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta) {
|
void TextureCache::DeleteImage(ImageId image_id) {
|
||||||
std::scoped_lock lk{mutex};
|
Image& image = slot_images[image_id];
|
||||||
const u64 num_pages = ((addr + size - 1) >> PageShift) - (addr >> PageShift) + 1;
|
ASSERT_MSG(False(image.flags & ImageFlagBits::Tracked), "Image was not untracked");
|
||||||
const u64 page_start = addr >> PageShift;
|
ASSERT_MSG(False(image.flags & ImageFlagBits::Registered), "Image was not unregistered");
|
||||||
const u64 page_end = page_start + num_pages;
|
|
||||||
|
|
||||||
const auto pages_interval =
|
// Remove any registered meta areas.
|
||||||
decltype(cached_pages)::interval_type::right_open(page_start, page_end);
|
const auto& meta_info = image.info.meta_info;
|
||||||
if (delta > 0) {
|
if (meta_info.cmask_addr) {
|
||||||
cached_pages.add({pages_interval, delta});
|
surface_metas.erase(meta_info.cmask_addr);
|
||||||
|
}
|
||||||
|
if (meta_info.fmask_addr) {
|
||||||
|
surface_metas.erase(meta_info.fmask_addr);
|
||||||
|
}
|
||||||
|
if (meta_info.htile_addr) {
|
||||||
|
surface_metas.erase(meta_info.htile_addr);
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& range = cached_pages.equal_range(pages_interval);
|
// Reclaim image and any image views it references.
|
||||||
for (const auto& [range, count] : boost::make_iterator_range(range)) {
|
scheduler.DeferOperation([this, image_id] {
|
||||||
const auto interval = range & pages_interval;
|
Image& image = slot_images[image_id];
|
||||||
const VAddr interval_start_addr = boost::icl::first(interval) << PageShift;
|
for (const ImageViewId image_view_id : image.image_view_ids) {
|
||||||
const VAddr interval_end_addr = boost::icl::last_next(interval) << PageShift;
|
slot_image_views.erase(image_view_id);
|
||||||
const u32 interval_size = interval_end_addr - interval_start_addr;
|
|
||||||
void* addr = reinterpret_cast<void*>(interval_start_addr);
|
|
||||||
if (delta > 0 && count == delta) {
|
|
||||||
mprotect(addr, interval_size, PAGE_READONLY);
|
|
||||||
} else if (delta < 0 && count == -delta) {
|
|
||||||
mprotect(addr, interval_size, PAGE_READWRITE);
|
|
||||||
} else {
|
|
||||||
ASSERT(count >= 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (delta < 0) {
|
|
||||||
cached_pages.add({pages_interval, delta});
|
|
||||||
}
|
}
|
||||||
|
slot_images.erase(image_id);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace VideoCore
|
} // namespace VideoCore
|
||||||
|
|
|
@ -4,12 +4,11 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <boost/container/small_vector.hpp>
|
#include <boost/container/small_vector.hpp>
|
||||||
#include <boost/icl/interval_map.hpp>
|
|
||||||
#include <tsl/robin_map.h>
|
#include <tsl/robin_map.h>
|
||||||
|
|
||||||
#include "common/slot_vector.h"
|
#include "common/slot_vector.h"
|
||||||
#include "video_core/amdgpu/resource.h"
|
#include "video_core/amdgpu/resource.h"
|
||||||
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
|
#include "video_core/multi_level_page_table.h"
|
||||||
#include "video_core/texture_cache/image.h"
|
#include "video_core/texture_cache/image.h"
|
||||||
#include "video_core/texture_cache/image_view.h"
|
#include "video_core/texture_cache/image_view.h"
|
||||||
#include "video_core/texture_cache/sampler.h"
|
#include "video_core/texture_cache/sampler.h"
|
||||||
|
@ -21,31 +20,28 @@ struct BufferAttributeGroup;
|
||||||
|
|
||||||
namespace VideoCore {
|
namespace VideoCore {
|
||||||
|
|
||||||
|
class BufferCache;
|
||||||
|
class PageManager;
|
||||||
|
|
||||||
class TextureCache {
|
class TextureCache {
|
||||||
// This is the page shift for adding images into the hash map. It isn't related to
|
struct Traits {
|
||||||
// the page size of the guest or the host and is chosen for convenience. A number too
|
using Entry = boost::container::small_vector<ImageId, 16>;
|
||||||
// small will increase the number of hash map lookups per image, while too large will
|
static constexpr size_t AddressSpaceBits = 39;
|
||||||
// increase the number of images per page.
|
static constexpr size_t FirstLevelBits = 9;
|
||||||
static constexpr u64 PageBits = 20;
|
static constexpr size_t PageBits = 22;
|
||||||
static constexpr u64 PageMask = (1ULL << PageBits) - 1;
|
|
||||||
|
|
||||||
struct MetaDataInfo {
|
|
||||||
enum class Type {
|
|
||||||
CMask,
|
|
||||||
FMask,
|
|
||||||
HTile,
|
|
||||||
};
|
|
||||||
|
|
||||||
Type type;
|
|
||||||
bool is_cleared;
|
|
||||||
};
|
};
|
||||||
|
using PageTable = MultiLevelPageTable<Traits>;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit TextureCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler);
|
explicit TextureCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||||
|
BufferCache& buffer_cache, PageManager& tracker);
|
||||||
~TextureCache();
|
~TextureCache();
|
||||||
|
|
||||||
/// Invalidates any image in the logical page range.
|
/// Invalidates any image in the logical page range.
|
||||||
void OnCpuWrite(VAddr address);
|
void InvalidateMemory(VAddr address, size_t size);
|
||||||
|
|
||||||
|
/// Evicts any images that overlap the unmapped range.
|
||||||
|
void UnmapMemory(VAddr cpu_addr, size_t size);
|
||||||
|
|
||||||
/// Retrieves the image handle of the image with the provided attributes.
|
/// Retrieves the image handle of the image with the provided attributes.
|
||||||
[[nodiscard]] ImageId FindImage(const ImageInfo& info, bool refresh_on_create = true);
|
[[nodiscard]] ImageId FindImage(const ImageInfo& info, bool refresh_on_create = true);
|
||||||
|
@ -101,8 +97,8 @@ private:
|
||||||
template <typename Func>
|
template <typename Func>
|
||||||
static void ForEachPage(PAddr addr, size_t size, Func&& func) {
|
static void ForEachPage(PAddr addr, size_t size, Func&& func) {
|
||||||
static constexpr bool RETURNS_BOOL = std::is_same_v<std::invoke_result<Func, u64>, bool>;
|
static constexpr bool RETURNS_BOOL = std::is_same_v<std::invoke_result<Func, u64>, bool>;
|
||||||
const u64 page_end = (addr + size - 1) >> PageBits;
|
const u64 page_end = (addr + size - 1) >> Traits::PageBits;
|
||||||
for (u64 page = addr >> PageBits; page <= page_end; ++page) {
|
for (u64 page = addr >> Traits::PageBits; page <= page_end; ++page) {
|
||||||
if constexpr (RETURNS_BOOL) {
|
if constexpr (RETURNS_BOOL) {
|
||||||
if (func(page)) {
|
if (func(page)) {
|
||||||
break;
|
break;
|
||||||
|
@ -120,14 +116,14 @@ private:
|
||||||
boost::container::small_vector<ImageId, 32> images;
|
boost::container::small_vector<ImageId, 32> images;
|
||||||
ForEachPage(cpu_addr, size, [this, &images, cpu_addr, size, func](u64 page) {
|
ForEachPage(cpu_addr, size, [this, &images, cpu_addr, size, func](u64 page) {
|
||||||
const auto it = page_table.find(page);
|
const auto it = page_table.find(page);
|
||||||
if (it == page_table.end()) {
|
if (it == nullptr) {
|
||||||
if constexpr (BOOL_BREAK) {
|
if constexpr (BOOL_BREAK) {
|
||||||
return false;
|
return false;
|
||||||
} else {
|
} else {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (const ImageId image_id : it->second) {
|
for (const ImageId image_id : *it) {
|
||||||
Image& image = slot_images[image_id];
|
Image& image = slot_images[image_id];
|
||||||
if (image.flags & ImageFlagBits::Picked) {
|
if (image.flags & ImageFlagBits::Picked) {
|
||||||
continue;
|
continue;
|
||||||
|
@ -166,25 +162,32 @@ private:
|
||||||
/// Stop tracking CPU reads and writes for image
|
/// Stop tracking CPU reads and writes for image
|
||||||
void UntrackImage(Image& image, ImageId image_id);
|
void UntrackImage(Image& image, ImageId image_id);
|
||||||
|
|
||||||
/// Increase/decrease the number of surface in pages touching the specified region
|
/// Removes the image and any views/surface metas that reference it.
|
||||||
void UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta);
|
void DeleteImage(ImageId image_id);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
const Vulkan::Instance& instance;
|
const Vulkan::Instance& instance;
|
||||||
Vulkan::Scheduler& scheduler;
|
Vulkan::Scheduler& scheduler;
|
||||||
Vulkan::StreamBuffer staging;
|
BufferCache& buffer_cache;
|
||||||
|
PageManager& tracker;
|
||||||
|
StreamBuffer staging;
|
||||||
TileManager tile_manager;
|
TileManager tile_manager;
|
||||||
Common::SlotVector<Image> slot_images;
|
Common::SlotVector<Image> slot_images;
|
||||||
Common::SlotVector<ImageView> slot_image_views;
|
Common::SlotVector<ImageView> slot_image_views;
|
||||||
tsl::robin_map<u64, Sampler> samplers;
|
tsl::robin_map<u64, Sampler> samplers;
|
||||||
tsl::robin_pg_map<u64, std::vector<ImageId>> page_table;
|
PageTable page_table;
|
||||||
boost::icl::interval_map<VAddr, s32> cached_pages;
|
|
||||||
tsl::robin_map<VAddr, MetaDataInfo> surface_metas;
|
|
||||||
std::mutex mutex;
|
std::mutex mutex;
|
||||||
#ifdef _WIN64
|
|
||||||
void* veh_handle{};
|
struct MetaDataInfo {
|
||||||
#endif
|
enum class Type {
|
||||||
std::mutex m_page_table;
|
CMask,
|
||||||
|
FMask,
|
||||||
|
HTile,
|
||||||
|
};
|
||||||
|
Type type;
|
||||||
|
bool is_cleared;
|
||||||
|
};
|
||||||
|
tsl::robin_map<VAddr, MetaDataInfo> surface_metas;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace VideoCore
|
} // namespace VideoCore
|
||||||
|
|
|
@ -183,10 +183,12 @@ vk::Format DemoteImageFormatForDetiling(vk::Format format) {
|
||||||
case vk::Format::eB8G8R8A8Srgb:
|
case vk::Format::eB8G8R8A8Srgb:
|
||||||
case vk::Format::eB8G8R8A8Unorm:
|
case vk::Format::eB8G8R8A8Unorm:
|
||||||
case vk::Format::eR8G8B8A8Unorm:
|
case vk::Format::eR8G8B8A8Unorm:
|
||||||
|
case vk::Format::eR8G8B8A8Uint:
|
||||||
case vk::Format::eR32Sfloat:
|
case vk::Format::eR32Sfloat:
|
||||||
case vk::Format::eR32Uint:
|
case vk::Format::eR32Uint:
|
||||||
case vk::Format::eR16G16Sfloat:
|
case vk::Format::eR16G16Sfloat:
|
||||||
return vk::Format::eR32Uint;
|
return vk::Format::eR32Uint;
|
||||||
|
case vk::Format::eBc1RgbaSrgbBlock:
|
||||||
case vk::Format::eBc1RgbaUnormBlock:
|
case vk::Format::eBc1RgbaUnormBlock:
|
||||||
case vk::Format::eBc4UnormBlock:
|
case vk::Format::eBc4UnormBlock:
|
||||||
case vk::Format::eR32G32Sfloat:
|
case vk::Format::eR32G32Sfloat:
|
||||||
|
@ -200,11 +202,20 @@ vk::Format DemoteImageFormatForDetiling(vk::Format format) {
|
||||||
case vk::Format::eBc5UnormBlock:
|
case vk::Format::eBc5UnormBlock:
|
||||||
case vk::Format::eBc7SrgbBlock:
|
case vk::Format::eBc7SrgbBlock:
|
||||||
case vk::Format::eBc7UnormBlock:
|
case vk::Format::eBc7UnormBlock:
|
||||||
|
case vk::Format::eBc6HUfloatBlock:
|
||||||
|
case vk::Format::eR32G32B32A32Sfloat:
|
||||||
return vk::Format::eR32G32B32A32Uint;
|
return vk::Format::eR32G32B32A32Uint;
|
||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Log missing formats only once to avoid spamming the log.
|
||||||
|
static constexpr size_t MaxFormatIndex = 256;
|
||||||
|
static std::array<bool, MaxFormatIndex> logged_formats{};
|
||||||
|
if (const u32 index = u32(format); !logged_formats[index]) {
|
||||||
LOG_ERROR(Render_Vulkan, "Unexpected format for demotion {}", vk::to_string(format));
|
LOG_ERROR(Render_Vulkan, "Unexpected format for demotion {}", vk::to_string(format));
|
||||||
|
logged_formats[index] = true;
|
||||||
|
}
|
||||||
return format;
|
return format;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -236,8 +247,11 @@ struct DetilerParams {
|
||||||
u32 sizes[14];
|
u32 sizes[14];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static constexpr size_t StreamBufferSize = 128_MB;
|
||||||
|
|
||||||
TileManager::TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler)
|
TileManager::TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler)
|
||||||
: instance{instance}, scheduler{scheduler} {
|
: instance{instance}, scheduler{scheduler},
|
||||||
|
stream_buffer{instance, scheduler, MemoryUsage::Stream, StreamBufferSize} {
|
||||||
static const std::array detiler_shaders{
|
static const std::array detiler_shaders{
|
||||||
HostShaders::DETILE_M8X1_COMP, HostShaders::DETILE_M8X2_COMP,
|
HostShaders::DETILE_M8X1_COMP, HostShaders::DETILE_M8X2_COMP,
|
||||||
HostShaders::DETILE_M32X1_COMP, HostShaders::DETILE_M32X2_COMP,
|
HostShaders::DETILE_M32X1_COMP, HostShaders::DETILE_M32X2_COMP,
|
||||||
|
@ -336,8 +350,7 @@ TileManager::ScratchBuffer TileManager::AllocBuffer(u32 size, bool is_storage /*
|
||||||
.flags = !is_storage ? VMA_ALLOCATION_CREATE_HOST_ACCESS_ALLOW_TRANSFER_INSTEAD_BIT |
|
.flags = !is_storage ? VMA_ALLOCATION_CREATE_HOST_ACCESS_ALLOW_TRANSFER_INSTEAD_BIT |
|
||||||
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT
|
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT
|
||||||
: static_cast<VmaAllocationCreateFlags>(0),
|
: static_cast<VmaAllocationCreateFlags>(0),
|
||||||
.usage = is_large_buffer ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST
|
.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
||||||
: VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
|
||||||
.requiredFlags = !is_storage ? VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT
|
.requiredFlags = !is_storage ? VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT
|
||||||
: static_cast<VkMemoryPropertyFlags>(0),
|
: static_cast<VkMemoryPropertyFlags>(0),
|
||||||
};
|
};
|
||||||
|
@ -373,37 +386,46 @@ std::optional<vk::Buffer> TileManager::TryDetile(Image& image) {
|
||||||
|
|
||||||
const auto* detiler = GetDetiler(image);
|
const auto* detiler = GetDetiler(image);
|
||||||
if (!detiler) {
|
if (!detiler) {
|
||||||
|
if (image.info.tiling_mode != AmdGpu::TilingMode::Texture_MacroTiled) {
|
||||||
LOG_ERROR(Render_Vulkan, "Unsupported tiled image: {} ({})",
|
LOG_ERROR(Render_Vulkan, "Unsupported tiled image: {} ({})",
|
||||||
vk::to_string(image.info.pixel_format), NameOf(image.info.tiling_mode));
|
vk::to_string(image.info.pixel_format), NameOf(image.info.tiling_mode));
|
||||||
|
}
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare input buffer
|
// Prepare input buffer
|
||||||
auto in_buffer = AllocBuffer(image.info.guest_size_bytes);
|
const u32 image_size = image.info.guest_size_bytes;
|
||||||
Upload(in_buffer, reinterpret_cast<const void*>(image.info.guest_address),
|
const auto [in_buffer, in_offset] = [&] -> std::pair<vk::Buffer, u32> {
|
||||||
image.info.guest_size_bytes);
|
// Use stream buffer for smaller textures.
|
||||||
|
if (image_size <= StreamBufferSize) {
|
||||||
|
u32 offset = stream_buffer.Copy(image.info.guest_address, image_size);
|
||||||
|
return {stream_buffer.Handle(), offset};
|
||||||
|
}
|
||||||
|
// Request temporary host buffer for larger sizes.
|
||||||
|
auto in_buffer = AllocBuffer(image_size);
|
||||||
|
const auto addr = reinterpret_cast<const void*>(image.info.guest_address);
|
||||||
|
Upload(in_buffer, addr, image_size);
|
||||||
|
scheduler.DeferOperation([=, this]() { FreeBuffer(in_buffer); });
|
||||||
|
return {in_buffer.first, 0};
|
||||||
|
}();
|
||||||
|
|
||||||
// Prepare output buffer
|
// Prepare output buffer
|
||||||
auto out_buffer = AllocBuffer(image.info.guest_size_bytes, true);
|
auto out_buffer = AllocBuffer(image_size, true);
|
||||||
|
scheduler.DeferOperation([=, this]() { FreeBuffer(out_buffer); });
|
||||||
scheduler.DeferOperation([=, this]() {
|
|
||||||
FreeBuffer(in_buffer);
|
|
||||||
FreeBuffer(out_buffer);
|
|
||||||
});
|
|
||||||
|
|
||||||
auto cmdbuf = scheduler.CommandBuffer();
|
auto cmdbuf = scheduler.CommandBuffer();
|
||||||
cmdbuf.bindPipeline(vk::PipelineBindPoint::eCompute, *detiler->pl);
|
cmdbuf.bindPipeline(vk::PipelineBindPoint::eCompute, *detiler->pl);
|
||||||
|
|
||||||
const vk::DescriptorBufferInfo input_buffer_info{
|
const vk::DescriptorBufferInfo input_buffer_info{
|
||||||
.buffer = in_buffer.first,
|
.buffer = in_buffer,
|
||||||
.offset = 0,
|
.offset = in_offset,
|
||||||
.range = image.info.guest_size_bytes,
|
.range = image_size,
|
||||||
};
|
};
|
||||||
|
|
||||||
const vk::DescriptorBufferInfo output_buffer_info{
|
const vk::DescriptorBufferInfo output_buffer_info{
|
||||||
.buffer = out_buffer.first,
|
.buffer = out_buffer.first,
|
||||||
.offset = 0,
|
.offset = 0,
|
||||||
.range = image.info.guest_size_bytes,
|
.range = image_size,
|
||||||
};
|
};
|
||||||
|
|
||||||
std::vector<vk::WriteDescriptorSet> set_writes{
|
std::vector<vk::WriteDescriptorSet> set_writes{
|
||||||
|
@ -442,16 +464,16 @@ std::optional<vk::Buffer> TileManager::TryDetile(Image& image) {
|
||||||
cmdbuf.pushConstants(*detiler->pl_layout, vk::ShaderStageFlagBits::eCompute, 0u, sizeof(params),
|
cmdbuf.pushConstants(*detiler->pl_layout, vk::ShaderStageFlagBits::eCompute, 0u, sizeof(params),
|
||||||
¶ms);
|
¶ms);
|
||||||
|
|
||||||
ASSERT((image.info.guest_size_bytes % 64) == 0);
|
ASSERT((image_size % 64) == 0);
|
||||||
const auto bpp = image.info.num_bits * (image.info.props.is_block ? 16u : 1u);
|
const auto bpp = image.info.num_bits * (image.info.props.is_block ? 16u : 1u);
|
||||||
const auto num_tiles = image.info.guest_size_bytes / (64 * (bpp / 8));
|
const auto num_tiles = image_size / (64 * (bpp / 8));
|
||||||
cmdbuf.dispatch(num_tiles, 1, 1);
|
cmdbuf.dispatch(num_tiles, 1, 1);
|
||||||
|
|
||||||
const vk::BufferMemoryBarrier post_barrier{
|
const vk::BufferMemoryBarrier post_barrier{
|
||||||
.srcAccessMask = vk::AccessFlagBits::eShaderWrite,
|
.srcAccessMask = vk::AccessFlagBits::eShaderWrite,
|
||||||
.dstAccessMask = vk::AccessFlagBits::eTransferRead,
|
.dstAccessMask = vk::AccessFlagBits::eTransferRead,
|
||||||
.buffer = out_buffer.first,
|
.buffer = out_buffer.first,
|
||||||
.size = image.info.guest_size_bytes,
|
.size = image_size,
|
||||||
};
|
};
|
||||||
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
|
cmdbuf.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
|
||||||
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
|
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "common/types.h"
|
#include "common/types.h"
|
||||||
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
|
#include "video_core/buffer_cache/buffer.h"
|
||||||
#include "video_core/texture_cache/image.h"
|
#include "video_core/texture_cache/image.h"
|
||||||
|
|
||||||
namespace VideoCore {
|
namespace VideoCore {
|
||||||
|
@ -34,7 +34,7 @@ struct DetilerContext {
|
||||||
|
|
||||||
class TileManager {
|
class TileManager {
|
||||||
public:
|
public:
|
||||||
using ScratchBuffer = std::pair<VkBuffer, VmaAllocation>;
|
using ScratchBuffer = std::pair<vk::Buffer, VmaAllocation>;
|
||||||
|
|
||||||
TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler);
|
TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler);
|
||||||
~TileManager();
|
~TileManager();
|
||||||
|
@ -51,6 +51,7 @@ private:
|
||||||
private:
|
private:
|
||||||
const Vulkan::Instance& instance;
|
const Vulkan::Instance& instance;
|
||||||
Vulkan::Scheduler& scheduler;
|
Vulkan::Scheduler& scheduler;
|
||||||
|
StreamBuffer stream_buffer;
|
||||||
std::array<DetilerContext, DetilerType::Max> detilers;
|
std::array<DetilerContext, DetilerType::Max> detilers;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue