video_core: initial support for CE and ASC queues

src/core/libraries/gnmdriver/gnmdriver.cpp

@@ -1447,7 +1447,9 @@ int PS4_SYSV_ABI sceGnmSubmitCommandBuffersForWorkload() {
 
 int PS4_SYSV_ABI sceGnmSubmitDone() {
     LOG_INFO(Lib_GnmDriver, "called");
+    if (!liverpool->IsGpuIdle()) {
         submission_lock = true;
+    }
     return ORBIS_OK;
 }
 

src/video_core/amdgpu/liverpool.cpp

@@ -10,6 +10,8 @@
 
 namespace AmdGpu {
 
+std::array<u8, 48_KB> Liverpool::ConstantEngine::constants_heap;
+
 Liverpool::Liverpool() {
     process_thread = std::jthread{std::bind_front(&Liverpool::Process, this)};
 }
@@ -20,50 +22,124 @@ Liverpool::~Liverpool() {
 }
 
 void Liverpool::Process(std::stop_token stoken) {
+    Common::SetCurrentThreadName("GPU_CommandProcessor");
+
     while (!stoken.stop_requested()) {
-        std::span<const u32> dcb{};
         {
-            std::unique_lock lock{m_ring_access};
+            std::unique_lock lock{m_submit};
-            cv_submit.wait(lock, stoken, [&]() { return !gfx_ring.empty(); });
+            cv_submit.wait(lock, stoken, [this]() { return num_submits != 0; });
+        }
 
         if (stoken.stop_requested()) {
             break;
         }
 
-            dcb = gfx_ring.front();
+        int qid = -1;
-            gfx_ring.pop();
-        }
 
-        ASSERT_MSG(!dcb.empty(), "Empty command list received");
+        while (num_submits) {
-        ProcessCmdList(dcb.data(), dcb.size_bytes());
+            qid = (qid + 1) % NumTotalQueues;
 
+            auto& queue = mapped_queues[qid];
+
+            Task::Handle task{};
             {
-            std::unique_lock lock{m_ring_access};
+                std::scoped_lock lock{queue.m_access};
-            if (gfx_ring.empty()) {
+
-                cv_complete.notify_all();
+                if (queue.submits.empty()) {
+                    continue;
+                }
+
+                task = queue.submits.front();
+            }
+            task.resume();
+
+            if (task.done()) {
+                std::scoped_lock lock{queue.m_access};
+                queue.submits.pop();
+
+                --num_submits;
             }
         }
+        cv_complete.notify_all(); // Notify GPU idle
     }
 }
 
 void Liverpool::WaitGpuIdle() {
-    std::unique_lock lock{m_ring_access};
+    std::unique_lock lock{m_submit};
-    cv_complete.wait(lock, [this]() { return gfx_ring.empty(); });
+    cv_complete.wait(lock, [this]() { return num_submits == 0; });
 }
 
-void Liverpool::ProcessCmdList(const u32* cmdbuf, u32 size_in_bytes) {
+Liverpool::Task Liverpool::ProcessCeUpdate(std::span<const u32> ccb) {
-    Common::SetCurrentThreadName("CommandProcessor_Gfx");
+    while (!ccb.empty()) {
-
+        const auto* header = reinterpret_cast<const PM4Header*>(ccb.data());
-    auto* header = reinterpret_cast<const PM4Header*>(cmdbuf);
-    u32 processed_cmd_size = 0;
-
-    while (processed_cmd_size < size_in_bytes) {
-        const PM4Header* next_header{};
         const u32 type = header->type;
-        switch (type) {
+        if (type != 3) {
-        case 3: {
+            // No other types of packets were spotted so far
+            UNREACHABLE_MSG("Invalid PM4 type {}", type);
+        }
+
         const PM4ItOpcode opcode = header->type3.opcode;
+        const auto* it_body = reinterpret_cast<const u32*>(header) + 1;
+        switch (opcode) {
+        case PM4ItOpcode::Nop: {
+            const auto* nop = reinterpret_cast<const PM4CmdNop*>(header);
+            break;
+        }
+        case PM4ItOpcode::WriteConstRam: {
+            const auto* write_const = reinterpret_cast<const PM4WriteConstRam*>(header);
+            memcpy(cblock.constants_heap.data() + write_const->Offset(), &write_const->data,
+                   write_const->Size());
+            break;
+        }
+        case PM4ItOpcode::DumpConstRam: {
+            const auto* dump_const = reinterpret_cast<const PM4DumpConstRam*>(header);
+            memcpy(dump_const->Address<void*>(),
+                   cblock.constants_heap.data() + dump_const->Offset(), dump_const->Size());
+            break;
+        }
+        case PM4ItOpcode::IncrementCeCounter: {
+            ++cblock.ce_count;
+            break;
+        }
+        case PM4ItOpcode::WaitOnDeCounterDiff: {
+            const auto diff = it_body[0];
+            while ((cblock.de_count - cblock.ce_count) >= diff) {
+                co_yield {};
+            }
+            break;
+        }
+        default:
             const u32 count = header->type3.NumWords();
+            UNREACHABLE_MSG("Unknown PM4 type 3 opcode {:#x} with count {}",
+                            static_cast<u32>(opcode), count);
+        }
+        ccb = ccb.subspan(header->type3.NumWords() + 1);
+    }
+}
+
+Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<const u32> ccb) {
+    cblock.Reset();
+
+    // TODO: potentially, ASCs also can depend on CE and in this case the
+    // CE task should be moved into more global scope
+    Task ce_task{};
+
+    if (!ccb.empty()) {
+        // In case of CCB provided kick off CE asap to have the constant heap ready to use
+        ce_task = ProcessCeUpdate(ccb);
+        ce_task.handle.resume();
+    }
+
+    while (!dcb.empty()) {
+        const auto* header = reinterpret_cast<const PM4Header*>(dcb.data());
+        const u32 type = header->type;
+        if (type != 3) {
+            // No other types of packets were spotted so far
+            UNREACHABLE_MSG("Invalid PM4 type {}", type);
+        }
+
+        const u32 count = header->type3.NumWords();
+        const PM4ItOpcode opcode = header->type3.opcode;
         switch (opcode) {
         case PM4ItOpcode::Nop: {
             const auto* nop = reinterpret_cast<const PM4CmdNop*>(header);
@@ -85,8 +161,8 @@ void Liverpool::ProcessCmdList(const u32* cmdbuf, u32 size_in_bytes) {
         }
         case PM4ItOpcode::SetContextReg: {
             const auto* set_data = reinterpret_cast<const PM4CmdSetData*>(header);
-                std::memcpy(&regs.reg_array[ContextRegWordOffset + set_data->reg_offset],
+            std::memcpy(&regs.reg_array[ContextRegWordOffset + set_data->reg_offset], header + 2,
-                            header + 2, (count - 1) * sizeof(u32));
+                        (count - 1) * sizeof(u32));
             break;
         }
         case PM4ItOpcode::SetShReg: {
@@ -97,8 +173,8 @@ void Liverpool::ProcessCmdList(const u32* cmdbuf, u32 size_in_bytes) {
         }
         case PM4ItOpcode::SetUconfigReg: {
             const auto* set_data = reinterpret_cast<const PM4CmdSetData*>(header);
-                std::memcpy(&regs.reg_array[UconfigRegWordOffset + set_data->reg_offset],
+            std::memcpy(&regs.reg_array[UconfigRegWordOffset + set_data->reg_offset], header + 2,
-                            header + 2, (count - 1) * sizeof(u32));
+                        (count - 1) * sizeof(u32));
             break;
         }
         case PM4ItOpcode::IndexType: {
@@ -131,6 +207,10 @@ void Liverpool::ProcessCmdList(const u32* cmdbuf, u32 size_in_bytes) {
             // const auto* dispatch_direct = reinterpret_cast<PM4CmdDispatchDirect*>(header);
             break;
         }
+        case PM4ItOpcode::EventWrite: {
+            // const auto* event = reinterpret_cast<const PM4CmdEventWrite*>(header);
+            break;
+        }
         case PM4ItOpcode::EventWriteEos: {
             const auto* event_eos = reinterpret_cast<const PM4CmdEventWriteEos*>(header);
             event_eos->SignalFence();
@@ -164,8 +244,17 @@ void Liverpool::ProcessCmdList(const u32* cmdbuf, u32 size_in_bytes) {
             const auto* wait_reg_mem = reinterpret_cast<const PM4CmdWaitRegMem*>(header);
             ASSERT(wait_reg_mem->engine.Value() == PM4CmdWaitRegMem::Engine::Me);
             while (!wait_reg_mem->Test()) {
-                    using namespace std::chrono_literals;
+                co_yield {};
-                    std::this_thread::sleep_for(1ms);
+            }
+            break;
+        }
+        case PM4ItOpcode::IncrementDeCounter: {
+            ++cblock.de_count;
+            break;
+        }
+        case PM4ItOpcode::WaitOnCeCounter: {
+            while (cblock.ce_count <= cblock.de_count) {
+                ce_task.handle.resume();
             }
             break;
         }
@@ -173,16 +262,71 @@ void Liverpool::ProcessCmdList(const u32* cmdbuf, u32 size_in_bytes) {
             UNREACHABLE_MSG("Unknown PM4 type 3 opcode {:#x} with count {}",
                             static_cast<u32>(opcode), count);
         }
-            next_header = header + header->type3.NumWords() + 1;
+
-            break;
+        dcb = dcb.subspan(header->type3.NumWords() + 1);
     }
-        default:
+
+    if (ce_task.handle) {
+        ASSERT_MSG(ce_task.handle.done(), "Partially processed CCB");
+    }
+}
+
+Liverpool::Task Liverpool::ProcessCompute(std::span<const u32> acb) {
+    while (!acb.empty()) {
+        const auto* header = reinterpret_cast<const PM4Header*>(acb.data());
+        const u32 type = header->type;
+        if (type != 3) {
+            // No other types of packets were spotted so far
             UNREACHABLE_MSG("Invalid PM4 type {}", type);
         }
 
-        processed_cmd_size += uintptr_t(next_header) - uintptr_t(header);
+        const u32 count = header->type3.NumWords();
-        header = next_header;
+        const PM4ItOpcode opcode = header->type3.opcode;
+        const auto* it_body = reinterpret_cast<const u32*>(header) + 1;
+        switch (opcode) {
+        default:
+            UNREACHABLE_MSG("Unknown PM4 type 3 opcode {:#x} with count {}",
+                            static_cast<u32>(opcode), count);
         }
+
+        acb = acb.subspan(header->type3.NumWords() + 1);
+    }
+
+    return {}; // Not a coroutine yet
+}
+
+void Liverpool::SubmitGfx(std::span<const u32> dcb, std::span<const u32> ccb) {
+    static constexpr u32 GfxQueueId = 0u;
+    auto& queue = mapped_queues[GfxQueueId];
+
+    auto task = ProcessGraphics(dcb, ccb);
+    {
+        std::unique_lock lock{queue.m_access};
+        queue.submits.emplace(task.handle);
+    }
+
+    {
+        std::unique_lock lock{m_submit};
+        ++num_submits;
+    }
+    cv_submit.notify_one();
+}
+
+void Liverpool::SubmitAsc(u32 vqid, std::span<const u32> acb) {
+    ASSERT_MSG(vqid > 0 && vqid < NumTotalQueues, "Invalid virtual ASC queue index");
+    auto& queue = mapped_queues[vqid];
+
+    const auto& task = ProcessCompute(acb);
+    {
+        std::unique_lock lock{queue.m_access};
+        queue.submits.emplace(task.handle);
+    }
+
+    {
+        std::unique_lock lock{m_submit};
+        ++num_submits;
+    }
+    cv_submit.notify_one();
 }
 
 } // namespace AmdGpu

src/video_core/amdgpu/liverpool.h

@@ -10,6 +10,7 @@
 
 #include <array>
 #include <condition_variable>
+#include <coroutine>
 #include <functional>
 #include <future>
 #include <span>
@@ -30,6 +31,12 @@ namespace AmdGpu {
     [[maybe_unused]] std::array<u32, num_words> CONCAT2(pad, __LINE__)
 
 struct Liverpool {
+    static constexpr u32 NumGfxRings = 1u;     // actually 2, but HP is reserved by system software
+    static constexpr u32 NumComputePipes = 7u; // actually 8, but #7 is reserved by system software
+    static constexpr u32 NumQueuesPerPipe = 8u;
+    static constexpr u32 NumTotalQueues = NumGfxRings + (NumComputePipes * NumQueuesPerPipe);
+    static_assert(NumTotalQueues < 64u); // need to fit into u64 bitmap for ffs
+
     static constexpr u32 NumColorBuffers = 8;
     static constexpr u32 NumViewports = 16;
     static constexpr u32 NumClipPlanes = 6;
@@ -631,32 +638,81 @@ public:
     Liverpool();
     ~Liverpool();
 
-    void SubmitGfx(std::span<const u32> dcb, std::span<const u32> ccb) {
+    void SubmitGfx(std::span<const u32> dcb, std::span<const u32> ccb);
-        {
+    void SubmitAsc(u32 vqid, std::span<const u32> acb);
-            std::scoped_lock lock{m_ring_access};
-            gfx_ring.emplace(dcb);
-
-            ASSERT_MSG(ccb.size() == 0, "CCBs are not supported yet");
-        }
-        cv_submit.notify_one();
-    }
 
     void WaitGpuIdle();
+    bool IsGpuIdle() const {
+        return num_submits == 0;
+    }
 
     void BindRasterizer(Vulkan::Rasterizer* rasterizer_) {
         rasterizer = rasterizer_;
     }
 
 private:
-    void ProcessCmdList(const u32* cmdbuf, u32 size_in_bytes);
+    struct Task {
+        struct promise_type {
+            auto get_return_object() {
+                Task task{};
+                task.handle = std::coroutine_handle<promise_type>::from_promise(*this);
+                return task;
+            }
+            static constexpr std::suspend_always initial_suspend() noexcept {
+                // We want the task to be suspended at start
+                return {};
+            }
+            static constexpr std::suspend_always final_suspend() noexcept {
+                return {};
+            }
+            void unhandled_exception() {}
+            void return_void() {}
+            struct empty {};
+            std::suspend_always yield_value(empty&&) {
+                return {};
+            }
+        };
+
+        using Handle = std::coroutine_handle<promise_type>;
+        Handle handle;
+    };
+
+    Task ProcessGraphics(std::span<const u32> dcb, std::span<const u32> ccb);
+    Task ProcessCeUpdate(std::span<const u32> ccb);
+    Task ProcessCompute(std::span<const u32> acb);
+
     void Process(std::stop_token stoken);
 
+    struct GpuQueue {
+        std::mutex m_access{};
+        std::queue<Task::Handle> submits{};
+    };
+    std::array<GpuQueue, NumTotalQueues> mapped_queues{};
+
+    struct ConstantEngine {
+        void Reset() {
+            ce_count = 0;
+            de_count = 0;
+            ce_compare_count = 0;
+        }
+
+        [[nodiscard]] u32 Diff() const {
+            ASSERT_MSG(ce_count >= de_count, "DE counter is ahead of CE");
+            return ce_count - de_count;
+        }
+
+        u32 ce_compare_count{};
+        u32 ce_count{};
+        u32 de_count{};
+        static std::array<u8, 48_KB> constants_heap;
+    } cblock{};
+
     Vulkan::Rasterizer* rasterizer{};
     std::jthread process_thread{};
-    std::queue<std::span<const u32>> gfx_ring{};
     std::condition_variable_any cv_submit{};
     std::condition_variable cv_complete{};
-    std::mutex m_ring_access{};
+    std::mutex m_submit{};
+    std::atomic<u32> num_submits{};
 };
 
 static_assert(GFX6_3D_REG_INDEX(ps_program) == 0x2C08);

src/video_core/amdgpu/pm4_cmds.h

@@ -494,4 +494,50 @@ struct PM4CmdEventWriteEos {
     }
 };
 
+struct PM4WriteConstRam {
+    PM4Type3Header header;
+    union {
+        BitField<0, 16, u32> offset; // in DWs
+        u32 dw1;
+    };
+    u32 data[0];
+
+    [[nodiscard]] u32 Offset() const {
+        return offset.Value() << 2u;
+    }
+
+    [[nodiscard]] u32 Size() const {
+        return header.count << 2u;
+    }
+};
+
+struct PM4DumpConstRam {
+    PM4Type3Header header;
+    union {
+        BitField<0, 16, u32> offset; ///< Starting byte offset into the Constant RAM. The minimum
+                                     ///< granularity is 4 bytes
+        u32 dw1;
+    };
+    union {
+        BitField<0, 15, u32>
+            num_dw; ///< Number of DWs to read from the constant RAM. The minimum granularity is DWs
+        u32 dw2;
+    };
+    u32 addr_lo;
+    u32 addr_hi;
+
+    template <typename T>
+    T* Address() const {
+        return reinterpret_cast<T*>((u64(addr_hi) << 32u) | addr_lo);
+    }
+
+    [[nodiscard]] u32 Offset() const {
+        return offset.Value();
+    }
+
+    [[nodiscard]] u32 Size() const {
+        return num_dw.Value() << 2u;
+    }
+};
+
 } // namespace AmdGpu

src/video_core/amdgpu/pm4_opcodes.h

@@ -58,7 +58,13 @@ enum class PM4ItOpcode : u32 {
     SetContextRegIndirect = 0x73,
     SetShReg = 0x76,
     SetShRegOffset = 0x77,
-    SetUconfigReg = 0x79
+    SetUconfigReg = 0x79,
+    WriteConstRam = 0x81,
+    DumpConstRam = 0x83,
+    IncrementCeCounter = 0x84,
+    IncrementDeCounter = 0x85,
+    WaitOnCeCounter = 0x86,
+    WaitOnDeCounterDiff = 0x88,
 };
 
 } // namespace AmdGpu