perf(agc): keep render targets on gpu across acquire_mem

core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp

@@ -25,6 +25,7 @@ std::string_view UnsupportedReason(std::uint32_t header);
 void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue);
 void Execute(std::span<const std::uint32_t> packet, QueueState& queue);
 bool AccessesMemory(std::uint32_t header);
+bool UsesGpuCacheBarrier(std::span<const std::uint32_t> packet);
 std::array<std::uint32_t, 5> ResolveDispatch(std::span<const std::uint32_t> packet, const QueueState& queue);
 DrawParameters ResolveDraw(std::span<const std::uint32_t> packet, const QueueState& queue);
 

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -19,6 +19,8 @@ public:
     VulkanDevice& operator=(const VulkanDevice&) = delete;
     ShaderRecompiler::SpirvTarget Target() const;
     void WaitIdle();
+    void WaitDraws();
+    void AcquireGpuMemory();
     void ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable);
     void* Window() const;
     void Resize(std::uint32_t width, std::uint32_t height);

core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp

@@ -227,8 +227,8 @@ public:
                 if (presenting->Presentable()) {
                     if (buffer != nullptr) {
                         require(buffer->width == window.width && buffer->height == window.height, "display buffer extent differs from output");
-                        presenting->WaitIdle();
+                        presenting->WaitDraws();
-                        timing.Mark("device_idle_wait");
+                        timing.Mark("draw_wait");
                         presenting->PresentDisplayBuffer(*buffer);
                         timing.Mark("present_display_buffer");
                     } else {
@@ -564,8 +564,20 @@ private:
                 if (opcode == 0x37 || opcode == 0x40 || opcode == 0x50 || opcode == 0x42 || opcode == 0x46 || opcode == 0x58 || header == FlipPacketHeader) {
                     std::lock_guard gpuLock(gpuMutex);
                     timing.Mark("gpu_mutex_wait");
-                    if (device != nullptr) device->WaitIdle();
+                    const auto eventType = opcode == 0x46 ? packet[1] & 0x3fu : 0u;
-                    timing.Mark("device_idle_wait");
+                    const auto memoryTransfer = opcode == 0x37 || opcode == 0x40 || opcode == 0x50;
+                    const auto waitDraws = memoryTransfer || opcode == 0x42 || (opcode == 0x46 && (eventType == 0x07 || eventType == 0x0f || eventType == 0x10));
+                    const auto gpuCacheBarrier = opcode == 0x58 && Pm4::UsesGpuCacheBarrier(packet);
+                    if (device != nullptr) {
+                        if (gpuCacheBarrier) device->AcquireGpuMemory();
+                        else if (waitDraws) device->WaitDraws();
+                        else {
+                            const auto scope = header == FlipPacketHeader ? "Driver.FlipWait" : opcode == 0x58 ? "Driver.AcquireMemoryWait" : "Driver.CacheEventWait";
+                            PerformanceTimer waitTiming(scope);
+                            device->WaitIdle();
+                        }
+                    }
+                    timing.Mark(gpuCacheBarrier ? "gpu_cache_barrier" : waitDraws ? "draw_wait" : "device_idle_wait");
                 }
                 if (header == FlipPacketHeader) {
                     CheckFailure();

core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp

@@ -266,6 +266,12 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
     }
 }
 
+bool UsesGpuCacheBarrier(std::span<const std::uint32_t> packet) {
+    require(!packet.empty() && ((packet[0] >> 8u) & 0xffu) == 0x58, "cache barrier requires ACQUIRE_MEM");
+    Validate(packet, 0);
+    return packet.size() == 8 && (packet[7] & 0xfc00u) == 0;
+}
+
 bool AccessesMemory(std::uint32_t header) {
     switch ((header >> 8u) & 0xffu) {
         case 0x16: case 0x2d: case 0x35: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true;

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -482,8 +482,32 @@ VulkanDevice::~VulkanDevice() = default;
 void VulkanDevice::WaitIdle() {
     PerformanceTimer timing("Vulkan.WaitIdle");
     state->drawQueue->Wait();
+    timing.Mark("draw_wait");
     state->renderCache->Flush();
+    timing.Mark("render_cache_flush");
     check(state->DeviceFunction<PFN_vkDeviceWaitIdle>("vkDeviceWaitIdle")(state->device), "vkDeviceWaitIdle");
+    timing.Mark("device_wait");
+}
+
+void VulkanDevice::WaitDraws() {
+    PerformanceTimer timing("Vulkan.WaitDraws");
+    state->drawQueue->Wait();
+}
+
+void VulkanDevice::AcquireGpuMemory() {
+    PerformanceTimer timing("Vulkan.AcquireGpuMemory");
+    state->drawQueue->Wait();
+    timing.Mark("draw_wait");
+    const auto context = graphicsContext();
+    Graphics::CommandBatch batch(context);
+    VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
+    barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT;
+    barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
+    context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(batch.Handle(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr);
+    batch.SubmitAndWait();
+    timing.Mark("gpu_barrier");
+    state->renderCache->InvalidateClean();
+    timing.Mark("invalidate_clean");
 }
 
 void* VulkanDevice::Window() const {

core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp

@@ -8,6 +8,7 @@ namespace AgcDriver::Graphics {
 class DrawQueue {
 public:
     ~DrawQueue();
+    std::unique_ptr<CommandBatch> Begin(const Context& context);
     void Submit(std::unique_ptr<CommandBatch> commands, std::shared_ptr<ShaderResources> resources, std::shared_ptr<void> storage);
     void Resolve(std::uint64_t address, std::size_t bytes);
     void Wait();
@@ -19,6 +20,7 @@ private:
         std::unique_ptr<CommandBatch> commands;
     };
     std::vector<Entry> pending;
+    std::vector<std::unique_ptr<CommandBatch>> available;
 };
 
 }

core/libs/prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp

@@ -4,6 +4,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp"
 #include <list>
+#include <map>
 
 namespace AgcDriver::Graphics {
 
@@ -20,6 +21,7 @@ private:
     };
     Context context;
     std::list<Entry> entries;
+    std::map<std::vector<std::byte>, std::list<Entry>::iterator> lookup;
 };
 
 }

core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp

@@ -39,6 +39,7 @@ public:
     std::shared_ptr<ResidentColor> Find(std::uint64_t address) const;
     void Resolve(std::uint64_t address, std::size_t bytes, bool writable);
     void Flush();
+    void InvalidateClean();
 
 private:
     Context context;

core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp

@@ -59,6 +59,7 @@ public:
     void SubmitAndWait();
     void Submit();
     void Wait();
+    void Reset();
 
 private:
     void release() noexcept;

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -107,7 +107,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
     storage->pipeline = context.graphicsPipelines->Get(state, storage->color, *resources, shaders);
     auto& pipeline = *storage->pipeline;
     timing.Mark("pipeline_cache");
-    auto batch = std::make_unique<CommandBatch>(context);
+    auto batch = context.drawQueue->Begin(context);
     const auto commands = batch->Handle();
     VkMemoryBarrier upload{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
     upload.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT;

core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp

@@ -9,6 +9,14 @@ DrawQueue::~DrawQueue() {
     for (auto& entry : pending) entry.commands.reset();
 }
 
+std::unique_ptr<CommandBatch> DrawQueue::Begin(const Context& context) {
+    if (available.empty()) return std::make_unique<CommandBatch>(context);
+    auto commands = std::move(available.back());
+    available.pop_back();
+    commands->Reset();
+    return commands;
+}
+
 void DrawQueue::Submit(std::unique_ptr<CommandBatch> commands, std::shared_ptr<ShaderResources> resources, std::shared_ptr<void> storage) {
     if (pending.size() >= 64) Wait();
     pending.push_back({std::move(storage), std::move(resources), std::move(commands)});
@@ -33,6 +41,7 @@ void DrawQueue::Wait() {
         timing.Mark("fence_wait");
         entry.resources->WriteBack();
         timing.Mark("resources_writeback");
+        available.push_back(std::move(entry.commands));
     }
     pending.clear();
 }

core/libs/prx/libSceAgcDriver/Graphics/src/GraphicsPipelineCache.cpp

@@ -1,7 +1,9 @@
 #include "prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include <type_traits>
 #include <algorithm>
+#include <iterator>
 
 namespace AgcDriver::Graphics {
 namespace {
@@ -15,6 +17,7 @@ void append(std::vector<std::byte>& key, const TValue& value) {
 
 std::vector<std::byte> makeKey(const Context& context, const State& state, const std::shared_ptr<ResidentColor>& target, const ShaderResources& resources, std::span<const CompiledShader> shaders) {
     std::vector<std::byte> key;
+    key.reserve(512);
     append(key, target ? target->Target().View() : VK_NULL_HANDLE);
     append(key, state.hasColorTarget);
     append(key, state.rectList);
@@ -94,18 +97,32 @@ std::vector<std::byte> makeKey(const Context& context, const State& state, const
 }
 
 std::shared_ptr<Pipeline> GraphicsPipelineCache::Get(const State& state, const std::shared_ptr<ResidentColor>& target, const ShaderResources& resources, std::span<const CompiledShader> shaders) {
+    PerformanceTimer timing("Graphics.PipelineCache");
     auto key = makeKey(context, state, target, resources, shaders);
-    for (auto it = entries.begin(); it != entries.end(); ++it) {
+    timing.Mark("key");
-        if (it->key != key) continue;
+    const auto found = lookup.find(key);
+    if (found != lookup.end()) {
+        const auto it = found->second;
         auto pipeline = it->pipeline;
         entries.splice(entries.end(), entries, it);
+        timing.Mark("hit");
         return pipeline;
     }
+    timing.Mark("miss");
     auto pipeline = std::make_shared<Pipeline>(context, state, target ? &target->Target() : nullptr, resources, shaders);
+    timing.Mark("create");
     entries.push_back({std::move(key), target, pipeline});
+    try {
+        const auto it = std::prev(entries.end());
+        Require(lookup.emplace(it->key, it).second, "duplicate graphics pipeline cache key");
+    } catch (...) {
+        entries.pop_back();
+        throw;
+    }
     while (entries.size() > 128) {
         const auto it = std::find_if(entries.begin(), entries.end(), [](const auto& entry) { return entry.pipeline.use_count() == 1; });
         if (it == entries.end()) break;
+        lookup.erase(it->key);
         entries.erase(it);
     }
     return pipeline;

core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp

@@ -131,4 +131,10 @@ void RenderCache::Flush() {
     Resolve(0, std::numeric_limits<std::size_t>::max(), true);
 }
 
+void RenderCache::InvalidateClean() {
+    for (const auto& [address, entry] : entries) {
+        if (!entry->Dirty()) entry->Invalidate();
+    }
+}
+
 }

core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp

@@ -172,6 +172,16 @@ void CommandBatch::SubmitAndWait() {
     Wait();
 }
 
+void CommandBatch::Reset() {
+    Require(submitted && !pending, "command batch must complete before reuse");
+    Check(context.Function<PFN_vkResetFences>("vkResetFences")(context.device, 1, &fence), "vkResetFences graphics");
+    Check(context.Function<PFN_vkResetCommandBuffer>("vkResetCommandBuffer")(commands, 0), "vkResetCommandBuffer graphics");
+    VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
+    begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+    Check(context.Function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer graphics");
+    submitted = false;
+}
+
 void CommandBatch::Submit() {
     PerformanceTimer timing("Graphics.Submit");
     Require(!submitted, "command batch has already been submitted");

core/libs/prx/libSceAgcDriver/tests/Pm4.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
 #include "prx/libc/include/Shutdown.hpp"
@@ -228,6 +229,44 @@ void testCopies() {
 #endif
 }
 
+void testMemorySynchronization() {
+    struct MemoryState {
+        std::uint32_t source = 0;
+        std::uint32_t destination = 0;
+        bool read = false;
+        bool written = false;
+    } memory;
+    AgcDriver::QueueState state;
+    const auto resolve = [](void* context, std::uint64_t address, std::size_t bytes, bool writable) {
+        auto& memory = *static_cast<MemoryState*>(context);
+        check(bytes == sizeof(std::uint32_t), "memory transfer resolved an unrelated range");
+        if (writable) {
+            check(address == reinterpret_cast<std::uintptr_t>(&memory.destination), "memory transfer resolved an unrelated destination");
+            memory.written = true;
+        } else {
+            check(address == reinterpret_cast<std::uintptr_t>(&memory.source), "memory transfer resolved an unrelated source");
+            memory.source = 42;
+            memory.read = true;
+        }
+    };
+    const AgcDriver::GuestMemory::MemoryAccessScope scope(&memory, resolve);
+    execute(state, makePacket(0x37, {0x100, low(&memory.destination), high(&memory.destination), 17}));
+    check(memory.written && !memory.read && memory.destination == 17, "WRITE_DATA did not synchronize its destination");
+    for (const auto opcode : {0x40u, 0x50u}) {
+        memory = {};
+        const auto packet = opcode == 0x40
+            ? makePacket(opcode, {0x101, low(&memory.source), high(&memory.source), low(&memory.destination), high(&memory.destination)})
+            : makePacket(opcode, {0x60000000, low(&memory.source), high(&memory.source), low(&memory.destination), high(&memory.destination), 4});
+        execute(state, packet);
+        check(memory.read && memory.written && memory.destination == 42, "memory copy used stale data before range synchronization");
+    }
+    const AgcDriver::GuestMemory::MemoryAccessScope rejecting(&memory, [](void*, std::uint64_t, std::size_t, bool) {
+        throw std::runtime_error("range synchronization failed");
+    });
+    expectFailure([&] { execute(state, makePacket(0x37, {0x100, low(&memory.destination), high(&memory.destination), 99})); }, "range synchronization failed");
+    check(memory.destination == 42, "failed synchronization changed the destination");
+}
+
 void testEventWrite() {
     for (const auto eventType : {0x07u, 0x0fu, 0x10u}) {
         AgcDriver::Pm4::Validate(makePacket(0x46, {0x400u | eventType}), 0);
@@ -262,6 +301,20 @@ void testEventWrite() {
 void testAcquireMem() {
     const auto captured = makePacket(0x58, {0x02007fc0, 0, 0, 0, 0, 10, 0x200});
     AgcDriver::Pm4::Validate(captured, 0);
+    check(AgcDriver::Pm4::UsesGpuCacheBarrier(captured), "L1 acquire must preserve GPU render targets");
+    for (const auto flags : {0u, 0x200u, 0x3ffu, 0x10200u, 0x20200u}) {
+        auto packet = captured;
+        packet[7] = flags;
+        check(AgcDriver::Pm4::UsesGpuCacheBarrier(packet), "GPU cache acquire requires an unnecessary host writeback");
+    }
+    for (const auto flags : {0x400u, 0x800u, 0x1000u, 0x4000u, 0x8000u}) {
+        auto packet = captured;
+        packet[7] = flags;
+        check(!AgcDriver::Pm4::UsesGpuCacheBarrier(packet), "L2 acquire lost host synchronization");
+    }
+    check(!AgcDriver::Pm4::UsesGpuCacheBarrier(makePacket(0x58, {0x00800000, 0xffffffff, 0, 0, 0, 10})), "legacy acquire lost host synchronization");
+    expectFailure([] { AgcDriver::Pm4::UsesGpuCacheBarrier({}); }, "requires ACQUIRE_MEM");
+    expectFailure([] { AgcDriver::Pm4::UsesGpuCacheBarrier(makePacket(0x58, {0, 0, 0, 0, 0, 0, 0x2000})); }, "cache discard");
     AgcDriver::Pm4::Validate(makePacket(0x58, {0x82007fc0, 1, 0, 0xffffffff, 0, 0xffff, 0x200}), 0);
     AgcDriver::Pm4::Validate(makePacket(0x58, {0x80000000, 0, 0, 0, 0, 10, 0x200}), 0x20);
     AgcDriver::Pm4::Validate(makePacket(0x58, {0x00800000, 0xffffffff, 0, 0, 0, 10}), 0);
@@ -359,6 +412,7 @@ int main(int argc, char** argv) {
         testAutoDraw();
         testMemory();
         testCopies();
+        testMemorySynchronization();
         testEventWrite();
         testAcquireMem();
         testDriverSubmission();

core/libs/prx/libSceVideoOut/src/DisplayWindow.cpp

@@ -98,7 +98,7 @@ void DisplayWindow::UpdateTitle() {
         fpsFrames = 0;
     }
     char text[160];
-    std::snprintf(text, sizeof(text), "%s | %llu", title.value, static_cast<unsigned long long>(frameNum));
+    std::snprintf(text, sizeof(text), "%s | FPS: %.2f (%llu)", title.value, currentFps, static_cast<unsigned long long>(frameNum));
     SDL_SetWindowTitle(window, text);
 }