diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 010b75fc53..98bd5ca486 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -185,6 +185,10 @@ public: std::uint64_t created, underGpuMutex; }; static ComputePipelineCreation ComputePipelineCounts(); + struct DispatchMoveCount { + std::uint64_t checked, exact, moved, refused, filtered, rebuilt, kept, served; + }; + static DispatchMoveCount DispatchMoveCounts(); void ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable); void* Window() const; void Resize(std::uint32_t width, std::uint32_t height); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 49479c7750..bdef4030c0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -3722,11 +3722,24 @@ bool TemplateDataRefresh() { return enabled; } +bool DispatchMove() { + static const bool enabled = std::getenv("APS5_NO_DISPATCH_MOVE") == nullptr && TemplateDataRefresh(); + return enabled; +} + +std::size_t DispatchMoveBytes() { + static const std::size_t bytes = [] { + const char* text = std::getenv("APS5_DISPMOVE_MAX_KIB"); + return static_cast(text != nullptr ? std::strtoull(text, nullptr, 10) : 1024ull) * 1024u; + }(); + return bytes; +} + // The content key of a compute stage, naming the device: the cache is process-wide and the driver // replaces the headless device with the windowed one while workers may still use the old one, so // the key names the device that built the entry (its descriptor set and pooled buffers belong to it). -ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device) { - auto key = Graphics::ShaderResources::ContentKey(shader, !TemplateDataRefresh()); +ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device, bool movable = true) { + auto key = Graphics::ShaderResources::ContentKey(shader, !TemplateDataRefresh(), movable && DispatchMove()); const auto deviceHandle = reinterpret_cast(device); key.push_back(static_cast(deviceHandle)); key.push_back(static_cast(deviceHandle >> 32u)); @@ -3757,6 +3770,7 @@ struct PreparedDispatch { std::shared_ptr resources; std::shared_ptr cached; ResourceCache::Key key; + bool exact = false; // APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch, and // the prepare's parts for the driver's 'prepare:' rows (PreparePhase order). double prepareMs = 0; @@ -3956,6 +3970,63 @@ RecipeCounters& Recipes() { return counters; } +constexpr std::size_t MoveRefusals = static_cast(Graphics::ShaderResources::MoveRefusal::Count); + +struct DispatchMoveCounters { + std::atomic checked{0}, exact{0}, moved{0}, buffers{0}, bytes{0}; + std::atomic rebuilt{0}, kept{0}, served{0}; + std::array, 5> sizes{}; + std::array, MoveRefusals> refused{}; + std::array, MoveRefusals> filtered{}; + std::atomic lastReport{0}; +}; + +DispatchMoveCounters& DispatchMoves() { + static DispatchMoveCounters counters; + return counters; +} + +void reportDispatchMove() { + auto& moves = DispatchMoves(); + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = moves.lastReport.load(); + if (nowMs - last < 10000 || !moves.lastReport.compare_exchange_strong(last, nowMs)) return; + const auto take = [](std::atomic& counter) { return static_cast(counter.exchange(0, std::memory_order_relaxed)); }; + const auto list = [&](std::array, MoveRefusals>& counters) { + std::string text; + unsigned long long total = 0; + char item[80]; + for (std::size_t i = 1; i < MoveRefusals; ++i) { + const auto count = take(counters[i]); + total += count; + if (count == 0) continue; + std::snprintf(item, sizeof(item), "%s %s %llu", text.empty() ? "" : ",", Graphics::ShaderResources::MoveRefusalName(static_cast(i)), count); + text += item; + } + std::snprintf(item, sizeof(item), "%llu (", total); + return std::string(item) + text + ")"; + }; + const auto checked = take(moves.checked); + const auto exact = take(moves.exact); + const auto moved = take(moves.moved); + const auto buffers = take(moves.buffers); + const auto bytes = take(moves.bytes); + const auto rebuilt = take(moves.rebuilt); + const auto kept = take(moves.kept); + const auto served = take(moves.served); + unsigned long long sizes[5]; + for (std::size_t i = 0; i < 5; ++i) sizes[i] = take(moves.sizes[i]); + const auto refused = list(moves.refused); + const auto filtered = list(moves.filtered); + AgcDriver::ProfilePrint_nid_no_patch("[dispmove] (10 s) %llu template hits checked: %llu with every read-only buffer where it was built, %llu with %llu moved ones (%.2f MiB snapshotted; per dispatch up to 1 KiB %llu, 16 KiB %llu, 256 KiB %llu, 4 MiB %llu, more %llu); refused %s at stage A (the hit was dropped before the record), %s at record; after a refusal: %llu rebuilt, %llu inserted beside the movable template (kept), %llu served by an exact entry\n", checked, exact, moved, buffers, static_cast(bytes) / 1048576.0, sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], filtered.c_str(), refused.c_str(), rebuilt, kept, served); +} + +void noteDispatchMoveSize(std::size_t bytes) { + auto& moves = DispatchMoves(); + const std::size_t bucket = bytes <= 1024 ? 0 : bytes <= 16384 ? 1 : bytes <= 262144 ? 2 : bytes <= 4194304 ? 3 : 4; + moves.sizes[bucket].fetch_add(1, std::memory_order_relaxed); +} + void reportRecipes() { auto& recipes = Recipes(); const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); @@ -4106,12 +4177,23 @@ struct RecordedDispatch { // Whether the data refresh recorded anything. bool refreshed = false; bool resourceKeyMatched = false; + std::shared_ptr bindings{}; }; VkDevice VulkanDevice::Device() const { return state->device; } +VulkanDevice::DispatchMoveCount VulkanDevice::DispatchMoveCounts() { + auto& moves = DispatchMoves(); + DispatchMoveCount count{moves.checked.load(), moves.exact.load(), moves.moved.load(), 0, 0, moves.rebuilt.load(), moves.kept.load(), moves.served.load()}; + for (std::size_t i = 1; i < MoveRefusals; ++i) { + count.refused += moves.refused[i].load(); + count.filtered += moves.filtered[i].load(); + } + return count; +} + bool VulkanDevice::DispatchRecipes() { static const bool noDispatchRecipe = std::getenv("APS5_NO_DISPATCH_RECIPE") != nullptr; return !noDispatchRecipe; @@ -4251,6 +4333,15 @@ std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderReco prepared->key = DispatchContentKey(compute, context.device); phase(PreparedDispatch::PrepareKey); cached = state->resourceCache.Find(prepared->key); + if (cached != nullptr && DispatchMove()) { + Graphics::ShaderResources::MoveProof proof; + if (!cached->MovedDispatchBuffers(compute, nullptr, DispatchMoveBytes(), proof).has_value()) { + DispatchMoves().filtered[static_cast(proof.refusal)].fetch_add(1, std::memory_order_relaxed); + prepared->key = DispatchContentKey(compute, context.device, false); + prepared->exact = true; + cached = state->resourceCache.Find(prepared->key); + } + } prepared->cached = cached; phase(PreparedDispatch::PrepareFind); } @@ -4448,6 +4539,10 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { recorder.Keep(record.objects); recorder.Keep(record.resources); if (record.resources != nullptr) recorder.KeepBytes(record.resources.get(), record.resources->CopiedBytes()); + if (record.bindings != nullptr) { + recorder.Keep(record.bindings); + for (const auto& snapshot : record.bindings->snapshots) recorder.KeepBytes(snapshot.buffer.get(), snapshot.buffer->Bytes().size()); + } recordStep(PhaseRecordKeeps); if (record.dataRefresh != RecordedDispatch::DataRefresh::None) { // The template's data buffers take this dispatch's words: a transfer write the pre-dispatch @@ -4497,7 +4592,9 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { ++d.preBarriersRecorded; } context.Resolved(&Graphics::DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->pipeline); - resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout); + if (record.bindings != nullptr) Graphics::Require(record.bindings->allocation.set != VK_NULL_HANDLE, "dispatch overlay has no descriptor set"); + if (record.bindings != nullptr) context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, 0, 1, &record.bindings->allocation.set, 0, nullptr); + else resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout); if (const auto dwords = record.shader->program->workgroupMemoryDwords; dwords != 0) { auto memory = state->WorkgroupMemoryFor(context, record.x, record.y, record.z, dwords); context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, ShaderRecompiler::WorkgroupMemoryDescriptorSet, 1, &memory->set, 0, nullptr); @@ -4710,7 +4807,9 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec // The object came from the resource cache: a compute template whose data buffers take this // dispatch's words at the record (ShaderResources::RefreshData). bool fromCache = false; + bool refusedMove = false; double revalidateMs = 0; + std::shared_ptr overlay; // The batches the pre-sync (PrepareDispatch) waited for are reaped first: their completions // (CPU write-backs) land before stage B or a cached object's Revalidate reads, and the flush // hook then finds nothing pending over their surfaces. @@ -4730,6 +4829,11 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec // The image lookups of this call by outcome (stage B, a Revalidate or a locked build), for the // [indirect] line: reset here, read after the resources phase. if (profile && arguments != 0) Graphics::ThreadLookupOutcomes() = {}; + const auto noteRefusalBuild = [](bool inserted) { + auto& moves = DispatchMoves(); + moves.rebuilt.fetch_add(1, std::memory_order_relaxed); + if (inserted) moves.kept.fetch_add(1, std::memory_order_relaxed); + }; // The resource cache insert evicts the oldest entry, destroying its ShaderResources here when the // cache was the last owner: timed apart from the build. const auto insert = [&] { @@ -4778,6 +4882,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } if (cacheable) { ++d.cacheMisses; + if (prepared->exact) noteRefusalBuild(resources->Reusable()); if (resources->Reusable()) insert(); } } else if (cacheable) { @@ -4785,13 +4890,54 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec // object it found: revalidated as a map hit would be (an entry replaced meanwhile is left // alone by the pointer-conditional Remove). APS5_NO_PREPARED_FIND=1 looks the key up again. static const bool preparedFind = std::getenv("APS5_NO_PREPARED_FIND") == nullptr; + const bool exactEntry = prepared != nullptr && prepared->exact; contentKey = prepared != nullptr && !prepared->key.empty() ? std::move(prepared->key) : DispatchContentKey(shaders[0], context.device); auto cached = preparedFind && prepared != nullptr && prepared->cached != nullptr ? std::move(prepared->cached) : state->resourceCache.Find(contentKey); if (cached != nullptr) { if (cached->Revalidate(shaders[0])) { - resources = std::move(cached); - fromCache = true; - ++d.cacheHits; + bool usable = true; + if (DispatchMove() && !exactEntry) { + auto& moves = DispatchMoves(); + Graphics::ShaderResources::MoveProof proof; + auto moved = cached->MovedDispatchBuffers(shaders[0], state->recorder.get(), DispatchMoveBytes(), proof); + moves.checked.fetch_add(1, std::memory_order_relaxed); + if (moved.has_value() && !moved->empty()) { + overlay = cached->PrepareDrawBindings(*state->recorder, *moved, true); + if (overlay != nullptr && overlay->allocation.set == VK_NULL_HANDLE) overlay.reset(); + if (overlay == nullptr) { + proof.refusal = Graphics::ShaderResources::MoveRefusal::Bindings; + moved.reset(); + } + } + if (!moved.has_value()) { + usable = false; + moves.refused[static_cast(proof.refusal)].fetch_add(1, std::memory_order_relaxed); + } else if (moved->empty()) { + moves.exact.fetch_add(1, std::memory_order_relaxed); + } else { + moves.moved.fetch_add(1, std::memory_order_relaxed); + moves.buffers.fetch_add(proof.buffers, std::memory_order_relaxed); + moves.bytes.fetch_add(proof.bytes, std::memory_order_relaxed); + noteDispatchMoveSize(proof.bytes); + } + } + if (!usable) { + contentKey = DispatchContentKey(shaders[0], context.device, false); + refusedMove = true; + cached = state->resourceCache.Find(contentKey); + if (cached != nullptr && cached->Revalidate(shaders[0])) usable = true; + else if (cached != nullptr) { + state->resourceCache.Remove(contentKey, cached.get()); + ++d.cacheInvalidated; + state->recorder->Keep(std::move(cached)); + } + } + if (usable) { + if (refusedMove || exactEntry) DispatchMoves().served.fetch_add(1, std::memory_order_relaxed); + resources = std::move(cached); + fromCache = true; + ++d.cacheHits; + } } else { state->resourceCache.Remove(contentKey, cached.get()); ++d.cacheInvalidated; @@ -4810,6 +4956,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec if (profile) timer.add(PhaseResourcesFullBuild, std::chrono::duration(std::chrono::steady_clock::now() - buildStart).count()); if (cacheable) { ++d.cacheMisses; + if (refusedMove) noteRefusalBuild(resources->Reusable()); if (resources->Reusable()) insert(); } } @@ -4832,6 +4979,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec AgcDriver::ProfilePrint_nid_no_patch("[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast(d.cacheHits), static_cast(d.cacheMisses), static_cast(d.cacheInvalidated), state->resourceCache.Size(), static_cast(d.templateRefreshed), d.templateRevalidateMs, static_cast(d.templateSameWords), static_cast(ResourceCache::Finds()), static_cast(ResourceCache::Touches())); AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call: %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast(d.procLookups), d.profiledDispatches != 0 ? static_cast(d.procLookups) / static_cast(d.profiledDispatches) : 0.0, static_cast(d.preBarriersRecorded), static_cast(d.preBarriersSkipped), static_cast(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast(computePipelinesCreatedLocked.load(std::memory_order_relaxed))); reportRecipes(); + reportDispatchMove(); } timing.Mark("shader_resources"); timer.phase(PhaseResources); @@ -4853,6 +5001,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec IndirectOutcome outcome{0, 0}; RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer}; record.resourceKeyMatched = fromCache; + record.bindings = overlay; if (arguments != 0) decideIndirect(record, outcome, groupsText); // The indirect hold total below still covers the whole call of a CPU-resolved one. const bool indirect = record.arguments != 0 || outcome.cpuReason != 0; @@ -4906,7 +5055,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec // object the resource cache serves under this content key (reusable: no lease, no copied // writes, every direct region import- or mirror-served), so a hit's proof is the template's // Revalidate and nothing needs completion work. - if (recipeOut != nullptr && DispatchRecipes() && cacheable && resources->Reusable() && !resources->NeedsCompletion() && objects != nullptr && !contentKey.empty()) { + if (recipeOut != nullptr && DispatchRecipes() && cacheable && overlay == nullptr && resources->Reusable() && !resources->NeedsCompletion() && objects != nullptr && !contentKey.empty()) { auto recipe = std::make_shared(); recipe->device = state->device; recipe->templateRef = resources; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp index 2f54bee302..40e1076b79 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp @@ -26,6 +26,7 @@ void RequireNoNativeDepthFeedback_nid_no_patch(const Context& context, const Dep void ClearDepthSurfaces(VkDevice device); void RetireDepthSurfaces(VkDevice device, std::uint64_t address, std::uint64_t bytes); bool DepthSurfaceAt(std::uint64_t address); +bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t bytes); bool DepthSurfaceAt(const Context& context, std::span words, const GuestTextureResource& resource); std::uint64_t HtileDepthClearAddress(std::span code, std::span userData, const std::array& numThreads); std::size_t HtileDepthClearBytes_nid_no_patch(std::span code, std::span userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, std::span packet, std::uint64_t indirectArguments); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index c57a2cea69..b64df636f7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -178,8 +178,16 @@ public: std::size_t size; std::vector words; }; - std::shared_ptr PrepareDrawBindings(Recorder& recorder, std::span moved = {}) const; + std::shared_ptr PrepareDrawBindings(Recorder& recorder, std::span moved = {}, bool movedOnly = false) const; std::optional> MovedReadOnlyBuffers(std::span shaders, Recorder& recorder) const; + enum class MoveRefusal : std::size_t { None, Shape, NotGuest, Written, Limit, Inaccessible, Overlap, Bindings, Count }; + struct MoveProof { + MoveRefusal refusal = MoveRefusal::None; + std::size_t buffers = 0; + std::size_t bytes = 0; + }; + static const char* MoveRefusalName(MoveRefusal refusal); + std::optional> MovedDispatchBuffers(const CompiledShader& shader, Recorder* recorder, std::size_t maxBytes, MoveProof& proof) const; static std::size_t MaxDrawSnapshotBytes(); void WriteBack(); // Deferred completion: MarkGpuWrites registers the results the recorded work leaves on the GPU @@ -237,6 +245,7 @@ public: bool Reusable() const { return reusable; } const std::vector& MultisampleSources() const { return multisampleSources; } static bool NeverReusable(std::span shaders); + static std::uint64_t DepthGuardRefusals(); const std::vector>& SampledTextures() const { return textures; } bool ChecksFaults() const { return bda != nullptr; } void CheckFaults() const; @@ -389,6 +398,7 @@ private: // same way but is left out of the write set (no write-back, no pending-write note). `atomic` // is its bufferAtomic (see GuestBufferMemory::AddWritable). std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read); + std::uint64_t guestBufferBytes(std::span words) const; std::size_t addDataBuffer(std::span words); // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache // locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding). diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index a2e45f7940..4f506c934a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -1576,6 +1576,11 @@ bool DepthSurfaceAt(std::uint64_t address) { return std::any_of(surfaces().begin(), surfaces().end(), [&](const auto& surface) { return surface->active->load(std::memory_order_acquire) && !surface->retired && (surface->target.address == address || surface->target.stencilAddress == address); }); } +bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t bytes) { + std::lock_guard lock(surfacesMutex()); + return std::any_of(surfaces().begin(), surfaces().end(), [&](const auto& surface) { return surface->active->load(std::memory_order_acquire) && !surface->retired && surface->context.device == device && surface->Overlaps(address, bytes); }); +} + void RetireDepthSurfaces(VkDevice device, std::uint64_t address, std::uint64_t bytes) { std::lock_guard lock(surfacesMutex()); for (const auto& surface : surfaces()) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index b8413ba34b..01adf45e3c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -2217,6 +2217,7 @@ void mixDataWords(std::uint64_t& hash, std::span words) { // ResourceCache::Find and Touch calls (the [rescache] line). std::atomic resourceCacheFinds{0}; std::atomic resourceCacheTouches{0}; +std::atomic depthGuardRefusals{0}; } @@ -2259,6 +2260,10 @@ void ShaderResources::noteReusable() { reusable = true; } +std::uint64_t ShaderResources::DepthGuardRefusals() { + return depthGuardRefusals.load(std::memory_order_relaxed); +} + bool ShaderResources::NeverReusable(std::span shaders) { const bool loopGuard = LoopGuardEnabled(); return std::any_of(shaders.begin(), shaders.end(), [loopGuard](const CompiledShader& shader) { @@ -2729,6 +2734,10 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::spanDescriptor(); const auto address = own.baseAddress; const auto bytes = static_cast(image->GuestBytes()); + if (DepthSurfaceOverlaps(context.device, address, bytes)) { + depthGuardRefusals.fetch_add(1, std::memory_order_relaxed); + return fail(FastFail::Depth); + } if (GuestMemory::CollectWrites(address, bytes) == 0) return fail(FastFail::Collect); if (own.dccAddress != 0) { // Refresh's unchanged branch (its key compare against the keys the content was @@ -2832,6 +2841,10 @@ bool ShaderResources::fastRevalidateEach() { if (own.dccAddress != 0) return false; const auto address = own.baseAddress; const auto bytes = static_cast(image->GuestBytes()); + if (DepthSurfaceOverlaps(context.device, address, bytes)) { + depthGuardRefusals.fetch_add(1, std::memory_order_relaxed); + return false; + } if (GuestMemory::CollectWrites(address, bytes) == 0) return false; if (PendingStorageOverlaps(address, bytes, image)) return false; if (!StorageImageCached(context, image)) return false; @@ -3699,11 +3712,8 @@ DescriptorCache::Stats DescriptorCache::Counters() const { return stats; } -std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) { - Require(words.size() == 4, "buffer descriptor must contain four DWORDs"); - Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set"); +std::uint64_t ShaderResources::guestBufferBytes(std::span words) const { const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; - Require(descriptor.Type() == 0u, "buffer descriptor uses an unsupported type"); const auto address = descriptor.Base48(); auto byteSize = descriptor.GetSize(); if (byteSize > context.limits.maxStorageBufferRange) { @@ -3711,22 +3721,33 @@ std::size_t ShaderResources::addGuestBuffer(std::span words if (mappedEnd <= address) byteSize = 0; else if (mappedEnd - address < context.limits.maxStorageBufferRange) byteSize = mappedEnd - address; } + if (byteSize == 0 || address == 0) return 0; + if (const std::uint64_t stride = descriptor.Stride(); stride != 0 && byteSize <= context.limits.maxStorageBufferRange) { + constexpr std::uint64_t GuestPageBytes = 0x4000; + const auto end = address + byteSize; + const auto overhang = end % GuestPageBytes; + if (overhang != 0 && overhang < stride) { + const auto allocationEnd = RegisteredReadableEnd(address); + if (allocationEnd > address && allocationEnd < end && end - allocationEnd < stride) byteSize = allocationEnd - address; + } + } + return byteSize; +} + +std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) { + Require(words.size() == 4, "buffer descriptor must contain four DWORDs"); + Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set"); + const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; + Require(descriptor.Type() == 0u, "buffer descriptor uses an unsupported type"); + const auto address = descriptor.Base48(); + const auto byteSize = guestBufferBytes(words); if (byteSize == 0 || address == 0) { allocations.push_back({0, EmptyBufferBytes, false, nullptr, ShaderRecompiler::DescriptorRole::GuestBuffers, false}); return allocations.size() - 1; } Require(byteSize <= context.limits.maxStorageBufferRange, "shader buffer exceeds descriptor range limit"); Require(byteSize <= std::numeric_limits::max(), "shader buffer size exceeds host address space"); - auto size = static_cast(byteSize); - if (const std::uint64_t stride = descriptor.Stride(); stride != 0) { - constexpr std::uint64_t GuestPageBytes = 0x4000; - const auto end = address + size; - const auto overhang = end % GuestPageBytes; - if (overhang != 0 && overhang < stride) { - const auto allocationEnd = RegisteredReadableEnd(address); - if (allocationEnd > address && allocationEnd < end && end - allocationEnd < stride) size = static_cast(allocationEnd - address); - } - } + const auto size = static_cast(byteSize); Require(target == nullptr || !overlap(address, size, target->address, target->bytes), "shader buffer aliases the render target"); // APS5_ALL_BUFFERS_WRITTEN=1: every element is noted as written, as before bufferWritten existed. static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr; @@ -4732,13 +4753,71 @@ std::optional> ShaderResources::MovedR return moved; } +const char* ShaderResources::MoveRefusalName(MoveRefusal refusal) { + switch (refusal) { + case MoveRefusal::None: return "none"; + case MoveRefusal::Shape: return "bindings differ"; + case MoveRefusal::NotGuest: return "not read in place"; + case MoveRefusal::Written: return "written or emptied"; + case MoveRefusal::Limit: return "over the limit"; + case MoveRefusal::Inaccessible: return "unmapped"; + case MoveRefusal::Overlap: return "pending write"; + case MoveRefusal::Bindings: return "no bindings made"; + case MoveRefusal::Count: break; + } + return "?"; +} + +std::optional> ShaderResources::MovedDispatchBuffers(const CompiledShader& shader, Recorder* recorder, std::size_t maxBytes, MoveProof& proof) const { + proof = {}; + const auto refuse = [&](MoveRefusal why) { + proof.refusal = why; + return std::optional>{}; + }; + std::vector moved; + if (shader.program == nullptr || usesBda) return refuse(MoveRefusal::Shape); + if (_set == VK_NULL_HANDLE) return moved; + for (const auto& binding : shader.program->bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; + const auto kept = std::find_if(bindings.begin(), bindings.end(), [&](const Binding& item) { return item.layout.binding == binding.binding; }); + if (kept == bindings.end() || kept->allocations.size() != binding.count || binding.guestDescriptor.size() != static_cast(binding.count) * 4u) return refuse(MoveRefusal::Shape); + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto* words = binding.guestDescriptor.data() + static_cast(element) * 4u; + const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; + const auto address = descriptor.Base48(); + const auto size = guestBufferBytes(std::span(words, 4)); + const auto& item = allocations[kept->allocations[element]]; + const bool empty = size == 0 || address == 0; + if (!item.guest) { + if (empty) continue; + return refuse(MoveRefusal::NotGuest); + } + if (item.address == address && item.size == size) continue; + const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; + if (written || empty || item.written) return refuse(MoveRefusal::Written); + if (size > context.limits.maxStorageBufferRange) return refuse(MoveRefusal::Limit); + const auto begin = address - item.adjustment; + const auto bytes = static_cast(GuestBufferMemory::ViewBytes(size, item.adjustment)); + if (maxBytes != 0 && proof.bytes + bytes > maxBytes) return refuse(MoveRefusal::Limit); + if (!GuestMemory::Accessible(reinterpret_cast(begin), bytes)) return refuse(MoveRefusal::Inaccessible); + const bool pending = recorder != nullptr ? recorder->PendingWriteOverlaps(begin, bytes) : Recorder::SnapshotWriteOverlaps(begin, bytes); + const std::array, 1> range{{{begin, begin + bytes}}}; + if (guestMemory.WritesOverlap(begin, bytes) || pending || StorageTexture::AnyPendingOverlaps(range) || AnyShadowedOverlaps(begin, bytes)) return refuse(MoveRefusal::Overlap); + moved.push_back({kept->allocations[element], address, static_cast(size)}); + ++proof.buffers; + proof.bytes += bytes; + } + } + return moved; +} + std::size_t ShaderResources::MaxDrawSnapshotBytes() { const char* text = std::getenv("APS5_DRAW_SNAPSHOT_MAX_MIB"); const auto mib = text != nullptr ? std::strtoull(text, nullptr, 10) : 64ull; return static_cast(mib * 1024ull * 1024ull); } -std::shared_ptr ShaderResources::PrepareDrawBindings(Recorder& recorder, std::span moved) const { +std::shared_ptr ShaderResources::PrepareDrawBindings(Recorder& recorder, std::span moved, bool movedOnly) const { if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE) || usesBda) return {}; const auto reads = guestMemory.InPlaceReads(); const auto maxSnapshotBytes = maxDrawSnapshotBytes; @@ -4778,7 +4857,7 @@ std::shared_ptr ShaderResources::PrepareDrawBindi address = override->address; size = override->size; } else { - if (!item.guest || item.written || guestMemory.WritesOverlap(item.address, item.size)) continue; + if (movedOnly || !item.guest || item.written || guestMemory.WritesOverlap(item.address, item.size)) continue; const bool direct = std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return item.address >= range.first && item.address < range.second && item.size <= range.second - item.address; }); if (!direct || recorder.PendingWriteOverlaps(item.address, item.size)) continue; } diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index 2b25b7b2a2..3c750efbbb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -4555,6 +4555,152 @@ void descriptorSnapshotTests() { } } +alignas(16) std::array guestMovedFirst{0x51515151, 0x52525252, 0x53535353, 0x54545454, 0x55555555, 0x56565656, 0x57575757, 0x58585858}; +alignas(16) std::array guestMovedSecond{0x61616161, 0x62626262, 0x63636363, 0x64646464, 0x65656565, 0x66666666, 0x67676767, 0x68686868}; + +void dispatchMoveTests() { + using AgcDriver::Graphics::CompiledShader; + using AgcDriver::Graphics::DescriptorCache; + using AgcDriver::Graphics::Recorder; + using AgcDriver::Graphics::ShaderResources; + using Refusal = ShaderResources::MoveRefusal; + mock = MockVulkan{}; + mock.allowCopies = true; + auto context = mockContext(); + { + DescriptorCache cache(context); + context.descriptorCache = &cache; + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + Recorder recorder(context); + ShaderRecompiler::RecompileResult built; + auto inputs = makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32))); + inputs.bufferWritten = {false, false}; + built.bindings.push_back(inputs); + auto output = makeBinding(Role::GuestBuffers, 4, 1, vsharp(guestThird.data(), 8)); + output.bufferWritten = {true}; + built.bindings.push_back(output); + const CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &built, 0}; + ShaderResources resources(context, shader); + const auto moveTo = [&](std::size_t element, const void* where, std::uint32_t bytes) { + auto program = built; + const auto words = vsharp(where, bytes); + std::copy(words.begin(), words.end(), program.bindings[0].guestDescriptor.begin() + static_cast(element) * 4); + return program; + }; + const auto prove = [&](const ShaderRecompiler::RecompileResult& program, std::size_t maxBytes, ShaderResources::MoveProof& proof) { + return resources.MovedDispatchBuffers(CompiledShader{ShaderRecompiler::ShaderStage::Compute, &program, 0}, &recorder, maxBytes, proof); + }; + const auto refuses = [&](const ShaderRecompiler::RecompileResult& program, std::size_t maxBytes, Refusal why, const char* what) { + ShaderResources::MoveProof refusal; + const auto listed = prove(program, maxBytes, refusal); + Require(!listed.has_value() && refusal.refusal == why, what); + }; + + ShaderResources::MoveProof proof; + auto moved = prove(built, 0, proof); + Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None && proof.buffers == 0, "a dispatch with every buffer where it was built moved something"); + + const auto relocated = moveTo(1, guestMovedSecond.data(), 24); + const CompiledShader relocatedShader{ShaderRecompiler::ShaderStage::Compute, &relocated, 0}; + Require(ShaderResources::ContentKey(shader, false, true) == ShaderResources::ContentKey(relocatedShader, false, true), "moving a read-only buffer changed the movable content key"); + Require(ShaderResources::ContentKey(shader, false, false) != ShaderResources::ContentKey(relocatedShader, false, false), "the fixed content key ignored a moved buffer"); + + moved = prove(relocated, 0, proof); + Require(moved.has_value() && moved->size() == 1 && proof.refusal == Refusal::None && proof.buffers == 1 && proof.bytes >= 24, "a moved read-only buffer was not listed"); + Require(moved->front().address == reinterpret_cast(guestMovedSecond.data()) && moved->front().size == 24 && moved->front().words.empty(), "the listed buffer is not the moved range"); + + moved = prove(moveTo(0, guestFirst.data(), 8), 0, proof); + Require(moved.has_value() && moved->size() == 1 && moved->front().address == reinterpret_cast(guestFirst.data()) && moved->front().size == 8, "a buffer that only changed its size was not listed"); + + auto both = relocated; + const auto firstWords = vsharp(guestMovedFirst.data(), 12); + std::copy(firstWords.begin(), firstWords.end(), both.bindings[0].guestDescriptor.begin()); + moved = prove(both, 1024, proof); + Require(moved.has_value() && moved->size() == 2 && proof.buffers == 2 && proof.bytes >= 36, "two moved buffers were not both listed"); + refuses(both, 30, Refusal::Limit, "moved buffers past the byte budget together were served"); + + auto written = relocated; + written.bindings[0].bufferWritten[1] = true; + refuses(written, 0, Refusal::Written, "a moved buffer the dispatch writes was served"); + refuses(moveTo(1, guestMovedSecond.data(), 20000), 0, Refusal::Written, "a moved buffer over the storage buffer range that no registered allocation holds was served"); + refuses(relocated, 16, Refusal::Limit, "a moved buffer over the byte budget was served"); + refuses(moveTo(1, reinterpret_cast(std::uintptr_t{0x10}), 24), 0, Refusal::Inaccessible, "a moved buffer over unmapped memory was served"); + refuses(moveTo(1, guestThird.data(), 8), 0, Refusal::Overlap, "a moved buffer over the dispatch's own written range was served"); + auto shaped = relocated; + shaped.bindings[0].guestDescriptor.resize(4); + refuses(shaped, 0, Refusal::Shape, "a dispatch with another binding shape was served"); + recorder.Sync(); + } + { + constexpr std::size_t blockBytes = 65536; + void* block = ::operator new(blockBytes, std::align_val_t{blockBytes}); + std::memset(block, 0x5a, blockBytes); + { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, blockBytes, true, true, true); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + } + const auto* end = static_cast(block) + blockBytes; + const auto strided = [](const void* pointer, std::uint32_t stride, std::uint32_t records) { + auto words = vsharp(pointer, records); + words[1] |= stride << 16u; + return words; + }; + DescriptorCache cache(context); + context.descriptorCache = &cache; + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + Recorder recorder(context); + const auto proveAgainst = [&](const ShaderRecompiler::RecompileResult& built, const std::vector& incoming, ShaderResources::MoveProof& proof, std::optional>& moved) { + const CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &built, 0}; + ShaderResources resources(context, shader); + auto program = built; + program.bindings[0].guestDescriptor = incoming; + moved = resources.MovedDispatchBuffers(CompiledShader{ShaderRecompiler::ShaderStage::Compute, &program, 0}, &recorder, 0, proof); + }; + const auto readOnly = [&](const std::vector& words) { + ShaderRecompiler::RecompileResult built; + auto binding = makeBinding(Role::GuestBuffers, 0, 1, words); + binding.bufferWritten = {false}; + built.bindings.push_back(binding); + return built; + }; + const auto address = [](const void* pointer) { return reinterpret_cast(pointer); }; + ShaderResources::MoveProof proof; + std::optional> moved; + + const auto* tail = end - 8192; + const auto* nearEnd = end - 4096; + const auto oversized = readOnly(vsharp(tail, 0xffffffffu)); + proveAgainst(oversized, vsharp(tail, 0xffffffffu), proof, moved); + Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None && proof.buffers == 0, "an unchanged V# the build cut to its allocation end was not an exact hit"); + proveAgainst(oversized, vsharp(nearEnd, 0xffffffffu), proof, moved); + Require(moved.has_value() && moved->size() == 1 && proof.buffers == 1 && proof.bytes == 4096 && moved->front().address == address(nearEnd) && moved->front().size == 4096, "a moved V# past the range limit was not snapshotted at its clamped range"); + proveAgainst(oversized, vsharp(nearEnd, 0xfffffff0u), proof, moved); + Require(moved.has_value() && moved->size() == 1 && moved->front().size == 4096, "another oversized size over the same allocation end listed another range"); + proveAgainst(oversized, vsharp(tail, 4096), proof, moved); + Require(moved.has_value() && moved->size() == 1 && moved->front().address == address(tail) && moved->front().size == 4096, "a V# that now fits the range limit was not listed at its own size"); + + proveAgainst(oversized, vsharp(block, 0xffffffffu), proof, moved); + Require(!moved.has_value() && proof.refusal == Refusal::Limit, "a moved V# the allocation holds past the range limit was served"); + + const auto overhang = readOnly(strided(tail, 24, 342)); + proveAgainst(overhang, strided(tail, 24, 342), proof, moved); + Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None, "an unchanged V# with a stride overhang past the allocation was not an exact hit"); + proveAgainst(overhang, strided(nearEnd, 24, 171), proof, moved); + Require(moved.has_value() && moved->size() == 1 && moved->front().address == address(nearEnd) && moved->front().size == 4096 && proof.bytes == 4096, "a moved V# with a stride overhang was snapshotted past the allocation end"); + proveAgainst(overhang, strided(nearEnd, 24, 100), proof, moved); + Require(moved.has_value() && moved->size() == 1 && moved->front().size == 2400, "a moved V# inside the allocation was not listed at its own size"); + recorder.Sync(); + { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + } + ::operator delete(block, std::align_val_t{blockBytes}); + } + Require(mock.live == 0, "dispatch move resources leaked Vulkan objects"); +} + void resourceTests() { mock = MockVulkan{}; const auto context = mockContext(); @@ -8397,6 +8543,7 @@ int main(int argc, char** argv) { oversizedDescriptorTests(); guestBufferViewBytesTests(); descriptorSnapshotTests(); + dispatchMoveTests(); descriptorCacheTests(); bufferBudgetPlacementTests(); imageTableAllocationTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DepthFastClearHarness.hpp b/core/libs/prx/libSceAgcDriver/tests/execution/DepthFastClearHarness.hpp index 3e21bfbceb..f87d9ce6b3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DepthFastClearHarness.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DepthFastClearHarness.hpp @@ -48,6 +48,8 @@ struct DrawOptions { VkCompareOp depthCompare = VK_COMPARE_OP_LESS; bool stencilTest = false; VkStencilOpState stencil{}; + float minDepth = 0.0f; + float maxDepth = 1.0f; }; inline std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { @@ -126,7 +128,7 @@ inline std::uint32_t Draw(AgcDriver::VulkanDevice& device, const Shaders& compil state.stencilBack = options.stencil; state.renderExtent = {Width, Height}; state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; - state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), options.minDepth, options.maxDepth}; state.negativeOneToOne = false; state.scissor = {{0, 0}, {Width, Height}}; state.cullMode = VK_CULL_MODE_NONE; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DispatchMove.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DispatchMove.cpp new file mode 100644 index 0000000000..770cd6ad19 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DispatchMove.cpp @@ -0,0 +1,285 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Words = 64; +constexpr std::uint32_t Bytes = Words * 4u; +constexpr std::uint32_t Constant = 0x00010000u; +constexpr std::uint32_t Sentinel = 0xdeadbeefu; +constexpr std::size_t Piece = 65536; +constexpr std::size_t BlockBytes = 5 * Piece; + +alignas(256) std::array Table{}; + +constexpr std::uint32_t Vop2(std::uint32_t opcode, std::uint32_t destination, std::uint32_t source0, std::uint32_t source1) { + return (opcode << 25u) | (destination << 17u) | (source1 << 9u) | source0; +} + +constexpr std::uint32_t BufferOperands = (0x80u << 24u) | (1u << 16u) | (2u << 8u) | 1u; + +alignas(256) constexpr std::array Code{ + Vop2(0x1au, 1u, 0x82u, 0u), + 0xf4080100u, 0xfa000000u, + 0xbf8cc07fu, + 0xe0301000u, BufferOperands, + 0xbf8c3f70u, + 0xf4080100u, 0xfa000010u, + 0xbf8cc07fu, + Vop2(0x25u, 2u, 0xffu, 2u), Constant, + 0xe0701000u, BufferOperands, + 0xbf810000u, +}; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, Piece)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); +#else + constexpr std::uintptr_t alignment = Piece; + void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast(aligned + BlockBytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes); + } +#endif + Require(block != nullptr, "dispatch move: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes); + GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes); +#else + munmap(block, BlockBytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint32_t* InputA() const { return reinterpret_cast(block); } + std::uint32_t* InputB() const { return reinterpret_cast(block + Piece + 12); } + std::uint32_t* InputC() const { return reinterpret_cast(block + 2 * Piece + 32); } + std::uint32_t* OutputO() const { return reinterpret_cast(block + 3 * Piece); } + std::uint32_t* OutputP() const { return reinterpret_cast(block + 4 * Piece); } + +private: + std::uint8_t* block = nullptr; +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Fill(std::uint32_t* where, std::uint32_t first) { + for (std::uint32_t index = 0; index < Words; ++index) where[index] = first + index; +} + +void Reset(std::uint32_t* output) { + std::fill(output, output + Words, Sentinel); +} + +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, const std::uint32_t* input, std::uint32_t inputBytes, const std::uint32_t* output) { + const auto in = BufferDescriptor(input, inputBytes); + const auto out = BufferDescriptor(output, Bytes); + std::copy(in.begin(), in.end(), Table.begin()); + std::copy(out.begin(), out.end(), Table.begin() + 4); + const auto table = reinterpret_cast(Table.data()); + const std::vector userData{static_cast(table), static_cast(table >> 32u)}; + const std::span code(Code); + const std::span tableWords(Table); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}, {table, std::as_bytes(tableWords)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +void Run(AgcDriver::VulkanDevice& device, const ShaderRecompiler::RecompileResult& compiled, const std::uint32_t* input, std::uint32_t inputBytes, const std::uint32_t* output, bool wait) { + auto result = compiled; + std::size_t elements = 0; + for (auto& binding : result.bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; + Require(binding.guestDescriptor.size() == static_cast(binding.count) * 4u && binding.bufferWritten.size() == binding.count, "dispatch move: the program's buffer bindings have another shape"); + for (std::uint32_t element = 0; element < binding.count; ++element, ++elements) { + const auto words = binding.bufferWritten[element] ? BufferDescriptor(output, Bytes) : BufferDescriptor(input, inputBytes); + std::copy(words.begin(), words.end(), binding.guestDescriptor.begin() + static_cast(element) * 4); + } + } + Require(elements == 2, "dispatch move: the program does not bind an input and an output"); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(Code.data())); + if (wait) device.WaitIdle(); +} + +void Expect(const std::uint32_t* input, const std::uint32_t* output, std::uint32_t added, const std::string& what) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto expected = input[tid] + added; + Require(output[tid] == expected, "dispatch move: " + what + ": lane " + std::to_string(tid) + " is " + Hex(output[tid]) + ", expected " + Hex(expected)); + } +} + +struct Step { + std::uint64_t checked, exact, moved, refused, rebuilt, kept, served; +}; + +Step Since(const AgcDriver::VulkanDevice::DispatchMoveCount& before) { + const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts(); + return {now.checked - before.checked, now.exact - before.exact, now.moved - before.moved, now.refused - before.refused, now.rebuilt - before.rebuilt, now.kept - before.kept, now.served - before.served}; +} + +void Print(const char* what, const Step& step) { + std::printf("dispatch move: %s: checked %llu, exact %llu, moved %llu, refused %llu\n", what, static_cast(step.checked), static_cast(step.exact), static_cast(step.moved), static_cast(step.refused)); +} + +bool RunTests(AgcDriver::VulkanDevice& device, const GuestBlock& guest) { + Fill(guest.InputA(), 0xa000u); + Fill(guest.InputB(), 0xb000u); + Fill(guest.InputC(), 0xc000u); + + const auto compiled = Compile(device, guest.InputA(), Bytes, guest.OutputO()); + Reset(guest.OutputO()); + Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true); + Expect(guest.InputA(), guest.OutputO(), Constant, "the first dispatch"); + const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts(); + + Reset(guest.OutputO()); + Run(device, compiled, guest.InputB(), Bytes, guest.OutputO(), true); + Expect(guest.InputB(), guest.OutputO(), Constant, "an input moved to another block"); + Reset(guest.OutputO()); + Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true); + Expect(guest.InputA(), guest.OutputO(), Constant, "the input back where the template was built"); + Reset(guest.OutputO()); + Run(device, compiled, guest.InputC(), Bytes, guest.OutputO(), true); + Expect(guest.InputC(), guest.OutputO(), Constant, "an input moved to a third block"); + const auto repeats = Since(start); + Print("an exact repeat between two moved ones", repeats); + const bool templates = repeats.checked != 0; + if (templates) { + Require(repeats.exact == 1 && repeats.moved == 2 && repeats.refused == 0, "dispatch move: the repeats were not served as one exact hit and two moved ones"); + } else { + std::puts("dispatch move: the device kept no template (no host imports); the moved-buffer path is not exercised"); + } + + Reset(guest.OutputO()); + Run(device, compiled, guest.InputB(), 192, guest.OutputO(), true); + Expect(guest.InputB(), guest.OutputO(), Constant, "an input moved and shortened"); + + std::array changed{}; + for (std::uint32_t index = 0; index < Words; ++index) changed[index] = 0xd000u + index * 3u; + AgcDriver::GuestMemory::Write(reinterpret_cast(guest.InputB()), std::as_bytes(std::span(changed))); + Reset(guest.OutputO()); + Run(device, compiled, guest.InputB(), Bytes, guest.OutputO(), true); + Expect(changed.data(), guest.OutputO(), Constant, "an input moved to a range the driver stored over"); + + const auto refusals = AgcDriver::VulkanDevice::DispatchMoveCounts(); + for (std::uint32_t index = 0; index < Words; ++index) guest.OutputO()[index] = 0x7000u + index; + Run(device, compiled, guest.OutputO() + Threads, Bytes / 2, guest.OutputO(), true); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto expected = 0x7000u + Threads + tid + Constant; + Require(guest.OutputO()[tid] == expected, "dispatch move: an input moved into the unwritten half of the dispatch's own output: lane " + std::to_string(tid) + " is " + Hex(guest.OutputO()[tid]) + ", expected " + Hex(expected)); + } + + Reset(guest.OutputO()); + Reset(guest.OutputP()); + Run(device, compiled, guest.InputA(), Bytes, guest.OutputP(), false); + Run(device, compiled, guest.OutputP(), Bytes, guest.OutputO(), true); + Expect(guest.InputA(), guest.OutputO(), 2u * Constant, "an input moved onto a range a recorded dispatch still writes"); + const auto refused = Since(refusals); + Print("two inputs moved onto written ranges", refused); + if (templates) { + Require(refused.refused == 2, "dispatch move: the inputs moved onto written ranges were not both refused"); + Require(refused.rebuilt == 2 && refused.kept == 2 && refused.served == 0, "dispatch move: the two refusals were not two rebuilds inserted beside the template"); + } + + const auto afterRefusals = AgcDriver::VulkanDevice::DispatchMoveCounts(); + Reset(guest.OutputO()); + Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true); + Expect(guest.InputA(), guest.OutputO(), Constant, "the input moved back after the refusals"); + const auto back = Since(afterRefusals); + Print("the input back after the refusals", back); + if (templates) Require(back.checked == 1 && back.exact == 1 && back.moved == 0 && back.refused == 0 && back.rebuilt == 0, "dispatch move: the input back at the template's address was not an exact hit after the refusals"); + + for (int round = 0; round < 2; ++round) { + const auto alternating = AgcDriver::VulkanDevice::DispatchMoveCounts(); + Reset(guest.OutputO()); + Reset(guest.OutputP()); + Run(device, compiled, guest.InputA(), Bytes, guest.OutputP(), false); + Run(device, compiled, guest.OutputP(), Bytes, guest.OutputO(), true); + Expect(guest.InputA(), guest.OutputO(), 2u * Constant, "an input refused again while A alternates"); + Reset(guest.OutputO()); + Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true); + Expect(guest.InputA(), guest.OutputO(), Constant, "the input at A after the refused one"); + const auto step = Since(alternating); + Print("A alternating with a refused input", step); + if (templates) Require(step.refused == 1 && step.served == 1 && step.rebuilt == 0 && step.exact == 2 && step.moved == 0, "dispatch move: a refused input rebuilt, or evicted the template of A, while A alternated with it"); + } + const auto total = AgcDriver::VulkanDevice::DispatchMoveCounts(); + std::printf("dispatch move: in total: checked %llu, exact %llu, moved %llu, refused %llu, dropped by stage A %llu\n", static_cast(total.checked), static_cast(total.exact), static_cast(total.moved), static_cast(total.refused), static_cast(total.filtered)); + return templates; +} + +} + +int main() { + try { + GuestBlock guest; + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!RunTests(*device, guest)) return VulkanTestSkipped; + std::puts("dispatch move tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DispatchMoveDepth.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DispatchMoveDepth.cpp new file mode 100644 index 0000000000..1d44a9207d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DispatchMoveDepth.cpp @@ -0,0 +1,460 @@ +#include "DepthFastClearHarness.hpp" +#include "VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#ifndef _WIN32 +namespace AgcDriver { + +struct VulkanDeviceTestAccess { + static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); } +}; + +} +#endif + +namespace { + +using AgcDriver::Graphics::Require; +using DepthFastClearHarness::Covered; +using DepthFastClearHarness::Draw; +using DepthFastClearHarness::Height; +using DepthFastClearHarness::Shaders; +using DepthFastClearHarness::Surface; +using DepthFastClearHarness::Width; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Bytes = Threads * 4u; +constexpr std::uint32_t Format32Float = 22; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Sentinel = 0xdeadbeefu; +constexpr std::size_t Piece = 65536; + +alignas(256) constexpr std::array RowCode{ + 0x7e3c0300u, + 0x7e3e0280u, + 0x34060082u, + 0xe0301000u, 0x80040b03u, + 0xbf8c3f70u, + 0x4a181681u, + 0xe0701000u, 0x80000c03u, + 0xf0201108u, 0x00010b1eu, + 0xbf810000u, 0xbf810000u, +}; + +alignas(256) constexpr std::array ProbeCode{ + 0x7e3c0300u, + 0x7e3e020cu, + 0xf0001108u, 0x00010a1eu, + 0x34060082u, + 0xbf8c3f70u, + 0xe0701000u, 0x80000a03u, + 0xbf810000u, 0xbf810000u, +}; + +class GuestBlock { +public: + GuestBlock(std::size_t pieces, bool inWatch) : bytes(pieces * Piece), watched(inWatch) { + if (inWatch) { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, Piece)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); +#else + constexpr std::uintptr_t alignment = Piece; + void* mapped = mmap(nullptr, bytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + bytes != begin + bytes + alignment) munmap(reinterpret_cast(aligned + bytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes); + } +#endif + } else { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(Piece, bytes)); +#endif + if (block != nullptr) std::memset(block, 0, bytes); + } + Require(block != nullptr, "dispatch move depth: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, bytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); + if (watched) { +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, bytes); + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); +#else + munmap(block, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes); +#endif + } else { +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* PieceAt(std::size_t index) const { return block + index * Piece; } + +private: + std::size_t bytes = 0; + bool watched = false; + std::uint8_t* block = nullptr; +}; + +struct Buffers { + explicit Buffers(const GuestBlock& block) : block(block) {} + std::uint32_t* InputA() const { return reinterpret_cast(block.PieceAt(4)); } + std::uint32_t* InputB() const { return reinterpret_cast(block.PieceAt(5) + 12); } + std::uint32_t* InputC() const { return reinterpret_cast(block.PieceAt(6) + 32); } + std::uint32_t* Output() const { return reinterpret_cast(block.PieceAt(8)); } + +private: + const GuestBlock& block; +}; + +struct Env { + AgcDriver::VulkanDevice& device; + const Shaders& shaders; + const Buffers& buffers; +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array TextureDescriptor(const void* data) { + const auto address = static_cast(reinterpret_cast(data)); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32Float << 20u) | (((Width - 1u) & 3u) << 30u), + ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), + 0xfacu | (Type2D << 28u), + 0u, 0u, 0u, 0u, + }; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span code, std::span userData) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +ShaderRecompiler::RecompileResult RowProgram(AgcDriver::VulkanDevice& device, const void* plane, const std::uint32_t* input, const std::uint32_t* output) { + std::vector userData(20, 0u); + const auto out = BufferDescriptor(output, Bytes); + const auto texture = TextureDescriptor(plane); + const auto in = BufferDescriptor(input, Bytes); + std::copy(out.begin(), out.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + std::copy(in.begin(), in.end(), userData.begin() + 16); + return Compile(device, RowCode, userData); +} + +void RunRow(AgcDriver::VulkanDevice& device, const ShaderRecompiler::RecompileResult& compiled, const std::uint32_t* input, const std::uint32_t* output) { + auto result = compiled; + std::size_t elements = 0; + for (auto& binding : result.bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; + Require(binding.guestDescriptor.size() == static_cast(binding.count) * 4u, "dispatch move depth: the program's buffer bindings have another shape"); + for (std::uint32_t element = 0; element < binding.count; ++element, ++elements) { + auto* current = binding.guestDescriptor.data() + static_cast(element) * 4u; + const auto built = static_cast(current[0]) | (static_cast(current[1] & 0xffffu) << 32u); + const auto words = built == static_cast(reinterpret_cast(output)) ? BufferDescriptor(output, Bytes) : BufferDescriptor(input, Bytes); + std::copy(words.begin(), words.end(), current); + } + } + Require(elements == 2, "dispatch move depth: the program does not bind an input and an output"); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(RowCode.data())); + device.WaitIdle(); +} + +std::array ReadRow(const Env& env, const void* plane, std::uint32_t y) { + std::vector userData(16, 0u); + auto* output = env.buffers.Output(); + const auto out = BufferDescriptor(output, Bytes); + const auto texture = TextureDescriptor(plane); + std::copy(out.begin(), out.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + userData[12] = y; + const auto program = Compile(env.device, ProbeCode, userData); + std::fill(output, output + Threads, Sentinel); + env.device.Dispatch(program, 1, 1, 1, {}, reinterpret_cast(ProbeCode.data())); + env.device.WaitIdle(); + std::array lanes{}; + std::copy(output, output + Threads, lanes.begin()); + return lanes; +} + +void FillInput(std::uint32_t* where, float first) { + for (std::uint32_t lane = 0; lane < Threads; ++lane) where[lane] = std::bit_cast(first + static_cast(lane) / 1024.0f); +} + +void FillPlane(std::uint8_t* plane, float value) { + auto* words = reinterpret_cast(plane); + std::fill(words, words + Piece / sizeof(std::uint32_t), std::bit_cast(value)); +} + +void DrawDepth(const Env& env, const void* plane, float depth) { + const Surface surface{reinterpret_cast(plane), 0, 0, false, VK_FORMAT_D32_SFLOAT}; + const float low = depth <= 0.5f ? 0.0f : 2.0f * depth - 1.0f; + const float high = depth <= 0.5f ? 2.0f * depth : 1.0f; + Require(Draw(env.device, env.shaders, surface, {.depthWrite = true, .depthCompare = VK_COMPARE_OP_ALWAYS, .minDepth = low, .maxDepth = high}) == Covered, "dispatch move depth: the depth producer did not cover the surface"); +} + +struct PlaneModel { + std::array row0{}; + std::uint32_t rest = 0; + + void Set(float value) { + rest = std::bit_cast(value); + row0.fill(rest); + } + + std::array Dispatch(const std::uint32_t* input) { + std::array sums{}; + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + sums[lane] = input[lane] + 1u; + row0[lane] = input[lane]; + } + return sums; + } +}; + +void Totals(const std::string& what); + +void Step(const Env& env, const ShaderRecompiler::RecompileResult& program, PlaneModel& model, const std::uint32_t* input, const std::string& what) { + const auto expected = model.Dispatch(input); + auto* output = env.buffers.Output(); + std::fill(output, output + Threads, Sentinel); + RunRow(env.device, program, input, output); + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + Require(output[lane] == expected[lane], "dispatch move depth: " + what + ": lane " + std::to_string(lane) + " is " + Hex(output[lane]) + ", expected " + Hex(expected[lane])); + } + Totals(what); +} + +void Verify(const Env& env, const void* plane, const PlaneModel& model, const std::string& what) { + for (const std::uint32_t y : {0u, 1u, Height - 1u}) { + const auto lanes = ReadRow(env, plane, y); + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + const auto expected = y == 0 ? model.row0[lane] : model.rest; + Require(lanes[lane] == expected, "dispatch move depth: " + what + ": row " + std::to_string(y) + ", texel " + std::to_string(lane) + " of the plane is " + Hex(lanes[lane]) + ", expected " + Hex(expected)); + } + } +} + +struct MoveSteps { + std::uint64_t checked, exact, moved, refused; +}; + +MoveSteps Since(const AgcDriver::VulkanDevice::DispatchMoveCount& before) { + const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts(); + return {now.checked - before.checked, now.exact - before.exact, now.moved - before.moved, now.refused - before.refused}; +} + +void Totals(const std::string& what) { + const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts(); + std::printf("dispatch move depth: %s: in total checked %llu, exact %llu, moved %llu, refused %llu, dropped by stage A %llu\n", what.c_str(), static_cast(now.checked), static_cast(now.exact), static_cast(now.moved), static_cast(now.refused), static_cast(now.filtered)); +} + +void DescribeBindings(const ShaderRecompiler::RecompileResult& program) { + for (const auto& binding : program.bindings) { + std::string written; + for (const bool bit : binding.bufferWritten) written += bit ? '1' : '0'; + std::printf("dispatch move depth: binding role %d kind %d count %u, %zu descriptor words, buffers written [%s], images written %zu\n", static_cast(binding.role), static_cast(binding.kind), binding.count, binding.guestDescriptor.size(), written.c_str(), binding.imageWritten.size()); + } +} + +void Print(const char* what, const MoveSteps& step) { + std::printf("dispatch move depth: %s: checked %llu, exact %llu, moved %llu, refused %llu\n", what, static_cast(step.checked), static_cast(step.exact), static_cast(step.moved), static_cast(step.refused)); +} + +void Warm(const Env& env, std::uint8_t* plane) { + FillPlane(plane, 0.5f); + PlaneModel model; + model.Set(0.5f); + const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output()); + Step(env, program, model, env.buffers.InputA(), "warm-up: the first input"); + Step(env, program, model, env.buffers.InputB(), "warm-up: the second input"); + Step(env, program, model, env.buffers.InputC(), "warm-up: the third input"); +} + +bool Control(const Env& env, std::uint8_t* plane) { + FillPlane(plane, 0.5f); + PlaneModel model; + model.Set(0.5f); + const auto refusals = AgcDriver::Graphics::ShaderResources::DepthGuardRefusals(); + const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output()); + DescribeBindings(program); + Step(env, program, model, env.buffers.InputA(), "control: the first dispatch"); + Totals("control, after the first dispatch"); + const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts(); + Step(env, program, model, env.buffers.InputB(), "control: an input moved"); + Totals("control, after an input moved"); + Step(env, program, model, env.buffers.InputC(), "control: the input moved again"); + Totals("control, after the input moved again"); + Step(env, program, model, env.buffers.InputC(), "control: an exact repeat"); + Totals("control, after an exact repeat"); + const auto steps = Since(start); + Print("control, a plane without a depth surface", steps); + Verify(env, plane, model, "control"); + Require(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() == refusals, "dispatch move depth: the depth guard refused a proof with no depth surface in sight"); + Require(steps.refused == 0, "dispatch move depth: the control's repeats were refused"); + if (steps.checked == 0) return false; + Require(steps.checked == 3 && steps.moved == 3, "dispatch move depth: the control's repeats were not all served by the template of the first dispatch"); + return true; +} + +void Mirror(const Env& env, std::uint8_t* plane, const char* where) { + const std::string at = std::string(where) + ", the mirror: "; + FillPlane(plane, 0.5f); + PlaneModel model; + DrawDepth(env, plane, 0.25f); + model.Set(0.25f); + const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output()); + Step(env, program, model, env.buffers.InputA(), at + "the first dispatch"); + Verify(env, plane, model, at + "after the first dispatch"); + DrawDepth(env, plane, 0.75f); + model.Set(0.75f); + const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts(); + Step(env, program, model, env.buffers.InputB(), at + "an input moved after the depth was drawn again"); + Verify(env, plane, model, at + "after the moved dispatch"); + Step(env, program, model, env.buffers.InputB(), at + "an exact repeat"); + DrawDepth(env, plane, 0.25f); + model.Set(0.25f); + Step(env, program, model, env.buffers.InputC(), at + "the input moved again after another redraw"); + Verify(env, plane, model, at + "after the last dispatch"); + Print((at + "repeats and redraws").c_str(), Since(start)); +} + +void Appears(const Env& env, std::uint8_t* plane, const char* where, bool moved, bool templates) { + const std::string at = std::string(where) + (moved ? ", a surface appears, input moved: " : ", a surface appears, exact repeat: "); + FillPlane(plane, 0.125f); + PlaneModel model; + model.Set(0.125f); + const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output()); + Step(env, program, model, env.buffers.InputA(), at + "the first dispatch, no depth surface yet"); + DrawDepth(env, plane, 0.25f); + model.Set(0.25f); + const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts(); + const auto refusals = AgcDriver::Graphics::ShaderResources::DepthGuardRefusals(); + Step(env, program, model, moved ? env.buffers.InputB() : env.buffers.InputA(), at + "the dispatch after the surface appeared"); + std::printf("dispatch move depth: %sthe depth guard refused %llu proofs\n", at.c_str(), static_cast(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() - refusals)); + if (templates) Require(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() > refusals, "dispatch move depth: " + at + "the template over the plain storage image was proved current although a depth surface covers its memory"); + Print((at + "the dispatch after the surface appeared").c_str(), Since(start)); + Totals(at + "after that dispatch"); + Verify(env, plane, model, at + "after that dispatch"); + DrawDepth(env, plane, 0.75f); + model.Set(0.75f); + Step(env, program, model, env.buffers.InputC(), at + "the input moved after the next redraw"); + Verify(env, plane, model, at + "after the last dispatch"); +} + +bool RunTests(const Env& env, const GuestBlock& watched, const GuestBlock& unwatched, std::size_t pass) { + const auto inWatch = pass * 10; + const auto outside = pass * 3; + FillInput(env.buffers.InputA(), 0.0625f); + FillInput(env.buffers.InputB(), 0.109375f); + FillInput(env.buffers.InputC(), 0.15625f); + Warm(env, watched.PieceAt(9)); + const bool templates = Control(env, watched.PieceAt(inWatch)); + if (!templates) std::puts("dispatch move depth: the device kept no template (no host imports); the moved-buffer path is not exercised"); + Mirror(env, watched.PieceAt(inWatch + 1), "inside the write watch"); + Appears(env, watched.PieceAt(inWatch + 2), "inside the write watch", true, templates); + Appears(env, watched.PieceAt(inWatch + 3), "inside the write watch", false, templates); + Mirror(env, unwatched.PieceAt(outside), "outside the write watch"); + Appears(env, unwatched.PieceAt(outside + 1), "outside the write watch", true, templates); + Appears(env, unwatched.PieceAt(outside + 2), "outside the write watch", false, templates); + return templates; +} + +} + +int main() { + try { + const GuestBlock watched(20, true); + const GuestBlock unwatched(6, false); + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto shaders = DepthFastClearHarness::Compile(*device); + const Buffers buffers(watched); + const Env env{*device, shaders, buffers}; + bool templates = true; +#ifndef _WIN32 + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(*device); + const auto decided = AgcDriver::Graphics::PrepareImportWatch(context); + struct Restore { + const AgcDriver::Graphics::Context& context; + AgcDriver::Graphics::ImportWatch decided; + ~Restore() { AgcDriver::Graphics::SetImportWatch(context, decided); } + } restore{context, decided}; + std::size_t pass = 0; + for (const auto mode : {AgcDriver::Graphics::ImportWatch::Watch, AgcDriver::Graphics::ImportWatch::Unwatch}) { + AgcDriver::Graphics::SetImportWatch(context, mode); + std::printf("dispatch move depth: pass %zu, imports %s\n", pass, mode == AgcDriver::Graphics::ImportWatch::Watch ? "under the write watch" : "left unwatched"); + templates = RunTests(env, watched, unwatched, pass++) && templates; + } +#else + templates = RunTests(env, watched, unwatched, 0); +#endif + if (!templates) return VulkanTestSkipped; + std::puts("dispatch move depth tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +}