starred/AnyPS5
1
0 Fork 0
mirror of https://github.com/boykopovar/AnyPS5.git synced 2026-10-18 23:07:07 -05:00
Code Issues Packages Projects Releases 3 Wiki Activity

Merge pull request #7044 from AlxcBTW/pr/dispatch-move

Browse Source 
perf(agc): serve compute dispatches whose read-only buffers moved from the cached template
This commit is contained in:
Dean authored and GitHub committed 2026-10-18 11:44:09 +13:00
2 parents b4f05ab328 5da6de62ad
commit 55f82fee83
10 files changed
+1167 -25
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp
+4
View File
@@ -185,6 +185,10 @@ public:
std::uint64_t created, underGpuMutex;
};
static ComputePipelineCreation ComputePipelineCounts();
struct DispatchMoveCount {
std::uint64_t checked, exact, moved, refused, filtered, rebuilt, kept, served;
};
static DispatchMoveCount DispatchMoveCounts();
void ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable);
void* Window() const;
void Resize(std::uint32_t width, std::uint32_t height);
core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp
+156 -7
View File
@@ -3722,11 +3722,24 @@ bool TemplateDataRefresh() {
return enabled;
}
bool DispatchMove() {
static const bool enabled = std::getenv("APS5_NO_DISPATCH_MOVE") == nullptr && TemplateDataRefresh();
return enabled;
}
std::size_t DispatchMoveBytes() {
static const std::size_t bytes = [] {
const char* text = std::getenv("APS5_DISPMOVE_MAX_KIB");
return static_cast<std::size_t>(text != nullptr ? std::strtoull(text, nullptr, 10) : 1024ull) * 1024u;
}();
return bytes;
}
// The content key of a compute stage, naming the device: the cache is process-wide and the driver
// replaces the headless device with the windowed one while workers may still use the old one, so
// the key names the device that built the entry (its descriptor set and pooled buffers belong to it).
ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device) {
auto key = Graphics::ShaderResources::ContentKey(shader, !TemplateDataRefresh());
ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device, bool movable = true) {
auto key = Graphics::ShaderResources::ContentKey(shader, !TemplateDataRefresh(), movable && DispatchMove());
const auto deviceHandle = reinterpret_cast<std::uint64_t>(device);
key.push_back(static_cast<std::uint32_t>(deviceHandle));
key.push_back(static_cast<std::uint32_t>(deviceHandle >> 32u));
@@ -3757,6 +3770,7 @@ struct PreparedDispatch {
std::shared_ptr<Graphics::ShaderResources> resources;
std::shared_ptr<Graphics::ShaderResources> cached;
ResourceCache::Key key;
bool exact = false;
// APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch, and
// the prepare's parts for the driver's 'prepare:' rows (PreparePhase order).
double prepareMs = 0;
@@ -3956,6 +3970,63 @@ RecipeCounters& Recipes() {
return counters;
}
constexpr std::size_t MoveRefusals = static_cast<std::size_t>(Graphics::ShaderResources::MoveRefusal::Count);
struct DispatchMoveCounters {
std::atomic<std::uint64_t> checked{0}, exact{0}, moved{0}, buffers{0}, bytes{0};
std::atomic<std::uint64_t> rebuilt{0}, kept{0}, served{0};
std::array<std::atomic<std::uint64_t>, 5> sizes{};
std::array<std::atomic<std::uint64_t>, MoveRefusals> refused{};
std::array<std::atomic<std::uint64_t>, MoveRefusals> filtered{};
std::atomic<std::int64_t> lastReport{0};
};
DispatchMoveCounters& DispatchMoves() {
static DispatchMoveCounters counters;
return counters;
}
void reportDispatchMove() {
auto& moves = DispatchMoves();
const auto nowMs = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
auto last = moves.lastReport.load();
if (nowMs - last < 10000 || !moves.lastReport.compare_exchange_strong(last, nowMs)) return;
const auto take = [](std::atomic<std::uint64_t>& counter) { return static_cast<unsigned long long>(counter.exchange(0, std::memory_order_relaxed)); };
const auto list = [&](std::array<std::atomic<std::uint64_t>, MoveRefusals>& counters) {
std::string text;
unsigned long long total = 0;
char item[80];
for (std::size_t i = 1; i < MoveRefusals; ++i) {
const auto count = take(counters[i]);
total += count;
if (count == 0) continue;
std::snprintf(item, sizeof(item), "%s %s %llu", text.empty() ? "" : ",", Graphics::ShaderResources::MoveRefusalName(static_cast<Graphics::ShaderResources::MoveRefusal>(i)), count);
text += item;
}
std::snprintf(item, sizeof(item), "%llu (", total);
return std::string(item) + text + ")";
};
const auto checked = take(moves.checked);
const auto exact = take(moves.exact);
const auto moved = take(moves.moved);
const auto buffers = take(moves.buffers);
const auto bytes = take(moves.bytes);
const auto rebuilt = take(moves.rebuilt);
const auto kept = take(moves.kept);
const auto served = take(moves.served);
unsigned long long sizes[5];
for (std::size_t i = 0; i < 5; ++i) sizes[i] = take(moves.sizes[i]);
const auto refused = list(moves.refused);
const auto filtered = list(moves.filtered);
AgcDriver::ProfilePrint_nid_no_patch("[dispmove] (10 s) %llu template hits checked: %llu with every read-only buffer where it was built, %llu with %llu moved ones (%.2f MiB snapshotted; per dispatch up to 1 KiB %llu, 16 KiB %llu, 256 KiB %llu, 4 MiB %llu, more %llu); refused %s at stage A (the hit was dropped before the record), %s at record; after a refusal: %llu rebuilt, %llu inserted beside the movable template (kept), %llu served by an exact entry\n", checked, exact, moved, buffers, static_cast<double>(bytes) / 1048576.0, sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], filtered.c_str(), refused.c_str(), rebuilt, kept, served);
}
void noteDispatchMoveSize(std::size_t bytes) {
auto& moves = DispatchMoves();
const std::size_t bucket = bytes <= 1024 ? 0 : bytes <= 16384 ? 1 : bytes <= 262144 ? 2 : bytes <= 4194304 ? 3 : 4;
moves.sizes[bucket].fetch_add(1, std::memory_order_relaxed);
}
void reportRecipes() {
auto& recipes = Recipes();
const auto nowMs = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
@@ -4106,12 +4177,23 @@ struct RecordedDispatch {
// Whether the data refresh recorded anything.
bool refreshed = false;
bool resourceKeyMatched = false;
std::shared_ptr<Graphics::ShaderResources::DrawBindings> bindings{};
};
VkDevice VulkanDevice::Device() const {
return state->device;
}
VulkanDevice::DispatchMoveCount VulkanDevice::DispatchMoveCounts() {
auto& moves = DispatchMoves();
DispatchMoveCount count{moves.checked.load(), moves.exact.load(), moves.moved.load(), 0, 0, moves.rebuilt.load(), moves.kept.load(), moves.served.load()};
for (std::size_t i = 1; i < MoveRefusals; ++i) {
count.refused += moves.refused[i].load();
count.filtered += moves.filtered[i].load();
}
return count;
}
bool VulkanDevice::DispatchRecipes() {
static const bool noDispatchRecipe = std::getenv("APS5_NO_DISPATCH_RECIPE") != nullptr;
return !noDispatchRecipe;
@@ -4251,6 +4333,15 @@ std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderReco
prepared->key = DispatchContentKey(compute, context.device);
phase(PreparedDispatch::PrepareKey);
cached = state->resourceCache.Find(prepared->key);
if (cached != nullptr && DispatchMove()) {
Graphics::ShaderResources::MoveProof proof;
if (!cached->MovedDispatchBuffers(compute, nullptr, DispatchMoveBytes(), proof).has_value()) {
DispatchMoves().filtered[static_cast<std::size_t>(proof.refusal)].fetch_add(1, std::memory_order_relaxed);
prepared->key = DispatchContentKey(compute, context.device, false);
prepared->exact = true;
cached = state->resourceCache.Find(prepared->key);
}
}
prepared->cached = cached;
phase(PreparedDispatch::PrepareFind);
}
@@ -4448,6 +4539,10 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
recorder.Keep(record.objects);
recorder.Keep(record.resources);
if (record.resources != nullptr) recorder.KeepBytes(record.resources.get(), record.resources->CopiedBytes());
if (record.bindings != nullptr) {
recorder.Keep(record.bindings);
for (const auto& snapshot : record.bindings->snapshots) recorder.KeepBytes(snapshot.buffer.get(), snapshot.buffer->Bytes().size());
}
recordStep(PhaseRecordKeeps);
if (record.dataRefresh != RecordedDispatch::DataRefresh::None) {
// The template's data buffers take this dispatch's words: a transfer write the pre-dispatch
@@ -4497,7 +4592,9 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
++d.preBarriersRecorded;
}
context.Resolved(&Graphics::DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->pipeline);
resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout);
if (record.bindings != nullptr) Graphics::Require(record.bindings->allocation.set != VK_NULL_HANDLE, "dispatch overlay has no descriptor set");
if (record.bindings != nullptr) context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, 0, 1, &record.bindings->allocation.set, 0, nullptr);
else resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout);
if (const auto dwords = record.shader->program->workgroupMemoryDwords; dwords != 0) {
auto memory = state->WorkgroupMemoryFor(context, record.x, record.y, record.z, dwords);
context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, ShaderRecompiler::WorkgroupMemoryDescriptorSet, 1, &memory->set, 0, nullptr);
@@ -4710,7 +4807,9 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
// The object came from the resource cache: a compute template whose data buffers take this
// dispatch's words at the record (ShaderResources::RefreshData).
bool fromCache = false;
bool refusedMove = false;
double revalidateMs = 0;
std::shared_ptr<Graphics::ShaderResources::DrawBindings> overlay;
// The batches the pre-sync (PrepareDispatch) waited for are reaped first: their completions
// (CPU write-backs) land before stage B or a cached object's Revalidate reads, and the flush
// hook then finds nothing pending over their surfaces.
@@ -4730,6 +4829,11 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
// The image lookups of this call by outcome (stage B, a Revalidate or a locked build), for the
// [indirect] line: reset here, read after the resources phase.
if (profile && arguments != 0) Graphics::ThreadLookupOutcomes() = {};
const auto noteRefusalBuild = [](bool inserted) {
auto& moves = DispatchMoves();
moves.rebuilt.fetch_add(1, std::memory_order_relaxed);
if (inserted) moves.kept.fetch_add(1, std::memory_order_relaxed);
};
// The resource cache insert evicts the oldest entry, destroying its ShaderResources here when the
// cache was the last owner: timed apart from the build.
const auto insert = [&] {
@@ -4778,6 +4882,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
}
if (cacheable) {
++d.cacheMisses;
if (prepared->exact) noteRefusalBuild(resources->Reusable());
if (resources->Reusable()) insert();
}
} else if (cacheable) {
@@ -4785,13 +4890,54 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
// object it found: revalidated as a map hit would be (an entry replaced meanwhile is left
// alone by the pointer-conditional Remove). APS5_NO_PREPARED_FIND=1 looks the key up again.
static const bool preparedFind = std::getenv("APS5_NO_PREPARED_FIND") == nullptr;
const bool exactEntry = prepared != nullptr && prepared->exact;
contentKey = prepared != nullptr && !prepared->key.empty() ? std::move(prepared->key) : DispatchContentKey(shaders[0], context.device);
auto cached = preparedFind && prepared != nullptr && prepared->cached != nullptr ? std::move(prepared->cached) : state->resourceCache.Find(contentKey);
if (cached != nullptr) {
if (cached->Revalidate(shaders[0])) {
resources = std::move(cached);
fromCache = true;
++d.cacheHits;
bool usable = true;
if (DispatchMove() && !exactEntry) {
auto& moves = DispatchMoves();
Graphics::ShaderResources::MoveProof proof;
auto moved = cached->MovedDispatchBuffers(shaders[0], state->recorder.get(), DispatchMoveBytes(), proof);
moves.checked.fetch_add(1, std::memory_order_relaxed);
if (moved.has_value() && !moved->empty()) {
overlay = cached->PrepareDrawBindings(*state->recorder, *moved, true);
if (overlay != nullptr && overlay->allocation.set == VK_NULL_HANDLE) overlay.reset();
if (overlay == nullptr) {
proof.refusal = Graphics::ShaderResources::MoveRefusal::Bindings;
moved.reset();
}
}
if (!moved.has_value()) {
usable = false;
moves.refused[static_cast<std::size_t>(proof.refusal)].fetch_add(1, std::memory_order_relaxed);
} else if (moved->empty()) {
moves.exact.fetch_add(1, std::memory_order_relaxed);
} else {
moves.moved.fetch_add(1, std::memory_order_relaxed);
moves.buffers.fetch_add(proof.buffers, std::memory_order_relaxed);
moves.bytes.fetch_add(proof.bytes, std::memory_order_relaxed);
noteDispatchMoveSize(proof.bytes);
}
}
if (!usable) {
contentKey = DispatchContentKey(shaders[0], context.device, false);
refusedMove = true;
cached = state->resourceCache.Find(contentKey);
if (cached != nullptr && cached->Revalidate(shaders[0])) usable = true;
else if (cached != nullptr) {
state->resourceCache.Remove(contentKey, cached.get());
++d.cacheInvalidated;
state->recorder->Keep(std::move(cached));
}
}
if (usable) {
if (refusedMove || exactEntry) DispatchMoves().served.fetch_add(1, std::memory_order_relaxed);
resources = std::move(cached);
fromCache = true;
++d.cacheHits;
}
} else {
state->resourceCache.Remove(contentKey, cached.get());
++d.cacheInvalidated;
@@ -4810,6 +4956,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
if (profile) timer.add(PhaseResourcesFullBuild, std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - buildStart).count());
if (cacheable) {
++d.cacheMisses;
if (refusedMove) noteRefusalBuild(resources->Reusable());
if (resources->Reusable()) insert();
}
}
@@ -4832,6 +4979,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
AgcDriver::ProfilePrint_nid_no_patch("[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast<unsigned long long>(d.cacheHits), static_cast<unsigned long long>(d.cacheMisses), static_cast<unsigned long long>(d.cacheInvalidated), state->resourceCache.Size(), static_cast<unsigned long long>(d.templateRefreshed), d.templateRevalidateMs, static_cast<unsigned long long>(d.templateSameWords), static_cast<unsigned long long>(ResourceCache::Finds()), static_cast<unsigned long long>(ResourceCache::Touches()));
AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call: %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast<unsigned long long>(d.procLookups), d.profiledDispatches != 0 ? static_cast<double>(d.procLookups) / static_cast<double>(d.profiledDispatches) : 0.0, static_cast<unsigned long long>(d.preBarriersRecorded), static_cast<unsigned long long>(d.preBarriersSkipped), static_cast<unsigned long long>(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast<unsigned long long>(computePipelinesCreatedLocked.load(std::memory_order_relaxed)));
reportRecipes();
reportDispatchMove();
}
timing.Mark("shader_resources");
timer.phase(PhaseResources);
@@ -4853,6 +5001,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
IndirectOutcome outcome{0, 0};
RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer};
record.resourceKeyMatched = fromCache;
record.bindings = overlay;
if (arguments != 0) decideIndirect(record, outcome, groupsText);
// The indirect hold total below still covers the whole call of a CPU-resolved one.
const bool indirect = record.arguments != 0 || outcome.cpuReason != 0;
@@ -4906,7 +5055,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
// object the resource cache serves under this content key (reusable: no lease, no copied
// writes, every direct region import- or mirror-served), so a hit's proof is the template's
// Revalidate and nothing needs completion work.
if (recipeOut != nullptr && DispatchRecipes() && cacheable && resources->Reusable() && !resources->NeedsCompletion() && objects != nullptr && !contentKey.empty()) {
if (recipeOut != nullptr && DispatchRecipes() && cacheable && overlay == nullptr && resources->Reusable() && !resources->NeedsCompletion() && objects != nullptr && !contentKey.empty()) {
auto recipe = std::make_shared<Recipe>();
recipe->device = state->device;
recipe->templateRef = resources;
core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp
+1
View File
@@ -26,6 +26,7 @@ void RequireNoNativeDepthFeedback_nid_no_patch(const Context& context, const Dep
void ClearDepthSurfaces(VkDevice device);
void RetireDepthSurfaces(VkDevice device, std::uint64_t address, std::uint64_t bytes);
bool DepthSurfaceAt(std::uint64_t address);
bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t bytes);
bool DepthSurfaceAt(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource);
std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const std::array<std::uint32_t, 3>& numThreads);
std::size_t HtileDepthClearBytes_nid_no_patch(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, std::span<const std::uint32_t> packet, std::uint64_t indirectArguments);
core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp
+11 -1
View File
@@ -178,8 +178,16 @@ public:
std::size_t size;
std::vector<std::uint32_t> words;
};
std::shared_ptr<DrawBindings> PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved = {}) const;
std::shared_ptr<DrawBindings> PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved = {}, bool movedOnly = false) const;
std::optional<std::vector<MovedBuffer>> MovedReadOnlyBuffers(std::span<const CompiledShader> shaders, Recorder& recorder) const;
enum class MoveRefusal : std::size_t { None, Shape, NotGuest, Written, Limit, Inaccessible, Overlap, Bindings, Count };
struct MoveProof {
MoveRefusal refusal = MoveRefusal::None;
std::size_t buffers = 0;
std::size_t bytes = 0;
};
static const char* MoveRefusalName(MoveRefusal refusal);
std::optional<std::vector<MovedBuffer>> MovedDispatchBuffers(const CompiledShader& shader, Recorder* recorder, std::size_t maxBytes, MoveProof& proof) const;
static std::size_t MaxDrawSnapshotBytes();
void WriteBack();
// Deferred completion: MarkGpuWrites registers the results the recorded work leaves on the GPU
@@ -237,6 +245,7 @@ public:
bool Reusable() const { return reusable; }
const std::vector<std::uint64_t>& MultisampleSources() const { return multisampleSources; }
static bool NeverReusable(std::span<const CompiledShader> shaders);
static std::uint64_t DepthGuardRefusals();
const std::vector<std::shared_ptr<Texture>>& SampledTextures() const { return textures; }
bool ChecksFaults() const { return bda != nullptr; }
void CheckFaults() const;
@@ -389,6 +398,7 @@ private:
// same way but is left out of the write set (no write-back, no pending-write note). `atomic`
// is its bufferAtomic (see GuestBufferMemory::AddWritable).
std::size_t addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read);
std::uint64_t guestBufferBytes(std::span<const std::uint32_t> words) const;
std::size_t addDataBuffer(std::span<const std::uint32_t> words);
// Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache
// locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding).
core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp
+5
View File
@@ -1576,6 +1576,11 @@ bool DepthSurfaceAt(std::uint64_t address) {
return std::any_of(surfaces().begin(), surfaces().end(), [&](const auto& surface) { return surface->active->load(std::memory_order_acquire) && !surface->retired && (surface->target.address == address || surface->target.stencilAddress == address); });
}
bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t bytes) {
std::lock_guard lock(surfacesMutex());
return std::any_of(surfaces().begin(), surfaces().end(), [&](const auto& surface) { return surface->active->load(std::memory_order_acquire) && !surface->retired && surface->context.device == device && surface->Overlaps(address, bytes); });
}
void RetireDepthSurfaces(VkDevice device, std::uint64_t address, std::uint64_t bytes) {
std::lock_guard lock(surfacesMutex());
for (const auto& surface : surfaces()) {
core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp
+95 -16
View File
@@ -2217,6 +2217,7 @@ void mixDataWords(std::uint64_t& hash, std::span<const std::uint32_t> words) {
// ResourceCache::Find and Touch calls (the [rescache] line).
std::atomic<std::uint64_t> resourceCacheFinds{0};
std::atomic<std::uint64_t> resourceCacheTouches{0};
std::atomic<std::uint64_t> depthGuardRefusals{0};
}
@@ -2259,6 +2260,10 @@ void ShaderResources::noteReusable() {
reusable = true;
}
std::uint64_t ShaderResources::DepthGuardRefusals() {
return depthGuardRefusals.load(std::memory_order_relaxed);
}
bool ShaderResources::NeverReusable(std::span<const CompiledShader> shaders) {
const bool loopGuard = LoopGuardEnabled();
return std::any_of(shaders.begin(), shaders.end(), [loopGuard](const CompiledShader& shader) {
@@ -2729,6 +2734,10 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::span<const
const auto& own = image->Descriptor();
const auto address = own.baseAddress;
const auto bytes = static_cast<std::size_t>(image->GuestBytes());
if (DepthSurfaceOverlaps(context.device, address, bytes)) {
depthGuardRefusals.fetch_add(1, std::memory_order_relaxed);
return fail(FastFail::Depth);
}
if (GuestMemory::CollectWrites(address, bytes) == 0) return fail(FastFail::Collect);
if (own.dccAddress != 0) {
// Refresh's unchanged branch (its key compare against the keys the content was
@@ -2832,6 +2841,10 @@ bool ShaderResources::fastRevalidateEach() {
if (own.dccAddress != 0) return false;
const auto address = own.baseAddress;
const auto bytes = static_cast<std::size_t>(image->GuestBytes());
if (DepthSurfaceOverlaps(context.device, address, bytes)) {
depthGuardRefusals.fetch_add(1, std::memory_order_relaxed);
return false;
}
if (GuestMemory::CollectWrites(address, bytes) == 0) return false;
if (PendingStorageOverlaps(address, bytes, image)) return false;
if (!StorageImageCached(context, image)) return false;
@@ -3699,11 +3712,8 @@ DescriptorCache::Stats DescriptorCache::Counters() const {
return stats;
}
std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) {
Require(words.size() == 4, "buffer descriptor must contain four DWORDs");
Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set");
std::uint64_t ShaderResources::guestBufferBytes(std::span<const std::uint32_t> words) const {
const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}};
Require(descriptor.Type() == 0u, "buffer descriptor uses an unsupported type");
const auto address = descriptor.Base48();
auto byteSize = descriptor.GetSize();
if (byteSize > context.limits.maxStorageBufferRange) {
@@ -3711,22 +3721,33 @@ std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words
if (mappedEnd <= address) byteSize = 0;
else if (mappedEnd - address < context.limits.maxStorageBufferRange) byteSize = mappedEnd - address;
}
if (byteSize == 0 || address == 0) return 0;
if (const std::uint64_t stride = descriptor.Stride(); stride != 0 && byteSize <= context.limits.maxStorageBufferRange) {
constexpr std::uint64_t GuestPageBytes = 0x4000;
const auto end = address + byteSize;
const auto overhang = end % GuestPageBytes;
if (overhang != 0 && overhang < stride) {
const auto allocationEnd = RegisteredReadableEnd(address);
if (allocationEnd > address && allocationEnd < end && end - allocationEnd < stride) byteSize = allocationEnd - address;
}
}
return byteSize;
}
std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) {
Require(words.size() == 4, "buffer descriptor must contain four DWORDs");
Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set");
const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}};
Require(descriptor.Type() == 0u, "buffer descriptor uses an unsupported type");
const auto address = descriptor.Base48();
const auto byteSize = guestBufferBytes(words);
if (byteSize == 0 || address == 0) {
allocations.push_back({0, EmptyBufferBytes, false, nullptr, ShaderRecompiler::DescriptorRole::GuestBuffers, false});
return allocations.size() - 1;
}
Require(byteSize <= context.limits.maxStorageBufferRange, "shader buffer exceeds descriptor range limit");
Require(byteSize <= std::numeric_limits<std::size_t>::max(), "shader buffer size exceeds host address space");
auto size = static_cast<std::size_t>(byteSize);
if (const std::uint64_t stride = descriptor.Stride(); stride != 0) {
constexpr std::uint64_t GuestPageBytes = 0x4000;
const auto end = address + size;
const auto overhang = end % GuestPageBytes;
if (overhang != 0 && overhang < stride) {
const auto allocationEnd = RegisteredReadableEnd(address);
if (allocationEnd > address && allocationEnd < end && end - allocationEnd < stride) size = static_cast<std::size_t>(allocationEnd - address);
}
}
const auto size = static_cast<std::size_t>(byteSize);
Require(target == nullptr || !overlap(address, size, target->address, target->bytes), "shader buffer aliases the render target");
// APS5_ALL_BUFFERS_WRITTEN=1: every element is noted as written, as before bufferWritten existed.
static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr;
@@ -4732,13 +4753,71 @@ std::optional<std::vector<ShaderResources::MovedBuffer>> ShaderResources::MovedR
return moved;
}
const char* ShaderResources::MoveRefusalName(MoveRefusal refusal) {
switch (refusal) {
case MoveRefusal::None: return "none";
case MoveRefusal::Shape: return "bindings differ";
case MoveRefusal::NotGuest: return "not read in place";
case MoveRefusal::Written: return "written or emptied";
case MoveRefusal::Limit: return "over the limit";
case MoveRefusal::Inaccessible: return "unmapped";
case MoveRefusal::Overlap: return "pending write";
case MoveRefusal::Bindings: return "no bindings made";
case MoveRefusal::Count: break;
}
return "?";
}
std::optional<std::vector<ShaderResources::MovedBuffer>> ShaderResources::MovedDispatchBuffers(const CompiledShader& shader, Recorder* recorder, std::size_t maxBytes, MoveProof& proof) const {
proof = {};
const auto refuse = [&](MoveRefusal why) {
proof.refusal = why;
return std::optional<std::vector<MovedBuffer>>{};
};
std::vector<MovedBuffer> moved;
if (shader.program == nullptr || usesBda) return refuse(MoveRefusal::Shape);
if (_set == VK_NULL_HANDLE) return moved;
for (const auto& binding : shader.program->bindings) {
if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue;
const auto kept = std::find_if(bindings.begin(), bindings.end(), [&](const Binding& item) { return item.layout.binding == binding.binding; });
if (kept == bindings.end() || kept->allocations.size() != binding.count || binding.guestDescriptor.size() != static_cast<std::size_t>(binding.count) * 4u) return refuse(MoveRefusal::Shape);
for (std::uint32_t element = 0; element < binding.count; ++element) {
const auto* words = binding.guestDescriptor.data() + static_cast<std::size_t>(element) * 4u;
const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}};
const auto address = descriptor.Base48();
const auto size = guestBufferBytes(std::span<const std::uint32_t>(words, 4));
const auto& item = allocations[kept->allocations[element]];
const bool empty = size == 0 || address == 0;
if (!item.guest) {
if (empty) continue;
return refuse(MoveRefusal::NotGuest);
}
if (item.address == address && item.size == size) continue;
const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element];
if (written || empty || item.written) return refuse(MoveRefusal::Written);
if (size > context.limits.maxStorageBufferRange) return refuse(MoveRefusal::Limit);
const auto begin = address - item.adjustment;
const auto bytes = static_cast<std::size_t>(GuestBufferMemory::ViewBytes(size, item.adjustment));
if (maxBytes != 0 && proof.bytes + bytes > maxBytes) return refuse(MoveRefusal::Limit);
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(begin), bytes)) return refuse(MoveRefusal::Inaccessible);
const bool pending = recorder != nullptr ? recorder->PendingWriteOverlaps(begin, bytes) : Recorder::SnapshotWriteOverlaps(begin, bytes);
const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> range{{{begin, begin + bytes}}};
if (guestMemory.WritesOverlap(begin, bytes) || pending || StorageTexture::AnyPendingOverlaps(range) || AnyShadowedOverlaps(begin, bytes)) return refuse(MoveRefusal::Overlap);
moved.push_back({kept->allocations[element], address, static_cast<std::size_t>(size)});
++proof.buffers;
proof.bytes += bytes;
}
}
return moved;
}
std::size_t ShaderResources::MaxDrawSnapshotBytes() {
const char* text = std::getenv("APS5_DRAW_SNAPSHOT_MAX_MIB");
const auto mib = text != nullptr ? std::strtoull(text, nullptr, 10) : 64ull;
return static_cast<std::size_t>(mib * 1024ull * 1024ull);
}
std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved) const {
std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved, bool movedOnly) const {
if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE) || usesBda) return {};
const auto reads = guestMemory.InPlaceReads();
const auto maxSnapshotBytes = maxDrawSnapshotBytes;
@@ -4778,7 +4857,7 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
address = override->address;
size = override->size;
} else {
if (!item.guest || item.written || guestMemory.WritesOverlap(item.address, item.size)) continue;
if (movedOnly || !item.guest || item.written || guestMemory.WritesOverlap(item.address, item.size)) continue;
const bool direct = std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return item.address >= range.first && item.address < range.second && item.size <= range.second - item.address; });
if (!direct || recorder.PendingWriteOverlaps(item.address, item.size)) continue;
}
core/libs/prx/libSceAgcDriver/tests/Graphics.cpp
+147
View File
@@ -4555,6 +4555,152 @@ void descriptorSnapshotTests() {
}
}
alignas(16) std::array<std::uint32_t, 8> guestMovedFirst{0x51515151, 0x52525252, 0x53535353, 0x54545454, 0x55555555, 0x56565656, 0x57575757, 0x58585858};
alignas(16) std::array<std::uint32_t, 8> guestMovedSecond{0x61616161, 0x62626262, 0x63636363, 0x64646464, 0x65656565, 0x66666666, 0x67676767, 0x68686868};
void dispatchMoveTests() {
using AgcDriver::Graphics::CompiledShader;
using AgcDriver::Graphics::DescriptorCache;
using AgcDriver::Graphics::Recorder;
using AgcDriver::Graphics::ShaderResources;
using Refusal = ShaderResources::MoveRefusal;
mock = MockVulkan{};
mock.allowCopies = true;
auto context = mockContext();
{
DescriptorCache cache(context);
context.descriptorCache = &cache;
std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
Recorder recorder(context);
ShaderRecompiler::RecompileResult built;
auto inputs = makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32)));
inputs.bufferWritten = {false, false};
built.bindings.push_back(inputs);
auto output = makeBinding(Role::GuestBuffers, 4, 1, vsharp(guestThird.data(), 8));
output.bufferWritten = {true};
built.bindings.push_back(output);
const CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &built, 0};
ShaderResources resources(context, shader);
const auto moveTo = [&](std::size_t element, const void* where, std::uint32_t bytes) {
auto program = built;
const auto words = vsharp(where, bytes);
std::copy(words.begin(), words.end(), program.bindings[0].guestDescriptor.begin() + static_cast<std::ptrdiff_t>(element) * 4);
return program;
};
const auto prove = [&](const ShaderRecompiler::RecompileResult& program, std::size_t maxBytes, ShaderResources::MoveProof& proof) {
return resources.MovedDispatchBuffers(CompiledShader{ShaderRecompiler::ShaderStage::Compute, &program, 0}, &recorder, maxBytes, proof);
};
const auto refuses = [&](const ShaderRecompiler::RecompileResult& program, std::size_t maxBytes, Refusal why, const char* what) {
ShaderResources::MoveProof refusal;
const auto listed = prove(program, maxBytes, refusal);
Require(!listed.has_value() && refusal.refusal == why, what);
};
ShaderResources::MoveProof proof;
auto moved = prove(built, 0, proof);
Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None && proof.buffers == 0, "a dispatch with every buffer where it was built moved something");
const auto relocated = moveTo(1, guestMovedSecond.data(), 24);
const CompiledShader relocatedShader{ShaderRecompiler::ShaderStage::Compute, &relocated, 0};
Require(ShaderResources::ContentKey(shader, false, true) == ShaderResources::ContentKey(relocatedShader, false, true), "moving a read-only buffer changed the movable content key");
Require(ShaderResources::ContentKey(shader, false, false) != ShaderResources::ContentKey(relocatedShader, false, false), "the fixed content key ignored a moved buffer");
moved = prove(relocated, 0, proof);
Require(moved.has_value() && moved->size() == 1 && proof.refusal == Refusal::None && proof.buffers == 1 && proof.bytes >= 24, "a moved read-only buffer was not listed");
Require(moved->front().address == reinterpret_cast<std::uintptr_t>(guestMovedSecond.data()) && moved->front().size == 24 && moved->front().words.empty(), "the listed buffer is not the moved range");
moved = prove(moveTo(0, guestFirst.data(), 8), 0, proof);
Require(moved.has_value() && moved->size() == 1 && moved->front().address == reinterpret_cast<std::uintptr_t>(guestFirst.data()) && moved->front().size == 8, "a buffer that only changed its size was not listed");
auto both = relocated;
const auto firstWords = vsharp(guestMovedFirst.data(), 12);
std::copy(firstWords.begin(), firstWords.end(), both.bindings[0].guestDescriptor.begin());
moved = prove(both, 1024, proof);
Require(moved.has_value() && moved->size() == 2 && proof.buffers == 2 && proof.bytes >= 36, "two moved buffers were not both listed");
refuses(both, 30, Refusal::Limit, "moved buffers past the byte budget together were served");
auto written = relocated;
written.bindings[0].bufferWritten[1] = true;
refuses(written, 0, Refusal::Written, "a moved buffer the dispatch writes was served");
refuses(moveTo(1, guestMovedSecond.data(), 20000), 0, Refusal::Written, "a moved buffer over the storage buffer range that no registered allocation holds was served");
refuses(relocated, 16, Refusal::Limit, "a moved buffer over the byte budget was served");
refuses(moveTo(1, reinterpret_cast<const void*>(std::uintptr_t{0x10}), 24), 0, Refusal::Inaccessible, "a moved buffer over unmapped memory was served");
refuses(moveTo(1, guestThird.data(), 8), 0, Refusal::Overlap, "a moved buffer over the dispatch's own written range was served");
auto shaped = relocated;
shaped.bindings[0].guestDescriptor.resize(4);
refuses(shaped, 0, Refusal::Shape, "a dispatch with another binding shape was served");
recorder.Sync();
}
{
constexpr std::size_t blockBytes = 65536;
void* block = ::operator new(blockBytes, std::align_val_t{blockBytes});
std::memset(block, 0x5a, blockBytes);
{
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, blockBytes, true, true, true);
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
}
const auto* end = static_cast<const std::byte*>(block) + blockBytes;
const auto strided = [](const void* pointer, std::uint32_t stride, std::uint32_t records) {
auto words = vsharp(pointer, records);
words[1] |= stride << 16u;
return words;
};
DescriptorCache cache(context);
context.descriptorCache = &cache;
std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
Recorder recorder(context);
const auto proveAgainst = [&](const ShaderRecompiler::RecompileResult& built, const std::vector<std::uint32_t>& incoming, ShaderResources::MoveProof& proof, std::optional<std::vector<ShaderResources::MovedBuffer>>& moved) {
const CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &built, 0};
ShaderResources resources(context, shader);
auto program = built;
program.bindings[0].guestDescriptor = incoming;
moved = resources.MovedDispatchBuffers(CompiledShader{ShaderRecompiler::ShaderStage::Compute, &program, 0}, &recorder, 0, proof);
};
const auto readOnly = [&](const std::vector<std::uint32_t>& words) {
ShaderRecompiler::RecompileResult built;
auto binding = makeBinding(Role::GuestBuffers, 0, 1, words);
binding.bufferWritten = {false};
built.bindings.push_back(binding);
return built;
};
const auto address = [](const void* pointer) { return reinterpret_cast<std::uintptr_t>(pointer); };
ShaderResources::MoveProof proof;
std::optional<std::vector<ShaderResources::MovedBuffer>> moved;
const auto* tail = end - 8192;
const auto* nearEnd = end - 4096;
const auto oversized = readOnly(vsharp(tail, 0xffffffffu));
proveAgainst(oversized, vsharp(tail, 0xffffffffu), proof, moved);
Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None && proof.buffers == 0, "an unchanged V# the build cut to its allocation end was not an exact hit");
proveAgainst(oversized, vsharp(nearEnd, 0xffffffffu), proof, moved);
Require(moved.has_value() && moved->size() == 1 && proof.buffers == 1 && proof.bytes == 4096 && moved->front().address == address(nearEnd) && moved->front().size == 4096, "a moved V# past the range limit was not snapshotted at its clamped range");
proveAgainst(oversized, vsharp(nearEnd, 0xfffffff0u), proof, moved);
Require(moved.has_value() && moved->size() == 1 && moved->front().size == 4096, "another oversized size over the same allocation end listed another range");
proveAgainst(oversized, vsharp(tail, 4096), proof, moved);
Require(moved.has_value() && moved->size() == 1 && moved->front().address == address(tail) && moved->front().size == 4096, "a V# that now fits the range limit was not listed at its own size");
proveAgainst(oversized, vsharp(block, 0xffffffffu), proof, moved);
Require(!moved.has_value() && proof.refusal == Refusal::Limit, "a moved V# the allocation holds past the range limit was served");
const auto overhang = readOnly(strided(tail, 24, 342));
proveAgainst(overhang, strided(tail, 24, 342), proof, moved);
Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None, "an unchanged V# with a stride overhang past the allocation was not an exact hit");
proveAgainst(overhang, strided(nearEnd, 24, 171), proof, moved);
Require(moved.has_value() && moved->size() == 1 && moved->front().address == address(nearEnd) && moved->front().size == 4096 && proof.bytes == 4096, "a moved V# with a stride overhang was snapshotted past the allocation end");
proveAgainst(overhang, strided(nearEnd, 24, 100), proof, moved);
Require(moved.has_value() && moved->size() == 1 && moved->front().size == 2400, "a moved V# inside the allocation was not listed at its own size");
recorder.Sync();
{
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block);
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
}
::operator delete(block, std::align_val_t{blockBytes});
}
Require(mock.live == 0, "dispatch move resources leaked Vulkan objects");
}
void resourceTests() {
mock = MockVulkan{};
const auto context = mockContext();
@@ -8397,6 +8543,7 @@ int main(int argc, char** argv) {
oversizedDescriptorTests();
guestBufferViewBytesTests();
descriptorSnapshotTests();
dispatchMoveTests();
descriptorCacheTests();
bufferBudgetPlacementTests();
imageTableAllocationTests();
core/libs/prx/libSceAgcDriver/tests/execution/DepthFastClearHarness.hpp
+3 -1
View File
@@ -48,6 +48,8 @@ struct DrawOptions {
VkCompareOp depthCompare = VK_COMPARE_OP_LESS;
bool stencilTest = false;
VkStencilOpState stencil{};
float minDepth = 0.0f;
float maxDepth = 1.0f;
};
inline std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
@@ -126,7 +128,7 @@ inline std::uint32_t Draw(AgcDriver::VulkanDevice& device, const Shaders& compil
state.stencilBack = options.stencil;
state.renderExtent = {Width, Height};
state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), options.minDepth, options.maxDepth};
state.negativeOneToOne = false;
state.scissor = {{0, 0}, {Width, Height}};
state.cullMode = VK_CULL_MODE_NONE;
core/libs/prx/libSceAgcDriver/tests/execution/DispatchMove.cpp
+285
View File
@@ -0,0 +1,285 @@
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include "prx/libc/include/GuestArena.hpp"
#include "prx/libc/include/GuestWriteWatch.hpp"
#include "Recompiler.hpp"
#include "VulkanTestDevice.hpp"
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#else
#include <sys/mman.h>
#endif
#include <algorithm>
#include <array>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <iostream>
#include <span>
#include <string>
#include <vector>
namespace {
using AgcDriver::Graphics::Require;
using ShaderRecompiler::ShaderStage;
constexpr std::uint32_t Threads = 32;
constexpr std::uint32_t Words = 64;
constexpr std::uint32_t Bytes = Words * 4u;
constexpr std::uint32_t Constant = 0x00010000u;
constexpr std::uint32_t Sentinel = 0xdeadbeefu;
constexpr std::size_t Piece = 65536;
constexpr std::size_t BlockBytes = 5 * Piece;
alignas(256) std::array<std::uint32_t, 8> Table{};
constexpr std::uint32_t Vop2(std::uint32_t opcode, std::uint32_t destination, std::uint32_t source0, std::uint32_t source1) {
return (opcode << 25u) | (destination << 17u) | (source1 << 9u) | source0;
}
constexpr std::uint32_t BufferOperands = (0x80u << 24u) | (1u << 16u) | (2u << 8u) | 1u;
alignas(256) constexpr std::array<std::uint32_t, 15> Code{
Vop2(0x1au, 1u, 0x82u, 0u),
0xf4080100u, 0xfa000000u,
0xbf8cc07fu,
0xe0301000u, BufferOperands,
0xbf8c3f70u,
0xf4080100u, 0xfa000010u,
0xbf8cc07fu,
Vop2(0x25u, 2u, 0xffu, 2u), Constant,
0xe0701000u, BufferOperands,
0xbf810000u,
};
class GuestBlock {
public:
GuestBlock() {
#ifdef _WIN32
block = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, Piece));
if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes);
#else
constexpr std::uintptr_t alignment = Piece;
void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (mapped != MAP_FAILED) {
const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
const auto aligned = (begin + alignment - 1) & ~(alignment - 1);
if (aligned != begin) munmap(mapped, aligned - begin);
if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast<void*>(aligned + BlockBytes), begin + alignment - aligned);
block = reinterpret_cast<std::uint8_t*>(aligned);
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes);
}
#endif
Require(block != nullptr, "dispatch move: cannot allocate the guest block");
GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true);
}
~GuestBlock() {
GuestAllocations::Mutation().Remove(block);
#ifdef _WIN32
GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes);
GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes);
#else
munmap(block, BlockBytes);
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes);
#endif
}
GuestBlock(const GuestBlock&) = delete;
GuestBlock& operator=(const GuestBlock&) = delete;
std::uint32_t* InputA() const { return reinterpret_cast<std::uint32_t*>(block); }
std::uint32_t* InputB() const { return reinterpret_cast<std::uint32_t*>(block + Piece + 12); }
std::uint32_t* InputC() const { return reinterpret_cast<std::uint32_t*>(block + 2 * Piece + 32); }
std::uint32_t* OutputO() const { return reinterpret_cast<std::uint32_t*>(block + 3 * Piece); }
std::uint32_t* OutputP() const { return reinterpret_cast<std::uint32_t*>(block + 4 * Piece); }
private:
std::uint8_t* block = nullptr;
};
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
const auto address = reinterpret_cast<std::uintptr_t>(data);
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
}
std::string Hex(std::uint32_t value) {
char text[16];
std::snprintf(text, sizeof(text), "0x%08x", value);
return text;
}
void Fill(std::uint32_t* where, std::uint32_t first) {
for (std::uint32_t index = 0; index < Words; ++index) where[index] = first + index;
}
void Reset(std::uint32_t* output) {
std::fill(output, output + Words, Sentinel);
}
ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, const std::uint32_t* input, std::uint32_t inputBytes, const std::uint32_t* output) {
const auto in = BufferDescriptor(input, inputBytes);
const auto out = BufferDescriptor(output, Bytes);
std::copy(in.begin(), in.end(), Table.begin());
std::copy(out.begin(), out.end(), Table.begin() + 4);
const auto table = reinterpret_cast<std::uintptr_t>(Table.data());
const std::vector<std::uint32_t> userData{static_cast<std::uint32_t>(table), static_cast<std::uint32_t>(table >> 32u)};
const std::span<const std::uint32_t> code(Code);
const std::span<const std::uint32_t> tableWords(Table);
const std::array<ShaderRecompiler::MemoryRegion, 2> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}, {table, std::as_bytes(tableWords)}}};
const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
ShaderRecompiler::RecompileRequest request{
{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
{32, 0, userData, compute, std::nullopt, std::nullopt, memory},
device.Target(),
{0, 0, 0, 128}
};
request.useCache = false;
return ShaderRecompiler::Recompile(request);
}
void Run(AgcDriver::VulkanDevice& device, const ShaderRecompiler::RecompileResult& compiled, const std::uint32_t* input, std::uint32_t inputBytes, const std::uint32_t* output, bool wait) {
auto result = compiled;
std::size_t elements = 0;
for (auto& binding : result.bindings) {
if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue;
Require(binding.guestDescriptor.size() == static_cast<std::size_t>(binding.count) * 4u && binding.bufferWritten.size() == binding.count, "dispatch move: the program's buffer bindings have another shape");
for (std::uint32_t element = 0; element < binding.count; ++element, ++elements) {
const auto words = binding.bufferWritten[element] ? BufferDescriptor(output, Bytes) : BufferDescriptor(input, inputBytes);
std::copy(words.begin(), words.end(), binding.guestDescriptor.begin() + static_cast<std::ptrdiff_t>(element) * 4);
}
}
Require(elements == 2, "dispatch move: the program does not bind an input and an output");
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(Code.data()));
if (wait) device.WaitIdle();
}
void Expect(const std::uint32_t* input, const std::uint32_t* output, std::uint32_t added, const std::string& what) {
for (std::uint32_t tid = 0; tid < Threads; ++tid) {
const auto expected = input[tid] + added;
Require(output[tid] == expected, "dispatch move: " + what + ": lane " + std::to_string(tid) + " is " + Hex(output[tid]) + ", expected " + Hex(expected));
}
}
struct Step {
std::uint64_t checked, exact, moved, refused, rebuilt, kept, served;
};
Step Since(const AgcDriver::VulkanDevice::DispatchMoveCount& before) {
const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts();
return {now.checked - before.checked, now.exact - before.exact, now.moved - before.moved, now.refused - before.refused, now.rebuilt - before.rebuilt, now.kept - before.kept, now.served - before.served};
}
void Print(const char* what, const Step& step) {
std::printf("dispatch move: %s: checked %llu, exact %llu, moved %llu, refused %llu\n", what, static_cast<unsigned long long>(step.checked), static_cast<unsigned long long>(step.exact), static_cast<unsigned long long>(step.moved), static_cast<unsigned long long>(step.refused));
}
bool RunTests(AgcDriver::VulkanDevice& device, const GuestBlock& guest) {
Fill(guest.InputA(), 0xa000u);
Fill(guest.InputB(), 0xb000u);
Fill(guest.InputC(), 0xc000u);
const auto compiled = Compile(device, guest.InputA(), Bytes, guest.OutputO());
Reset(guest.OutputO());
Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true);
Expect(guest.InputA(), guest.OutputO(), Constant, "the first dispatch");
const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts();
Reset(guest.OutputO());
Run(device, compiled, guest.InputB(), Bytes, guest.OutputO(), true);
Expect(guest.InputB(), guest.OutputO(), Constant, "an input moved to another block");
Reset(guest.OutputO());
Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true);
Expect(guest.InputA(), guest.OutputO(), Constant, "the input back where the template was built");
Reset(guest.OutputO());
Run(device, compiled, guest.InputC(), Bytes, guest.OutputO(), true);
Expect(guest.InputC(), guest.OutputO(), Constant, "an input moved to a third block");
const auto repeats = Since(start);
Print("an exact repeat between two moved ones", repeats);
const bool templates = repeats.checked != 0;
if (templates) {
Require(repeats.exact == 1 && repeats.moved == 2 && repeats.refused == 0, "dispatch move: the repeats were not served as one exact hit and two moved ones");
} else {
std::puts("dispatch move: the device kept no template (no host imports); the moved-buffer path is not exercised");
}
Reset(guest.OutputO());
Run(device, compiled, guest.InputB(), 192, guest.OutputO(), true);
Expect(guest.InputB(), guest.OutputO(), Constant, "an input moved and shortened");
std::array<std::uint32_t, Words> changed{};
for (std::uint32_t index = 0; index < Words; ++index) changed[index] = 0xd000u + index * 3u;
AgcDriver::GuestMemory::Write(reinterpret_cast<std::uintptr_t>(guest.InputB()), std::as_bytes(std::span(changed)));
Reset(guest.OutputO());
Run(device, compiled, guest.InputB(), Bytes, guest.OutputO(), true);
Expect(changed.data(), guest.OutputO(), Constant, "an input moved to a range the driver stored over");
const auto refusals = AgcDriver::VulkanDevice::DispatchMoveCounts();
for (std::uint32_t index = 0; index < Words; ++index) guest.OutputO()[index] = 0x7000u + index;
Run(device, compiled, guest.OutputO() + Threads, Bytes / 2, guest.OutputO(), true);
for (std::uint32_t tid = 0; tid < Threads; ++tid) {
const auto expected = 0x7000u + Threads + tid + Constant;
Require(guest.OutputO()[tid] == expected, "dispatch move: an input moved into the unwritten half of the dispatch's own output: lane " + std::to_string(tid) + " is " + Hex(guest.OutputO()[tid]) + ", expected " + Hex(expected));
}
Reset(guest.OutputO());
Reset(guest.OutputP());
Run(device, compiled, guest.InputA(), Bytes, guest.OutputP(), false);
Run(device, compiled, guest.OutputP(), Bytes, guest.OutputO(), true);
Expect(guest.InputA(), guest.OutputO(), 2u * Constant, "an input moved onto a range a recorded dispatch still writes");
const auto refused = Since(refusals);
Print("two inputs moved onto written ranges", refused);
if (templates) {
Require(refused.refused == 2, "dispatch move: the inputs moved onto written ranges were not both refused");
Require(refused.rebuilt == 2 && refused.kept == 2 && refused.served == 0, "dispatch move: the two refusals were not two rebuilds inserted beside the template");
}
const auto afterRefusals = AgcDriver::VulkanDevice::DispatchMoveCounts();
Reset(guest.OutputO());
Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true);
Expect(guest.InputA(), guest.OutputO(), Constant, "the input moved back after the refusals");
const auto back = Since(afterRefusals);
Print("the input back after the refusals", back);
if (templates) Require(back.checked == 1 && back.exact == 1 && back.moved == 0 && back.refused == 0 && back.rebuilt == 0, "dispatch move: the input back at the template's address was not an exact hit after the refusals");
for (int round = 0; round < 2; ++round) {
const auto alternating = AgcDriver::VulkanDevice::DispatchMoveCounts();
Reset(guest.OutputO());
Reset(guest.OutputP());
Run(device, compiled, guest.InputA(), Bytes, guest.OutputP(), false);
Run(device, compiled, guest.OutputP(), Bytes, guest.OutputO(), true);
Expect(guest.InputA(), guest.OutputO(), 2u * Constant, "an input refused again while A alternates");
Reset(guest.OutputO());
Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true);
Expect(guest.InputA(), guest.OutputO(), Constant, "the input at A after the refused one");
const auto step = Since(alternating);
Print("A alternating with a refused input", step);
if (templates) Require(step.refused == 1 && step.served == 1 && step.rebuilt == 0 && step.exact == 2 && step.moved == 0, "dispatch move: a refused input rebuilt, or evicted the template of A, while A alternated with it");
}
const auto total = AgcDriver::VulkanDevice::DispatchMoveCounts();
std::printf("dispatch move: in total: checked %llu, exact %llu, moved %llu, refused %llu, dropped by stage A %llu\n", static_cast<unsigned long long>(total.checked), static_cast<unsigned long long>(total.exact), static_cast<unsigned long long>(total.moved), static_cast<unsigned long long>(total.refused), static_cast<unsigned long long>(total.filtered));
return templates;
}
}
int main() {
try {
GuestBlock guest;
const auto device = OpenVulkanTestDevice();
if (!device) return VulkanTestSkipped;
if (!RunTests(*device, guest)) return VulkanTestSkipped;
std::puts("dispatch move tests passed");
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}
core/libs/prx/libSceAgcDriver/tests/execution/DispatchMoveDepth.cpp
+460
View File
@@ -0,0 +1,460 @@
#include "DepthFastClearHarness.hpp"
#include "VulkanTestDevice.hpp"
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include "prx/libc/include/GuestArena.hpp"
#include "prx/libc/include/GuestWriteWatch.hpp"
#include "Recompiler.hpp"
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#else
#include <sys/mman.h>
#endif
#include <algorithm>
#include <array>
#include <bit>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <span>
#include <string>
#include <vector>
#ifndef _WIN32
namespace AgcDriver {
struct VulkanDeviceTestAccess {
static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
};
}
#endif
namespace {
using AgcDriver::Graphics::Require;
using DepthFastClearHarness::Covered;
using DepthFastClearHarness::Draw;
using DepthFastClearHarness::Height;
using DepthFastClearHarness::Shaders;
using DepthFastClearHarness::Surface;
using DepthFastClearHarness::Width;
constexpr std::uint32_t Threads = 32;
constexpr std::uint32_t Bytes = Threads * 4u;
constexpr std::uint32_t Format32Float = 22;
constexpr std::uint32_t Type2D = 9;
constexpr std::uint32_t Sentinel = 0xdeadbeefu;
constexpr std::size_t Piece = 65536;
alignas(256) constexpr std::array<std::uint32_t, 13> RowCode{
0x7e3c0300u,
0x7e3e0280u,
0x34060082u,
0xe0301000u, 0x80040b03u,
0xbf8c3f70u,
0x4a181681u,
0xe0701000u, 0x80000c03u,
0xf0201108u, 0x00010b1eu,
0xbf810000u, 0xbf810000u,
};
alignas(256) constexpr std::array<std::uint32_t, 10> ProbeCode{
0x7e3c0300u,
0x7e3e020cu,
0xf0001108u, 0x00010a1eu,
0x34060082u,
0xbf8c3f70u,
0xe0701000u, 0x80000a03u,
0xbf810000u, 0xbf810000u,
};
class GuestBlock {
public:
GuestBlock(std::size_t pieces, bool inWatch) : bytes(pieces * Piece), watched(inWatch) {
if (inWatch) {
#ifdef _WIN32
block = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, Piece));
if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes);
#else
constexpr std::uintptr_t alignment = Piece;
void* mapped = mmap(nullptr, bytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (mapped != MAP_FAILED) {
const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
const auto aligned = (begin + alignment - 1) & ~(alignment - 1);
if (aligned != begin) munmap(mapped, aligned - begin);
if (aligned + bytes != begin + bytes + alignment) munmap(reinterpret_cast<void*>(aligned + bytes), begin + alignment - aligned);
block = reinterpret_cast<std::uint8_t*>(aligned);
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes);
}
#endif
} else {
#ifdef _WIN32
block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
#else
block = static_cast<std::uint8_t*>(std::aligned_alloc(Piece, bytes));
#endif
if (block != nullptr) std::memset(block, 0, bytes);
}
Require(block != nullptr, "dispatch move depth: cannot allocate the guest block");
GuestAllocations::Mutation().Add(block, bytes, true, true, true);
}
~GuestBlock() {
GuestAllocations::Mutation().Remove(block);
if (watched) {
#ifdef _WIN32
GuestArena::GuestArenaReset_nid_postfix(block, bytes);
GuestArena::GuestArenaRelease_nid_postfix(block, bytes);
#else
munmap(block, bytes);
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes);
#endif
} else {
#ifdef _WIN32
VirtualFree(block, 0, MEM_RELEASE);
#else
std::free(block);
#endif
}
}
GuestBlock(const GuestBlock&) = delete;
GuestBlock& operator=(const GuestBlock&) = delete;
std::uint8_t* PieceAt(std::size_t index) const { return block + index * Piece; }
private:
std::size_t bytes = 0;
bool watched = false;
std::uint8_t* block = nullptr;
};
struct Buffers {
explicit Buffers(const GuestBlock& block) : block(block) {}
std::uint32_t* InputA() const { return reinterpret_cast<std::uint32_t*>(block.PieceAt(4)); }
std::uint32_t* InputB() const { return reinterpret_cast<std::uint32_t*>(block.PieceAt(5) + 12); }
std::uint32_t* InputC() const { return reinterpret_cast<std::uint32_t*>(block.PieceAt(6) + 32); }
std::uint32_t* Output() const { return reinterpret_cast<std::uint32_t*>(block.PieceAt(8)); }
private:
const GuestBlock& block;
};
struct Env {
AgcDriver::VulkanDevice& device;
const Shaders& shaders;
const Buffers& buffers;
};
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
const auto address = reinterpret_cast<std::uintptr_t>(data);
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
}
std::array<std::uint32_t, 8> TextureDescriptor(const void* data) {
const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
return {
static_cast<std::uint32_t>(address >> 8u),
static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (Format32Float << 20u) | (((Width - 1u) & 3u) << 30u),
((Width - 1u) >> 2u) | ((Height - 1u) << 14u),
0xfacu | (Type2D << 28u),
0u, 0u, 0u, 0u,
};
}
std::string Hex(std::uint32_t value) {
char text[16];
std::snprintf(text, sizeof(text), "0x%08x", value);
return text;
}
ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData) {
const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
ShaderRecompiler::RecompileRequest request{
{ShaderRecompiler::ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
{32, 0, userData, compute, std::nullopt, std::nullopt, memory},
device.Target(),
{0, 0, 0, 128}
};
request.useCache = false;
return ShaderRecompiler::Recompile(request);
}
ShaderRecompiler::RecompileResult RowProgram(AgcDriver::VulkanDevice& device, const void* plane, const std::uint32_t* input, const std::uint32_t* output) {
std::vector<std::uint32_t> userData(20, 0u);
const auto out = BufferDescriptor(output, Bytes);
const auto texture = TextureDescriptor(plane);
const auto in = BufferDescriptor(input, Bytes);
std::copy(out.begin(), out.end(), userData.begin());
std::copy(texture.begin(), texture.end(), userData.begin() + 4);
std::copy(in.begin(), in.end(), userData.begin() + 16);
return Compile(device, RowCode, userData);
}
void RunRow(AgcDriver::VulkanDevice& device, const ShaderRecompiler::RecompileResult& compiled, const std::uint32_t* input, const std::uint32_t* output) {
auto result = compiled;
std::size_t elements = 0;
for (auto& binding : result.bindings) {
if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue;
Require(binding.guestDescriptor.size() == static_cast<std::size_t>(binding.count) * 4u, "dispatch move depth: the program's buffer bindings have another shape");
for (std::uint32_t element = 0; element < binding.count; ++element, ++elements) {
auto* current = binding.guestDescriptor.data() + static_cast<std::size_t>(element) * 4u;
const auto built = static_cast<std::uint64_t>(current[0]) | (static_cast<std::uint64_t>(current[1] & 0xffffu) << 32u);
const auto words = built == static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(output)) ? BufferDescriptor(output, Bytes) : BufferDescriptor(input, Bytes);
std::copy(words.begin(), words.end(), current);
}
}
Require(elements == 2, "dispatch move depth: the program does not bind an input and an output");
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(RowCode.data()));
device.WaitIdle();
}
std::array<std::uint32_t, Threads> ReadRow(const Env& env, const void* plane, std::uint32_t y) {
std::vector<std::uint32_t> userData(16, 0u);
auto* output = env.buffers.Output();
const auto out = BufferDescriptor(output, Bytes);
const auto texture = TextureDescriptor(plane);
std::copy(out.begin(), out.end(), userData.begin());
std::copy(texture.begin(), texture.end(), userData.begin() + 4);
userData[12] = y;
const auto program = Compile(env.device, ProbeCode, userData);
std::fill(output, output + Threads, Sentinel);
env.device.Dispatch(program, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(ProbeCode.data()));
env.device.WaitIdle();
std::array<std::uint32_t, Threads> lanes{};
std::copy(output, output + Threads, lanes.begin());
return lanes;
}
void FillInput(std::uint32_t* where, float first) {
for (std::uint32_t lane = 0; lane < Threads; ++lane) where[lane] = std::bit_cast<std::uint32_t>(first + static_cast<float>(lane) / 1024.0f);
}
void FillPlane(std::uint8_t* plane, float value) {
auto* words = reinterpret_cast<std::uint32_t*>(plane);
std::fill(words, words + Piece / sizeof(std::uint32_t), std::bit_cast<std::uint32_t>(value));
}
void DrawDepth(const Env& env, const void* plane, float depth) {
const Surface surface{reinterpret_cast<std::uintptr_t>(plane), 0, 0, false, VK_FORMAT_D32_SFLOAT};
const float low = depth <= 0.5f ? 0.0f : 2.0f * depth - 1.0f;
const float high = depth <= 0.5f ? 2.0f * depth : 1.0f;
Require(Draw(env.device, env.shaders, surface, {.depthWrite = true, .depthCompare = VK_COMPARE_OP_ALWAYS, .minDepth = low, .maxDepth = high}) == Covered, "dispatch move depth: the depth producer did not cover the surface");
}
struct PlaneModel {
std::array<std::uint32_t, Threads> row0{};
std::uint32_t rest = 0;
void Set(float value) {
rest = std::bit_cast<std::uint32_t>(value);
row0.fill(rest);
}
std::array<std::uint32_t, Threads> Dispatch(const std::uint32_t* input) {
std::array<std::uint32_t, Threads> sums{};
for (std::uint32_t lane = 0; lane < Threads; ++lane) {
sums[lane] = input[lane] + 1u;
row0[lane] = input[lane];
}
return sums;
}
};
void Totals(const std::string& what);
void Step(const Env& env, const ShaderRecompiler::RecompileResult& program, PlaneModel& model, const std::uint32_t* input, const std::string& what) {
const auto expected = model.Dispatch(input);
auto* output = env.buffers.Output();
std::fill(output, output + Threads, Sentinel);
RunRow(env.device, program, input, output);
for (std::uint32_t lane = 0; lane < Threads; ++lane) {
Require(output[lane] == expected[lane], "dispatch move depth: " + what + ": lane " + std::to_string(lane) + " is " + Hex(output[lane]) + ", expected " + Hex(expected[lane]));
}
Totals(what);
}
void Verify(const Env& env, const void* plane, const PlaneModel& model, const std::string& what) {
for (const std::uint32_t y : {0u, 1u, Height - 1u}) {
const auto lanes = ReadRow(env, plane, y);
for (std::uint32_t lane = 0; lane < Threads; ++lane) {
const auto expected = y == 0 ? model.row0[lane] : model.rest;
Require(lanes[lane] == expected, "dispatch move depth: " + what + ": row " + std::to_string(y) + ", texel " + std::to_string(lane) + " of the plane is " + Hex(lanes[lane]) + ", expected " + Hex(expected));
}
}
}
struct MoveSteps {
std::uint64_t checked, exact, moved, refused;
};
MoveSteps Since(const AgcDriver::VulkanDevice::DispatchMoveCount& before) {
const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts();
return {now.checked - before.checked, now.exact - before.exact, now.moved - before.moved, now.refused - before.refused};
}
void Totals(const std::string& what) {
const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts();
std::printf("dispatch move depth: %s: in total checked %llu, exact %llu, moved %llu, refused %llu, dropped by stage A %llu\n", what.c_str(), static_cast<unsigned long long>(now.checked), static_cast<unsigned long long>(now.exact), static_cast<unsigned long long>(now.moved), static_cast<unsigned long long>(now.refused), static_cast<unsigned long long>(now.filtered));
}
void DescribeBindings(const ShaderRecompiler::RecompileResult& program) {
for (const auto& binding : program.bindings) {
std::string written;
for (const bool bit : binding.bufferWritten) written += bit ? '1' : '0';
std::printf("dispatch move depth: binding role %d kind %d count %u, %zu descriptor words, buffers written [%s], images written %zu\n", static_cast<int>(binding.role), static_cast<int>(binding.kind), binding.count, binding.guestDescriptor.size(), written.c_str(), binding.imageWritten.size());
}
}
void Print(const char* what, const MoveSteps& step) {
std::printf("dispatch move depth: %s: checked %llu, exact %llu, moved %llu, refused %llu\n", what, static_cast<unsigned long long>(step.checked), static_cast<unsigned long long>(step.exact), static_cast<unsigned long long>(step.moved), static_cast<unsigned long long>(step.refused));
}
void Warm(const Env& env, std::uint8_t* plane) {
FillPlane(plane, 0.5f);
PlaneModel model;
model.Set(0.5f);
const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output());
Step(env, program, model, env.buffers.InputA(), "warm-up: the first input");
Step(env, program, model, env.buffers.InputB(), "warm-up: the second input");
Step(env, program, model, env.buffers.InputC(), "warm-up: the third input");
}
bool Control(const Env& env, std::uint8_t* plane) {
FillPlane(plane, 0.5f);
PlaneModel model;
model.Set(0.5f);
const auto refusals = AgcDriver::Graphics::ShaderResources::DepthGuardRefusals();
const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output());
DescribeBindings(program);
Step(env, program, model, env.buffers.InputA(), "control: the first dispatch");
Totals("control, after the first dispatch");
const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts();
Step(env, program, model, env.buffers.InputB(), "control: an input moved");
Totals("control, after an input moved");
Step(env, program, model, env.buffers.InputC(), "control: the input moved again");
Totals("control, after the input moved again");
Step(env, program, model, env.buffers.InputC(), "control: an exact repeat");
Totals("control, after an exact repeat");
const auto steps = Since(start);
Print("control, a plane without a depth surface", steps);
Verify(env, plane, model, "control");
Require(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() == refusals, "dispatch move depth: the depth guard refused a proof with no depth surface in sight");
Require(steps.refused == 0, "dispatch move depth: the control's repeats were refused");
if (steps.checked == 0) return false;
Require(steps.checked == 3 && steps.moved == 3, "dispatch move depth: the control's repeats were not all served by the template of the first dispatch");
return true;
}
void Mirror(const Env& env, std::uint8_t* plane, const char* where) {
const std::string at = std::string(where) + ", the mirror: ";
FillPlane(plane, 0.5f);
PlaneModel model;
DrawDepth(env, plane, 0.25f);
model.Set(0.25f);
const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output());
Step(env, program, model, env.buffers.InputA(), at + "the first dispatch");
Verify(env, plane, model, at + "after the first dispatch");
DrawDepth(env, plane, 0.75f);
model.Set(0.75f);
const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts();
Step(env, program, model, env.buffers.InputB(), at + "an input moved after the depth was drawn again");
Verify(env, plane, model, at + "after the moved dispatch");
Step(env, program, model, env.buffers.InputB(), at + "an exact repeat");
DrawDepth(env, plane, 0.25f);
model.Set(0.25f);
Step(env, program, model, env.buffers.InputC(), at + "the input moved again after another redraw");
Verify(env, plane, model, at + "after the last dispatch");
Print((at + "repeats and redraws").c_str(), Since(start));
}
void Appears(const Env& env, std::uint8_t* plane, const char* where, bool moved, bool templates) {
const std::string at = std::string(where) + (moved ? ", a surface appears, input moved: " : ", a surface appears, exact repeat: ");
FillPlane(plane, 0.125f);
PlaneModel model;
model.Set(0.125f);
const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output());
Step(env, program, model, env.buffers.InputA(), at + "the first dispatch, no depth surface yet");
DrawDepth(env, plane, 0.25f);
model.Set(0.25f);
const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts();
const auto refusals = AgcDriver::Graphics::ShaderResources::DepthGuardRefusals();
Step(env, program, model, moved ? env.buffers.InputB() : env.buffers.InputA(), at + "the dispatch after the surface appeared");
std::printf("dispatch move depth: %sthe depth guard refused %llu proofs\n", at.c_str(), static_cast<unsigned long long>(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() - refusals));
if (templates) Require(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() > refusals, "dispatch move depth: " + at + "the template over the plain storage image was proved current although a depth surface covers its memory");
Print((at + "the dispatch after the surface appeared").c_str(), Since(start));
Totals(at + "after that dispatch");
Verify(env, plane, model, at + "after that dispatch");
DrawDepth(env, plane, 0.75f);
model.Set(0.75f);
Step(env, program, model, env.buffers.InputC(), at + "the input moved after the next redraw");
Verify(env, plane, model, at + "after the last dispatch");
}
bool RunTests(const Env& env, const GuestBlock& watched, const GuestBlock& unwatched, std::size_t pass) {
const auto inWatch = pass * 10;
const auto outside = pass * 3;
FillInput(env.buffers.InputA(), 0.0625f);
FillInput(env.buffers.InputB(), 0.109375f);
FillInput(env.buffers.InputC(), 0.15625f);
Warm(env, watched.PieceAt(9));
const bool templates = Control(env, watched.PieceAt(inWatch));
if (!templates) std::puts("dispatch move depth: the device kept no template (no host imports); the moved-buffer path is not exercised");
Mirror(env, watched.PieceAt(inWatch + 1), "inside the write watch");
Appears(env, watched.PieceAt(inWatch + 2), "inside the write watch", true, templates);
Appears(env, watched.PieceAt(inWatch + 3), "inside the write watch", false, templates);
Mirror(env, unwatched.PieceAt(outside), "outside the write watch");
Appears(env, unwatched.PieceAt(outside + 1), "outside the write watch", true, templates);
Appears(env, unwatched.PieceAt(outside + 2), "outside the write watch", false, templates);
return templates;
}
}
int main() {
try {
const GuestBlock watched(20, true);
const GuestBlock unwatched(6, false);
const auto device = OpenVulkanTestDevice();
if (!device) return VulkanTestSkipped;
const auto shaders = DepthFastClearHarness::Compile(*device);
const Buffers buffers(watched);
const Env env{*device, shaders, buffers};
bool templates = true;
#ifndef _WIN32
const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(*device);
const auto decided = AgcDriver::Graphics::PrepareImportWatch(context);
struct Restore {
const AgcDriver::Graphics::Context& context;
AgcDriver::Graphics::ImportWatch decided;
~Restore() { AgcDriver::Graphics::SetImportWatch(context, decided); }
} restore{context, decided};
std::size_t pass = 0;
for (const auto mode : {AgcDriver::Graphics::ImportWatch::Watch, AgcDriver::Graphics::ImportWatch::Unwatch}) {
AgcDriver::Graphics::SetImportWatch(context, mode);
std::printf("dispatch move depth: pass %zu, imports %s\n", pass, mode == AgcDriver::Graphics::ImportWatch::Watch ? "under the write watch" : "left unwatched");
templates = RunTests(env, watched, unwatched, pass++) && templates;
}
#else
templates = RunTests(env, watched, unwatched, 0);
#endif
if (!templates) return VulkanTestSkipped;
std::puts("dispatch move depth tests passed");
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}