From f234476b217a02342efe7b9e34fc0e8366f7e2e0 Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Tue, 13 Oct 2026 06:14:41 -0400 Subject: [PATCH 1/6] feat(recompiler): plan waterfall loops over per-lane T#s as non-uniform tables A waterfall loop (s_ff1 or v_readfirstlane picks a lane, v_readlane copies its T# to SGPRs, v_cmpx_eq keeps the lanes whose T# VGPRs match, the image instruction runs, the matched lanes leave the loop) reads the T# from VGPRs, which no descriptor source evaluates, so planning failed with "GetImageResource dword 0 is not a valid runtime value". When every T# dword of an image read, sample, gather or query comes from v_readlane_b32 or v_readfirstlane_b32 and the EXEC its results are written under implies that each of them equals the lane's own VGPR, the access takes each lane's own T#. A T# that each lane loaded with buffer_load_dword* (IDXEN) from one evaluable V# becomes a bindless image table keyed by the lane's index, addressed by the V# stride, and its slot is decorated NonUniform. Co-Authored-By: Claude Opus 5.5 --- .../tests/execution/WaterfallImages.cpp | 175 ++++++++++ .../IrMetadata/ControlFlowInfo.hpp | 2 + .../Optimization/src/ResourceMaterializer.cpp | 15 +- .../Optimization/src/ResourceTracker.cpp | 304 +++++++++++++++++- .../src/SpirvMemory/SpirvDescriptors.cpp | 24 +- docs/dev/debt/shader.md | 1 + 6 files changed, 510 insertions(+), 11 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp new file mode 100644 index 0000000000..23ab9a56bb --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp @@ -0,0 +1,175 @@ +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using namespace ShaderRecompiler; + +constexpr std::uint32_t Records = 4; +constexpr std::uint32_t Textures = 3; + +struct alignas(4096) GuestData { + std::array, Records> table{}; + std::array output{}; +}; + +alignas(256) std::array, Textures> textures{}; +std::array guest; + +alignas(256) constexpr std::array WholeWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, + 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PartialWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x36200087u, 0xbe9c037eu, 0x7daa2087u, 0xe0382000u, + 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, + 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, + 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, + 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, + 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu, 0xe0702000u, + 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PartialWave64{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x36200087u, 0xbe9c047eu, 0x7daa2087u, 0xe0382000u, + 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94047eu, 0xbe961414u, 0xd7600008u, 0x00002d02u, + 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, + 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98047eu, 0x7da40408u, + 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, + 0x00020a0eu, 0xbe9a047eu, 0x8a941a14u, 0xbefe0418u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe041cu, 0xe0702000u, + 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array StaleWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x7e040208u, 0x7e060209u, 0x7e08020au, 0x7e0a020bu, + 0x7e0c020cu, 0x7e0e020du, 0x7e10020eu, 0x7e12020fu, 0x36200087u, 0xbe9c037eu, 0x7daa2087u, 0xe0382000u, + 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbefe031cu, 0xbe94037eu, 0xbe961314u, 0xd7600008u, + 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, + 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, + 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, + 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu, + 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +std::array Buffer(const void* pointer, std::uint32_t stride, std::uint32_t records) { + const auto address = reinterpret_cast(pointer); + return {static_cast(address), static_cast(address >> 32u) | (stride << 16u), records, 0x01016facu}; +} + +std::array Image(std::uint32_t texture) { + const auto address = reinterpret_cast(textures[texture].data()); + const std::uint32_t format = texture == 1u ? 22u : 20u; + return {static_cast(address >> 8u), static_cast(address >> 40u) | (format << 20u) | (3u << 30u), 7u, 0x90000facu, 0u, 0u, 0u, 0u}; +} + +std::uint32_t TextureOf(std::uint32_t iteration, std::uint32_t key) { + return key == Records - 1u - iteration ? Textures : (key + iteration) % Textures; +} + +bool DecoratesNonUniform(const std::vector& words) { + for (std::size_t index = 5u; index < words.size();) { + const auto count = words[index] >> 16u; + if (count == 0u) return false; + if ((words[index] & 0xffffu) == spv::OpDecorate && count >= 3u && index + 2u < words.size() && words[index + 2u] == spv::DecorationNonUniform) return true; + index += count; + } + return false; +} + +RecompileRequest Request(const AgcDriver::VulkanDevice& device, std::span code, std::uint32_t wave, std::uint32_t lanes, const std::array& userData, const std::array& regions) { + const ShaderComputeStageInfo compute{{lanes, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + return {{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {wave, 0u, userData, compute, std::nullopt, std::nullopt, regions}, device.Target(), {0u, 0u, 0u, 128u}}; +} + +std::array UserData(GuestData& data, std::uint32_t lanes) { + const auto table = Buffer(data.table.data(), 32u, Records); + const auto output = Buffer(data.output.data(), 4u, lanes); + const auto fixed = Image(0u); + std::array userData{}; + std::copy(table.begin(), table.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4u); + std::copy(fixed.begin(), fixed.end(), userData.begin() + 8u); + return userData; +} + +void Run(AgcDriver::VulkanDevice& device, std::span code, std::uint32_t wave, bool partial, const std::string& what) { + for (std::uint32_t texture = 0u; texture < Textures; ++texture) { + for (std::uint32_t x = 0u; x < textures[texture].size(); ++x) textures[texture][x] = 0x1000u * (texture + 1u) + x; + } + const auto lanes = wave == 64u ? 64u : std::min(32u, device.Target().subgroupSize); + CompiledShaderArtifact artifact; + for (std::uint32_t iteration = 0u; iteration < guest.size(); ++iteration) { + auto& data = guest[iteration]; + for (std::uint32_t key = 0u; key < Records; ++key) { + const auto texture = TextureOf(iteration, key); + data.table[key] = texture == Textures ? std::array{} : Image(texture); + } + data.output.fill(0xdeadbeefu); + const auto userData = UserData(data, lanes); + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + const auto shader = Recompile(Request(device, code, wave, lanes, userData, regions)); + Require(DecoratesNonUniform(shader.spirv.Words()), what + ": the per-lane table slot is not decorated NonUniform"); + if (artifact.variantId == 0u) artifact = shader; + else Require(shader.cacheHit && shader.variantId == artifact.variantId, what + ": a new table changed the compiled artifact"); + device.Dispatch(shader, 1u, 1u, 1u); + device.SubmitRecorded(false); + } + device.WaitIdle(); + for (std::uint32_t iteration = 0u; iteration < guest.size(); ++iteration) { + for (std::uint32_t lane = 0u; lane < lanes; ++lane) { + const auto texture = TextureOf(iteration, lane % Records); + const bool skipped = partial && lane % 8u == 7u; + const auto expected = skipped || texture == Textures ? 0u : textures[texture][lane / 4u]; + const auto actual = guest[iteration].output[lane]; + Require(actual == expected, what + ": iteration " + std::to_string(iteration) + " lane " + std::to_string(lane) + " read " + std::to_string(actual) + ", expected " + std::to_string(expected)); + } + } +} + +void RunStale(AgcDriver::VulkanDevice& device) { + auto& data = guest[0]; + const auto userData = UserData(data, 32u); + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + auto request = Request(device, StaleWave32, 32u, std::min(32u, device.Target().subgroupSize), userData, regions); + request.useCache = false; + bool refused = false; + try { + static_cast(Recompile(request)); + } catch (const std::exception& error) { + refused = std::string_view(error.what()).find("GetImageResource dword 0 is not a valid runtime value") != std::string_view::npos; + } + Require(refused, "a waterfall over a table T# and a T# left in the VGPRs of lanes the load skipped was accepted"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, WholeWave32, 32u, false, "wave32 waterfall over the whole wave"); + Run(*device, PartialWave32, 32u, true, "wave32 waterfall under a partial EXEC"); + if (device->Target().subgroupSize >= 32u) Run(*device, PartialWave64, 64u, true, "wave64 waterfall under a partial EXEC"); + RunStale(*device); + std::cout << "waterfall image tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index f9df378d48..6e8cba248a 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -32,6 +32,8 @@ struct DescriptorSource { std::uint32_t keyArg = 0; std::uint32_t entryOffset = 0; bool hasMaterial = false; + bool laneKey = false; + bool indexed = false; bool operator==(const IndirectImage& other) const = default; }; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 37da1a30cf..fddb3ca566 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -333,8 +333,17 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con const ShaderBufferResource heap = decodeBufferDescriptor(heapValue); const std::uint64_t heapSize = heap.GetSize(); if (heap.Type() != 0u || heap.Base48() > std::numeric_limits::max() - heapSize) throw std::runtime_error("bindless heap buffer is invalid"); - if (heapSize != 0u && (heapSize < table.entryOffset || (heapSize - table.entryOffset) % TableEntryBytes != 0u)) throw std::runtime_error("bindless heap contains a partial descriptor"); - const auto entries = heapSize > table.entryOffset ? static_cast(std::min((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits::max())) : 0u; + std::uint64_t entryBytes = TableEntryBytes; + std::uint32_t entries = 0u; + if (table.indexed) { + if (heap.SwizzleEnabled() || heap.AddTid() || heap.OutOfBounds() > 1u) throw std::runtime_error("bindless image table indexed through a V# with swizzling, ADD_TID or an unbounded range check is not implemented"); + entryBytes = heap.Stride(); + if (entryBytes < static_cast(table.entryOffset) + TableEntryBytes) throw std::runtime_error("bindless heap record is smaller than its descriptor"); + entries = heap.NumRecords(); + } else { + if (heapSize != 0u && (heapSize < table.entryOffset || (heapSize - table.entryOffset) % TableEntryBytes != 0u)) throw std::runtime_error("bindless heap contains a partial descriptor"); + entries = heapSize > table.entryOffset ? static_cast(std::min((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits::max())) : 0u; + } const auto readWord = [&](std::uint64_t address, std::uint32_t& word) { if (!runtime.readMemory(runtime.userContext, address, &word)) { throw std::runtime_error("failed to read a bindless image table from memory"); @@ -392,7 +401,7 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con for (std::size_t i = 0u; i < keys.size(); ++i) { auto& candidate = candidates[i]; candidate.dwordCount = 8u; - const std::uint64_t address = heapBase + table.entryOffset + static_cast(keys[i]) * TableEntryBytes; + const std::uint64_t address = heapBase + table.entryOffset + static_cast(keys[i]) * entryBytes; for (std::uint32_t dword = 0u; dword < 8u; ++dword) readWord(address + dword * sizeof(std::uint32_t), candidate.dwords[dword]); if (nullImageDescriptor(candidate)) { ++paddedNull; diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index ee4eaee5f8..2deaaab35a 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -112,6 +112,7 @@ public: if (!m_program.Resources().srtPlanComplete) { fail("SRT plan is not ready"); } + FoldWaterfallDescriptors(); SplitDescriptorPhis(); PlanIndirectImages(); std::unordered_map blockIndices; @@ -150,7 +151,9 @@ public: plan.handle->ReplaceArgument(dword, plan.key); } for (const auto index : plan.memory) { - m_program.Resources().memoryInfo[index].planningOnly = true; + if (plan.planningReads) { + m_program.Resources().memoryInfo[index].planningOnly = true; + } } } std::erase_if(m_program.Metadata().dynamicReads, [&](IrValue* value) { @@ -184,6 +187,14 @@ private: std::array roots {}; std::array memory {}; std::array reads {}; + bool planningReads = true; + }; + + struct LaneTableRead { + IrValue* written = nullptr; + IrValue* other = nullptr; + IrValue* load = nullptr; + std::uint32_t component = 0; }; void MakeSource(const IrValue& handle, std::uint32_t width, bool sampler, bool sampleAdjust, DescriptorSource& descriptor) { @@ -590,6 +601,291 @@ private: return true; } + static bool MatchLaneTableRead(IrValue* value, LaneTableRead& read) { + value = value->Resolve(); + read = LaneTableRead {}; + if (value->Opcode() == IrOpcode::SelectU32 && value->ArgumentCount() == 3u) { + read.written = value->Argument(0)->Resolve(); + read.other = value->Argument(2)->Resolve(); + value = value->Argument(1)->Resolve(); + } + const auto extract = value->Opcode(); + const bool extracted = extract == IrOpcode::CompositeExtractU32x2 || extract == IrOpcode::CompositeExtractU32x3 || extract == IrOpcode::CompositeExtractU32x4; + if (extracted) { + if (value->ArgumentCount() != 2u || !immediateU32(value->Argument(1), read.component)) { + return false; + } + value = value->Argument(0)->Resolve(); + } + const auto load = value->Opcode(); + const bool wide = load == IrOpcode::LoadBufferU32x2 || load == IrOpcode::LoadBufferU32x3 || load == IrOpcode::LoadBufferU32x4; + if ((wide != extracted) || (!wide && load != IrOpcode::LoadBufferU32) || value->ArgumentCount() != 5u) { + return false; + } + read.load = value; + return true; + } + + bool SameBufferAddress(const IrValue& left, const IrValue& right) const { + if (&left == &right) { + return true; + } + const IrValue* leftHandle = left.Argument(0)->Resolve(); + const IrValue* rightHandle = right.Argument(0)->Resolve(); + if (leftHandle != rightHandle) { + if (leftHandle->Opcode() != IrOpcode::GetBufferResource || rightHandle->Opcode() != IrOpcode::GetBufferResource || leftHandle->ArgumentCount() != 4u || rightHandle->ArgumentCount() != 4u) { + return false; + } + for (std::size_t dword = 0; dword < 4u; dword++) { + if (!EquivalentValue(m_program.Resources(), leftHandle->Argument(dword), rightHandle->Argument(dword))) { + return false; + } + } + } + for (std::size_t argument = 1; argument < 4u; argument++) { + if (!EquivalentValue(m_program.Resources(), left.Argument(argument), right.Argument(argument))) { + return false; + } + } + return true; + } + + bool TryMakeLaneTableImage(IrValue& handle, IndirectImagePlan& plan) { + if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u || handle.Parent() == nullptr) { + return false; + } + const auto& memoryInfo = m_program.Resources().memoryInfo; + std::array reads {}; + const MemoryInfo* first = nullptr; + std::uint64_t firstPosition = 0; + for (std::uint32_t dword = 0; dword < reads.size(); dword++) { + auto& read = reads[dword]; + if (!MatchLaneTableRead(handle.Argument(dword), read) || read.written != reads[0].written) { + return false; + } + const auto index = read.load->Flags().index; + if (index >= memoryInfo.size()) { + return false; + } + const auto& memory = memoryInfo[index]; + if (memory.kind != ResourceKind::Buffer || memory.dataBits != 32u || memory.formatted || memory.typed || !memory.idxen || read.component >= memory.dataDwords) { + return false; + } + const std::uint64_t position = static_cast(memory.offset) + sizeof(std::uint32_t) * read.component; + if (dword == 0u) { + first = &memory; + firstPosition = position; + } else if (memory.offen != first->offen || position != firstPosition + sizeof(std::uint32_t) * dword || !SameBufferAddress(*reads[0].load, *read.load)) { + return false; + } + } + const bool nullOther = std::ranges::all_of(reads, [](const LaneTableRead& read) { + return read.other == nullptr || (read.other->HasImmediate() && read.other->Type() == IrType::U32 && read.other->ImmediateU32() == 0u); + }); + if (!nullOther) { + constexpr SrtWalker walker; + bool descriptor = true; + bool table = true; + for (const auto& read : reads) { + LaneTableRead other; + descriptor = descriptor && read.other->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), read.other); + table = table && MatchLaneTableRead(read.other, other); + } + if (descriptor || table) { + return false; + } + } + + const IrValue& load = *reads[0].load; + std::uint32_t voffset = 0; + std::uint32_t soffset = 0; + if (!immediateU32(load.Argument(3), soffset) || (first->offen && !immediateU32(load.Argument(2), voffset))) { + return false; + } + const std::uint64_t entryOffset = firstPosition + voffset + soffset; + IrValue* key = load.Argument(1)->Resolve(); + if (entryOffset > std::numeric_limits::max() || key->Type() != IrType::U32) { + return false; + } + DescriptorSource heapSource; + std::uint32_t heapSourceIndex = 0; + if (!MakeRuntimeBufferSource(*load.Argument(0)->Resolve(), heapSourceIndex, heapSource)) { + return false; + } + if (reads[0].written != nullptr) { + const std::array arguments {reads[0].written, key, &m_builder.Constant(std::numeric_limits::max())}; + key = EmitBefore(handle, IrOpcode::SelectU32, IrType::U32, arguments); + } + + DescriptorSource::IndirectImage table; + table.heapSource = heapSourceIndex; + table.materialSource = heapSourceIndex; + table.entryOffset = static_cast(entryOffset); + table.laneKey = true; + table.indexed = true; + DescriptorSource imageSource; + imageSource.dwordCount = 8u; + std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin()); + std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin() + 4u); + imageSource.indirectImage = table; + + plan.handle = &handle; + plan.source = InternSource(imageSource); + plan.key = key; + plan.roots = imageSource.dwords; + plan.planningReads = false; + return true; + } + + static bool LaneRead(const IrValue& value) { + return (value.Opcode() == IrOpcode::ReadLane || value.Opcode() == IrOpcode::ReadFirstLane) && value.ArgumentCount() == 2u; + } + + static bool FoldableImageRead(IrOpcode op) { + switch (op) { + case IrOpcode::ImageSampleRaw: + case IrOpcode::ImageGatherRaw: + case IrOpcode::ImageQueryLod: + case IrOpcode::ImageQueryDimensions: + case IrOpcode::ImageRead: return true; + default: return false; + } + } + + static IrValue* ObservingCondition(const IrValue& inst) { + IrValue* condition = nullptr; + std::vector pending {&inst}; + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + for (const IrUse& use : value->OperandUses()) { + const IrValue* user = use.user; + if (user->IsIdentity() || (user->Opcode() == IrOpcode::CompositeExtractU32x4 && use.operand == 0u)) { + pending.push_back(user); + continue; + } + if (user->Opcode() != IrOpcode::SelectU32 || user->ArgumentCount() != 3u || use.operand != 1u) { + return nullptr; + } + IrValue* selector = user->Argument(0)->Resolve(); + if (condition != nullptr && selector != condition) { + return nullptr; + } + condition = selector; + } + } + return condition; + } + + using Equalities = std::vector>; + + static Equalities ImpliedEqualities(IrValue* condition) { + constexpr std::size_t limit = 1024u; + Equalities equal; + std::vector> pending {{condition, true}}; + for (std::size_t visited = 0; !pending.empty() && visited < limit; visited++) { + auto [value, holds] = pending.back(); + pending.pop_back(); + value = value->Resolve(); + const auto op = value->Opcode(); + if (op == IrOpcode::LogicalNot && value->ArgumentCount() == 1u) { + pending.emplace_back(value->Argument(0), !holds); + } else if (((op == IrOpcode::LogicalAnd && holds) || (op == IrOpcode::LogicalOr && !holds)) && value->ArgumentCount() == 2u) { + pending.emplace_back(value->Argument(0), holds); + pending.emplace_back(value->Argument(1), holds); + } else if (((op == IrOpcode::IEqual32 && holds) || (op == IrOpcode::INotEqual32 && !holds)) && value->ArgumentCount() == 2u) { + equal.emplace_back(value->Argument(0)->Resolve(), value->Argument(1)->Resolve()); + } else if (((op == IrOpcode::IEqual64 && holds) || (op == IrOpcode::INotEqual64 && !holds)) && value->ArgumentCount() == 2u) { + const IrValue* left = value->Argument(0)->Resolve(); + const IrValue* right = value->Argument(1)->Resolve(); + if (left->Opcode() != IrOpcode::CompositeConstructU64 || right->Opcode() != IrOpcode::CompositeConstructU64 || left->ArgumentCount() != 2u || right->ArgumentCount() != 2u) { + continue; + } + for (std::size_t half = 0; half < 2u; half++) { + equal.emplace_back(left->Argument(half)->Resolve(), right->Argument(half)->Resolve()); + } + } + } + return equal; + } + + void FoldLaneReads(IrValue& inst, std::uint32_t slot, IrOpcode expected, std::size_t width, const Equalities& equal) { + IrValue* handle = inst.Argument(slot)->Resolve(); + if (handle->Opcode() != expected || handle->ArgumentCount() != width) { + return; + } + constexpr SrtWalker walker; + std::vector dwords; + bool valid = true; + bool folded = false; + for (std::size_t dword = 0; dword < width; dword++) { + IrValue* value = handle->Argument(dword)->Resolve(); + valid = valid && value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value); + if (!LaneRead(*value)) { + dwords.push_back(value); + continue; + } + IrValue* source = value->Argument(0)->Resolve(); + const bool proven = std::ranges::any_of(equal, [&](const auto& pair) { + return (pair.first == value && pair.second == source) || (pair.first == source && pair.second == value); + }); + if (!proven) { + return; + } + dwords.push_back(source); + folded = true; + } + if (valid || !folded) { + return; + } + IrValue* copy = EmitCopy(*handle, dwords, inst); + bool accepted = std::ranges::all_of(dwords, [&](IrValue* value) { + return value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value); + }); + static const bool tables = std::getenv("APS5_NO_BINDLESS_IMAGES") == nullptr; + IndirectImagePlan plan; + if (!accepted && tables && expected == IrOpcode::GetImageResource && TryMakeLaneTableImage(*copy, plan)) { + m_indirectImages.push_back(std::move(plan)); + accepted = true; + } + if (!accepted) { + copy->Invalidate(); + copy->Parent()->RemoveInstruction(copy); + return; + } + inst.ReplaceArgument(slot, copy); + m_foldedHandles.push_back(copy); + } + + void FoldWaterfallDescriptors() { + std::vector candidates; + for (auto& block : m_program.Blocks()) { + for (IrValue* inst : block->Instructions()) { + if (FoldableImageRead(inst->Opcode())) { + candidates.push_back(inst); + } + } + } + for (IrValue* inst : candidates) { + const bool sampler = ImageOpcodeInfoOf(inst->Opcode()).needsSampler; + if (inst->Parent() == nullptr || inst->ArgumentCount() < (sampler ? 2u : 1u)) { + continue; + } + IrValue* condition = ObservingCondition(*inst); + if (condition == nullptr) { + continue; + } + const Equalities equal = ImpliedEqualities(condition); + if (equal.empty()) { + continue; + } + FoldLaneReads(*inst, 0u, IrOpcode::GetImageResource, 8u, equal); + if (sampler) { + FoldLaneReads(*inst, 1u, IrOpcode::GetSamplerResource, 4u, equal); + } + } + } + const IndirectImagePlan* FindIndirectImage(const IrValue& handle) const { const auto found = std::ranges::find_if(m_indirectImages, [&](const IndirectImagePlan& plan) { return plan.handle == &handle; @@ -599,7 +895,7 @@ private: bool IsIndirectPlanningMemory(std::uint32_t index) const { return std::ranges::any_of(m_indirectImages, [&](const IndirectImagePlan& plan) { - return std::ranges::find(plan.memory, index) != plan.memory.end(); + return plan.planningReads && std::ranges::find(plan.memory, index) != plan.memory.end(); }); } @@ -1117,6 +1413,9 @@ private: } } for (IrValue* inst : candidates) { + if (std::ranges::find(m_foldedHandles, inst->Argument(0)->Resolve()) != m_foldedHandles.end()) { + continue; + } SplitDescriptorPhi(*inst); } } @@ -1494,6 +1793,7 @@ private: std::vector m_indirectImages; std::unordered_map> m_memoryUsers; bool m_memoryUsersIndexed = false; + std::vector m_foldedHandles; struct EdgeSelectorEntry { IrBlock* block = nullptr; std::vector edges; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp index 4989ed437a..ee2e467613 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp @@ -72,6 +72,18 @@ std::uint32_t RuntimeMipCount(SpirvEmitterState& state, std::uint32_t resource) return Binary(state, spv::OpIAdd, TypeU32(state), Binary(state, spv::OpISub, TypeU32(state), last, first), ConstantU32(state, 1u)); } +namespace { + +bool TableKeyPerLane(const SpirvEmitterState& state, std::uint32_t resource) { + const auto& images = state.program.Info().images; + const auto& sources = state.program.Resources().descriptorSources; + if (resource >= images.size() || images[resource].source >= sources.size()) return false; + const auto& table = sources[images[resource].source].indirectImage; + return table.has_value() && table->laneKey; +} + +} + std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource) { const bool storage = ImageBindingResourceClass(kind) == ImageResourceClass::Storage; const auto supported = [&](std::uint32_t capability) { @@ -88,7 +100,7 @@ std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind ki state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), metadata, state.shaderDataStorageVariable, ConstantU32(state, 0u), offset); const auto index = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, TypeU32(state), index, metadata); - if (!state.tableIndexNonUniform) { + if (!state.tableIndexNonUniform && !TableKeyPerLane(state, resource)) { return index; } const auto nonUniform = storage ? spv::CapabilityStorageImageArrayNonUniformIndexing : spv::CapabilitySampledImageArrayNonUniformIndexing; @@ -104,8 +116,8 @@ std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind ki return index; } -void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t slotId, std::uint32_t id) { - if (slotId != 0 && state.tableIndexNonUniform) state.module.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform); +void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t slotId, std::uint32_t id) { + if (slotId != 0 && (state.tableIndexNonUniform || TableKeyPerLane(state, resource))) state.module.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform); } std::uint32_t ImageScalarType(SpirvEmitterState& state, IrTextureNumericClass numericClass) { @@ -184,11 +196,11 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t } else { pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, variable, TableImageIndex(state, kind, resource)); - DecorateTableNonUniform(state, slotId, pointer); + DecorateTableNonUniform(state, resource, slotId, pointer); } const auto image = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, imageType, image, pointer); - DecorateTableNonUniform(state, slotId, image); + DecorateTableNonUniform(state, resource, slotId, image); return image; } @@ -210,7 +222,7 @@ std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, const auto sampledImage = state.module.AllocateId(); const auto sampledType = state.module.Type(spv::OpTypeSampledImage, ImageType(state, imageResource)); state.module.AddFunction(spv::OpSampledImage, sampledType, sampledImage, image, samplerId); - DecorateTableNonUniform(state, slotId, sampledImage); + DecorateTableNonUniform(state, resource, slotId, sampledImage); return sampledImage; } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index f8f267fd2c..deffd1c11b 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -43,6 +43,7 @@ - [8_SRGB and 8_8_SRGB loads](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) on a device that cannot sample R8_SRGB or R8G8_SRGB return the sRGB EOTF rounded to nearest even at bf16 precision, as adriadam10 measured on RDNA2 (gfx1035) in #866; the PS5's gfx1013 is not measured. - [Samples and gathers](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB. +- A [waterfall loop](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) over T#s is compiled as one access per lane only when each lane loaded its T# with `buffer_load_dword*` (IDXEN, immediate VOFFSET and SOFFSET) from one V# that the SRT walker evaluates: the access becomes a [bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) keyed by the lane's index, with the NonUniform decoration. The loop is recognised when every T# dword of an image read, sample, gather or query comes from `v_readlane_b32` or `v_readfirstlane_b32` and the EXEC its results are written under implies that each of them equals the lane's own VGPR. Waterfalls over other per-lane T#s, over S#s, around image stores or atomics, and over T# VGPRs that hold another descriptor in lanes the load skipped still throw. A lane the load skipped whose T# VGPRs hold any other value reads zeros instead of using that value as a T#. The table V# must be structured (stride at least the record offset plus 32 bytes, `OOB_SELECT` 0 or 1) without swizzling or `ADD_TID`, else the dispatch throws, and like other bindless tables it binds at most `BindlessTableSlots` records. None of this was measured on hardware. - [Shader MODE register access](../../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) reads the initial rounding fields from `GuestContext::floatMode`; missing metadata retains the legacy round-to-nearest-even assumption. Immediate rounding writes are accepted only when they preserve that state. State-changing or SGPR-sourced MODE writes throw until runtime mode tracking and affected arithmetic are implemented. Non-rounding register reads still throw; `s_setreg_b32` writes to other hardware registers are still ignored. Tests follow the public RDNA2/LLVM contract, not console measurements. - A [flat SRT read](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read. - Repeated `s_load`s of one [flat SRT](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own. From 8bca46fb068ca71f72e3fe82132e70c9a752183e Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Tue, 13 Oct 2026 06:57:37 -0400 Subject: [PATCH 2/6] fix(recompiler): refuse per-lane waterfall tables over masked T#s or mixed modes A lane whose IDXEN load was skipped keeps the T# VGPRs it had before, and a waterfall that reaches it uses that value as its T#. The fold refused only a value that was itself a uniform descriptor or a table read, so a default T# copied into the VGPRs under EXEC, or merged by a phi, was accepted and those lanes read zeros. The skipped lanes' values are now followed through the masked selects and phis that all eight dwords share, and the fold is refused when an arm is a descriptor or a table read. Every lane of a per-lane table runs the access with its own record, and the runtime image mode picks the SPIR-V path per lane, so lanes of one quad whose records need different modes take different paths around an implicit-derivative sample or LOD query, where derivatives are undefined. Such a table now throws at dispatch; accesses of one table share the check, so they keep one table. The test also covers v_readfirstlane and v_cmpx_eq_u64 waterfalls and refuses loops that compare seven T# dwords, run the image instruction after EXEC is restored, or store. Co-Authored-By: Claude Opus 5.5 --- .../tests/execution/WaterfallImages.cpp | 107 ++++++++++++++++-- .../IrMetadata/ControlFlowInfo.hpp | 6 +- .../Optimization/src/ResourceMaterializer.cpp | 8 +- .../Optimization/src/ResourceTracker.cpp | 98 +++++++++++++--- docs/dev/debt/shader.md | 2 +- 5 files changed, 195 insertions(+), 26 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp index 23ab9a56bb..14e0496103 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp @@ -66,6 +66,71 @@ alignas(256) constexpr std::array StaleWave32{ 0xe0702000u, 0x80010a00u, 0xbf810000u, }; +alignas(256) constexpr std::array FirstLaneWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0x7e100502u, 0x7e120503u, 0x7e140504u, 0x7e160505u, 0x7e180506u, 0x7e1a0507u, + 0x7e1c0508u, 0x7e1e0509u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, + 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a7e1a18u, 0xbf85ffeau, + 0xbf8c3f70u, 0xbefe0314u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array Compare64Wave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7de40408u, 0x7de4080au, 0x7de40c0cu, 0x7de4100eu, + 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe4u, 0xbf8c3f70u, 0xe0702000u, + 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array MaskedStaleWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x7e040280u, 0x7e060280u, 0x7e080280u, 0x7e0a0280u, + 0x7e0c0280u, 0x7e0e0280u, 0x7e100280u, 0x7e120280u, 0x36200087u, 0xbe9c037eu, 0x7da42087u, 0x7e040208u, + 0x7e060209u, 0x7e08020au, 0x7e0a020bu, 0x7e0c020cu, 0x7e0e020du, 0x7e10020eu, 0x7e12020fu, 0xbefe031cu, + 0x7daa2087u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbefe031cu, 0xbe94037eu, + 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, + 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, + 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, + 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, + 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array SevenDwordsWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, + 0xbf85ffe1u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array OutsideWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xf0000f08u, + 0x00020a0eu, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array StoreWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0200f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, + 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array LodWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf1800308u, 0x00820a0eu, 0xbe9a037eu, 0x8a141a14u, + 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + std::array Buffer(const void* pointer, std::uint32_t stride, std::uint32_t records) { const auto address = reinterpret_cast(pointer); return {static_cast(address), static_cast(address >> 32u) | (stride << 16u), records, 0x01016facu}; @@ -91,19 +156,21 @@ bool DecoratesNonUniform(const std::vector& words) { return false; } -RecompileRequest Request(const AgcDriver::VulkanDevice& device, std::span code, std::uint32_t wave, std::uint32_t lanes, const std::array& userData, const std::array& regions) { +RecompileRequest Request(const AgcDriver::VulkanDevice& device, std::span code, std::uint32_t wave, std::uint32_t lanes, std::span userData, const std::array& regions) { const ShaderComputeStageInfo compute{{lanes, 1u, 1u}, 0u, {false, false, false}, false, 1u}; return {{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {wave, 0u, userData, compute, std::nullopt, std::nullopt, regions}, device.Target(), {0u, 0u, 0u, 128u}}; } -std::array UserData(GuestData& data, std::uint32_t lanes) { +std::array UserData(GuestData& data, std::uint32_t lanes) { const auto table = Buffer(data.table.data(), 32u, Records); const auto output = Buffer(data.output.data(), 4u, lanes); const auto fixed = Image(0u); - std::array userData{}; + constexpr std::array sampler{0x00000092u, 0x00fff000u, 0u, 0u}; + std::array userData{}; std::copy(table.begin(), table.end(), userData.begin()); std::copy(output.begin(), output.end(), userData.begin() + 4u); std::copy(fixed.begin(), fixed.end(), userData.begin() + 8u); + std::copy(sampler.begin(), sampler.end(), userData.begin() + 16u); return userData; } @@ -141,11 +208,11 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, s } } -void RunStale(AgcDriver::VulkanDevice& device) { +void RequireRefused(AgcDriver::VulkanDevice& device, std::span code, const std::string& what) { auto& data = guest[0]; const auto userData = UserData(data, 32u); const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; - auto request = Request(device, StaleWave32, 32u, std::min(32u, device.Target().subgroupSize), userData, regions); + auto request = Request(device, code, 32u, std::min(32u, device.Target().subgroupSize), userData, regions); request.useCache = false; bool refused = false; try { @@ -153,7 +220,25 @@ void RunStale(AgcDriver::VulkanDevice& device) { } catch (const std::exception& error) { refused = std::string_view(error.what()).find("GetImageResource dword 0 is not a valid runtime value") != std::string_view::npos; } - Require(refused, "a waterfall over a table T# and a T# left in the VGPRs of lanes the load skipped was accepted"); + Require(refused, what + " was accepted"); +} + +void RunLod(AgcDriver::VulkanDevice& device, bool mixed, const std::string& what) { + auto& data = guest[0]; + for (std::uint32_t key = 0u; key < Records; ++key) data.table[key] = Image(mixed && key % 2u == 0u ? 0u : 1u); + const auto userData = UserData(data, 32u); + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + auto request = Request(device, LodWave32, 32u, std::min(32u, device.Target().subgroupSize), userData, regions); + request.useCache = false; + std::string failure; + try { + const auto shader = Recompile(request); + Require(DecoratesNonUniform(shader.spirv.Words()), what + ": the per-lane table slot is not decorated NonUniform"); + } catch (const std::exception& error) { + failure = error.what(); + } + if (mixed) Require(failure.find("different runtime modes under an implicit-derivative access") != std::string::npos, what + " was not refused: " + failure.substr(0u, 300u)); + else Require(failure.empty(), what + " failed: " + failure.substr(0u, 300u)); } } @@ -165,7 +250,15 @@ int main() { Run(*device, WholeWave32, 32u, false, "wave32 waterfall over the whole wave"); Run(*device, PartialWave32, 32u, true, "wave32 waterfall under a partial EXEC"); if (device->Target().subgroupSize >= 32u) Run(*device, PartialWave64, 64u, true, "wave64 waterfall under a partial EXEC"); - RunStale(*device); + Run(*device, FirstLaneWave32, 32u, false, "wave32 waterfall picking lanes with v_readfirstlane"); + Run(*device, Compare64Wave32, 32u, false, "wave32 waterfall comparing the T# with v_cmpx_eq_u64"); + RequireRefused(*device, StaleWave32, "a waterfall over a table T# and a T# left in the VGPRs of lanes the load skipped"); + RequireRefused(*device, MaskedStaleWave32, "a waterfall over a table T# and a T# written under EXEC to the VGPRs of lanes the load skipped"); + RequireRefused(*device, SevenDwordsWave32, "a waterfall that compares seven of the eight T# dwords"); + RequireRefused(*device, OutsideWave32, "an image load after the waterfall restores EXEC"); + RequireRefused(*device, StoreWave32, "an image store in a waterfall"); + RunLod(*device, false, "an LOD query in a waterfall over a table of one image mode"); + RunLod(*device, true, "an LOD query in a waterfall over a table of float and integer images"); std::cout << "waterfall image tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 6e8cba248a..12d5018dc7 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -20,8 +20,9 @@ struct BlockInfo { }; struct DescriptorSource { - // A T# loaded from a table buffer (`heapSource`) at `entryOffset + key * 32`, the key a - // wave-uniform runtime value (handle argument `keyArg`). With `hasMaterial` the key is itself + // A T# loaded from a table buffer (`heapSource`) at `entryOffset + key * 32` (the table V#'s + // stride when `indexed`), the key a wave-uniform runtime value (handle argument `keyArg`), or + // each lane's own value when `laneKey`. With `hasMaterial` the key is itself // `M[readfirstlane(i) * selectorStride + selectorOffset]` over `materialSource`, so the key // set can be enumerated from the material records (see ResourceMaterializer). struct IndirectImage { @@ -34,6 +35,7 @@ struct DescriptorSource { bool hasMaterial = false; bool laneKey = false; bool indexed = false; + bool implicitDerivatives = false; bool operator==(const IndirectImage& other) const = default; }; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index fddb3ca566..33633b9791 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -397,6 +397,7 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con const std::uint64_t heapBase = heap.Base48(); std::vector candidates(keys.size()); std::vector valid(keys.size(), 0u); + std::vector modes(keys.size(), 0u); std::uint32_t paddedNull = 0u; for (std::size_t i = 0u; i < keys.size(); ++i) { auto& candidate = candidates[i]; @@ -410,9 +411,14 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con if (!validImageDescriptor(candidate, image.r128)) throw std::runtime_error("bindless image table contains an invalid descriptor"); const auto decoded = decodeImageDescriptor(candidate, image, plan.srgbDecodeFormats); if (decoded.fmask) throw std::runtime_error("bindless FMASK images are unsupported"); - static_cast(ResourceMaterializer::RuntimeImageMode(image, candidate, plan.info.runtimeImageModes.at(imageIndex))); + modes[i] = ResourceMaterializer::RuntimeImageMode(image, candidate, plan.info.runtimeImageModes.at(imageIndex)); valid[i] = 1u; } + if (table.laneKey && table.implicitDerivatives) { + for (std::size_t i = 1u; i < keys.size(); ++i) { + if (valid[i] != 0u && modes[i] != modes[0]) throw std::runtime_error("bindless image table keyed per lane holds images of different runtime modes under an implicit-derivative access, which is not implemented"); + } + } resolution.mapping.clear(); for (std::size_t i = 0u; i < keys.size(); ++i) { if (valid[i] != 0u) resolution.mapping.emplace_back(keys[i], static_cast(i)); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 2deaaab35a..749c249ad8 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -650,7 +650,63 @@ private: return true; } - bool TryMakeLaneTableImage(IrValue& handle, IndirectImagePlan& plan) { + bool MayHoldDescriptor(std::array tuple, std::uint32_t depth, std::vector>& visited) const { + for (IrValue*& value : tuple) { + value = value->Resolve(); + } + if (std::ranges::find(visited, tuple) != visited.end()) { + return false; + } + visited.push_back(tuple); + if (std::ranges::all_of(tuple, [](const IrValue* value) { return value->HasImmediate() && value->Type() == IrType::U32 && value->ImmediateU32() == 0u; })) { + return false; + } + constexpr SrtWalker walker; + LaneTableRead read; + const bool runtime = std::ranges::all_of(tuple, [&](const IrValue* value) { return value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value); }); + if (runtime || std::ranges::all_of(tuple, [&](IrValue* value) { return MatchLaneTableRead(value, read); })) { + return true; + } + if (depth == phiSearchDepth) { + return true; + } + const IrValue& first = *tuple[0]; + if (first.Opcode() == IrOpcode::SelectU32 && first.ArgumentCount() == 3u) { + const IrValue* condition = first.Argument(0)->Resolve(); + std::array selected {}; + std::array kept {}; + for (std::size_t dword = 0; dword < tuple.size(); dword++) { + const IrValue& value = *tuple[dword]; + if (value.Opcode() != IrOpcode::SelectU32 || value.ArgumentCount() != 3u || value.Argument(0)->Resolve() != condition) { + return false; + } + selected[dword] = value.Argument(1); + kept[dword] = value.Argument(2); + } + return MayHoldDescriptor(selected, depth + 1u, visited) || MayHoldDescriptor(kept, depth + 1u, visited); + } + if (!first.IsPhi()) { + return false; + } + for (std::size_t edge = 0; edge < first.ArgumentCount(); edge++) { + std::array incoming {}; + for (std::size_t dword = 0; dword < tuple.size(); dword++) { + if (!tuple[dword]->IsPhi() || tuple[dword]->Parent() != first.Parent()) { + return false; + } + incoming[dword] = PhiIncoming(*tuple[dword], first.PhiBlock(edge)); + if (incoming[dword] == nullptr) { + return true; + } + } + if (MayHoldDescriptor(incoming, depth + 1u, visited)) { + return true; + } + } + return false; + } + + bool TryMakeLaneTableImage(IrValue& handle, bool implicitDerivatives, IndirectImagePlan& plan) { if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u || handle.Parent() == nullptr) { return false; } @@ -679,19 +735,11 @@ private: return false; } } - const bool nullOther = std::ranges::all_of(reads, [](const LaneTableRead& read) { - return read.other == nullptr || (read.other->HasImmediate() && read.other->Type() == IrType::U32 && read.other->ImmediateU32() == 0u); - }); - if (!nullOther) { - constexpr SrtWalker walker; - bool descriptor = true; - bool table = true; - for (const auto& read : reads) { - LaneTableRead other; - descriptor = descriptor && read.other->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), read.other); - table = table && MatchLaneTableRead(read.other, other); - } - if (descriptor || table) { + if (reads[0].other != nullptr) { + std::array others {}; + std::ranges::transform(reads, others.begin(), &LaneTableRead::other); + std::vector> visited; + if (MayHoldDescriptor(others, 0u, visited)) { return false; } } @@ -723,10 +771,21 @@ private: table.entryOffset = static_cast(entryOffset); table.laneKey = true; table.indexed = true; + table.implicitDerivatives = implicitDerivatives; DescriptorSource imageSource; imageSource.dwordCount = 8u; std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin()); std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin() + 4u); + for (auto& known : m_sources) { + if (!known.indirectImage.has_value() || !known.indirectImage->laneKey) { + continue; + } + auto shared = *known.indirectImage; + shared.implicitDerivatives = table.implicitDerivatives; + if (shared == table) { + table.implicitDerivatives = known.indirectImage->implicitDerivatives = known.indirectImage->implicitDerivatives || table.implicitDerivatives; + } + } imageSource.indirectImage = table; plan.handle = &handle; @@ -737,6 +796,15 @@ private: return true; } + bool ImplicitDerivatives(const IrValue& inst) const { + if (inst.Opcode() == IrOpcode::ImageQueryLod) { + return true; + } + const auto& memoryInfo = m_program.Resources().memoryInfo; + const auto index = inst.Flags().index; + return inst.Opcode() == IrOpcode::ImageSampleRaw && (index >= memoryInfo.size() || !ImageSampleExplicitLod(memoryInfo[index].imageSampleFlags, m_program.Resources().stage)); + } + static bool LaneRead(const IrValue& value) { return (value.Opcode() == IrOpcode::ReadLane || value.Opcode() == IrOpcode::ReadFirstLane) && value.ArgumentCount() == 2u; } @@ -844,7 +912,7 @@ private: }); static const bool tables = std::getenv("APS5_NO_BINDLESS_IMAGES") == nullptr; IndirectImagePlan plan; - if (!accepted && tables && expected == IrOpcode::GetImageResource && TryMakeLaneTableImage(*copy, plan)) { + if (!accepted && tables && expected == IrOpcode::GetImageResource && TryMakeLaneTableImage(*copy, ImplicitDerivatives(inst), plan)) { m_indirectImages.push_back(std::move(plan)); accepted = true; } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index deffd1c11b..fdab62a30f 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -43,7 +43,7 @@ - [8_SRGB and 8_8_SRGB loads](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) on a device that cannot sample R8_SRGB or R8G8_SRGB return the sRGB EOTF rounded to nearest even at bf16 precision, as adriadam10 measured on RDNA2 (gfx1035) in #866; the PS5's gfx1013 is not measured. - [Samples and gathers](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB. -- A [waterfall loop](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) over T#s is compiled as one access per lane only when each lane loaded its T# with `buffer_load_dword*` (IDXEN, immediate VOFFSET and SOFFSET) from one V# that the SRT walker evaluates: the access becomes a [bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) keyed by the lane's index, with the NonUniform decoration. The loop is recognised when every T# dword of an image read, sample, gather or query comes from `v_readlane_b32` or `v_readfirstlane_b32` and the EXEC its results are written under implies that each of them equals the lane's own VGPR. Waterfalls over other per-lane T#s, over S#s, around image stores or atomics, and over T# VGPRs that hold another descriptor in lanes the load skipped still throw. A lane the load skipped whose T# VGPRs hold any other value reads zeros instead of using that value as a T#. The table V# must be structured (stride at least the record offset plus 32 bytes, `OOB_SELECT` 0 or 1) without swizzling or `ADD_TID`, else the dispatch throws, and like other bindless tables it binds at most `BindlessTableSlots` records. None of this was measured on hardware. +- A [waterfall loop](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) over T#s is compiled as one access per lane only when each lane loaded its T# with `buffer_load_dword*` (IDXEN, immediate VOFFSET and SOFFSET) from one V# that the SRT walker evaluates: the access becomes a [bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) keyed by the lane's index, with the NonUniform decoration. The loop is recognised when every T# dword of an image read, sample, gather or query comes from `v_readlane_b32` or `v_readfirstlane_b32` and the EXEC its results are written under implies that each of them equals the lane's own VGPR. Waterfalls over other per-lane T#s, over S#s, around image stores or atomics, and over T# VGPRs that hold another descriptor in lanes the load skipped (in all eight dwords alike, directly or through the same masked writes and phis) still throw. A lane the load skipped whose T# VGPRs hold any other value reads zeros instead of using that value as a T#. Every lane runs the access with its own record, so a sample with implicit derivatives or an LOD query throws at dispatch when the records, and record 0 that lanes without a record use, need different runtime image modes. The table V# must be structured (stride at least the record offset plus 32 bytes, `OOB_SELECT` 0 or 1) without swizzling or `ADD_TID`, else the dispatch throws, and like other bindless tables it binds at most `BindlessTableSlots` records. None of this was measured on hardware. - [Shader MODE register access](../../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) reads the initial rounding fields from `GuestContext::floatMode`; missing metadata retains the legacy round-to-nearest-even assumption. Immediate rounding writes are accepted only when they preserve that state. State-changing or SGPR-sourced MODE writes throw until runtime mode tracking and affected arithmetic are implemented. Non-rounding register reads still throw; `s_setreg_b32` writes to other hardware registers are still ignored. Tests follow the public RDNA2/LLVM contract, not console measurements. - A [flat SRT read](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read. - Repeated `s_load`s of one [flat SRT](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own. From bdb0a3f59f8db0b4c35b6c1e4804e8f41164ae1d Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:29:30 -0400 Subject: [PATCH 3/6] fix(recompiler): bind no record of empty per-lane tables and compare bound modes only A per-lane table V# with num_records 0, a null V# included, threw at dispatch on its stride (0 is less than the record offset plus 32) although no lane can hold a record, also when no lane enters the waterfall. With no records every OOB_SELECT puts each load out of range (the GFX10 table in Mesa's ac_descriptors.c), so such a table now binds no record and every lane that runs the access reads zeros; the swizzle, ADD_TID, OOB_SELECT and stride checks apply only to a V# with records. The implicit-derivative mode check compared every bound record with record 0, whose mode is 0 when it is null, so a table with a null record 0 and bound records of one other mode was refused. Only bound records are compared now. Lanes whose record is null or outside the table take the zero path under record 0's mode, so their quad neighbours' derivatives can still be undefined; the debt entry records this and the 16-record limit of the whole table. Co-Authored-By: Claude Opus 5.5 --- .../tests/execution/WaterfallImages.cpp | 52 +++++++++++++++++-- .../Optimization/src/ResourceMaterializer.cpp | 17 +++--- docs/dev/debt/shader.md | 2 +- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp index 14e0496103..ac9d82217c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp @@ -55,6 +55,16 @@ alignas(256) constexpr std::array PartialWave64{ 0x80010a00u, 0xbf810000u, }; +alignas(256) constexpr std::array GuardedWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e1402c1u, 0xbe9c037eu, 0x7da80202u, 0xbf880026u, 0xe0382000u, + 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, + 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, + 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, + 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, + 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu, 0xe0702000u, + 0x80010a00u, 0xbf810000u, +}; + alignas(256) constexpr std::array StaleWave32{ 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x7e040208u, 0x7e060209u, 0x7e08020au, 0x7e0a020bu, 0x7e0c020cu, 0x7e0e020du, 0x7e10020eu, 0x7e12020fu, 0x36200087u, 0xbe9c037eu, 0x7daa2087u, 0xe0382000u, @@ -223,9 +233,32 @@ void RequireRefused(AgcDriver::VulkanDevice& device, std::span& table, const std::string& what) { auto& data = guest[0]; - for (std::uint32_t key = 0u; key < Records; ++key) data.table[key] = Image(mixed && key % 2u == 0u ? 0u : 1u); + data.output.fill(0xdeadbeefu); + const auto lanes = std::min(32u, device.Target().subgroupSize); + auto userData = UserData(data, lanes); + std::copy(table.begin(), table.end(), userData.begin()); + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + std::string failure; + try { + const auto shader = Recompile(Request(device, GuardedWave32, 32u, lanes, userData, regions)); + device.Dispatch(shader, 1u, 1u, 1u); + device.SubmitRecorded(false); + device.WaitIdle(); + } catch (const std::exception& error) { + failure = error.what(); + } + Require(failure.empty(), what + " failed: " + failure.substr(0u, 300u)); + for (std::uint32_t lane = 0u; lane < lanes; ++lane) { + const auto actual = data.output[lane]; + Require(actual == 0xffffffffu, what + ": lane " + std::to_string(lane) + " read " + std::to_string(actual) + ", expected the value it holds outside the loop"); + } +} + +void RunLod(AgcDriver::VulkanDevice& device, const std::array& records, bool refused, const std::string& what) { + auto& data = guest[0]; + for (std::uint32_t key = 0u; key < Records; ++key) data.table[key] = records[key] == Textures ? std::array{} : Image(records[key]); const auto userData = UserData(data, 32u); const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; auto request = Request(device, LodWave32, 32u, std::min(32u, device.Target().subgroupSize), userData, regions); @@ -237,7 +270,7 @@ void RunLod(AgcDriver::VulkanDevice& device, bool mixed, const std::string& what } catch (const std::exception& error) { failure = error.what(); } - if (mixed) Require(failure.find("different runtime modes under an implicit-derivative access") != std::string::npos, what + " was not refused: " + failure.substr(0u, 300u)); + if (refused) Require(failure.find("different runtime modes under an implicit-derivative access") != std::string::npos, what + " was not refused: " + failure.substr(0u, 300u)); else Require(failure.empty(), what + " failed: " + failure.substr(0u, 300u)); } @@ -252,13 +285,22 @@ int main() { if (device->Target().subgroupSize >= 32u) Run(*device, PartialWave64, 64u, true, "wave64 waterfall under a partial EXEC"); Run(*device, FirstLaneWave32, 32u, false, "wave32 waterfall picking lanes with v_readfirstlane"); Run(*device, Compare64Wave32, 32u, false, "wave32 waterfall comparing the T# with v_cmpx_eq_u64"); + Run(*device, GuardedWave32, 32u, false, "wave32 waterfall behind a branch over the lanes inside the table"); + RunWithoutRecords(*device, {0u, 0u, 0u, 0u}, "a waterfall no lane enters over a null table V#"); + RunWithoutRecords(*device, Buffer(guest[0].table.data(), 0u, 0u), "a waterfall no lane enters over a table V# with stride 0 and no records"); + auto unchecked = Buffer(guest[0].table.data(), 32u, 0u); + unchecked[1] |= 1u << 31u; + unchecked[3] |= (1u << 23u) | (2u << 28u); + RunWithoutRecords(*device, unchecked, "a waterfall no lane enters over a swizzled ADD_TID table V# with OOB_SELECT 2 and no records"); RequireRefused(*device, StaleWave32, "a waterfall over a table T# and a T# left in the VGPRs of lanes the load skipped"); RequireRefused(*device, MaskedStaleWave32, "a waterfall over a table T# and a T# written under EXEC to the VGPRs of lanes the load skipped"); RequireRefused(*device, SevenDwordsWave32, "a waterfall that compares seven of the eight T# dwords"); RequireRefused(*device, OutsideWave32, "an image load after the waterfall restores EXEC"); RequireRefused(*device, StoreWave32, "an image store in a waterfall"); - RunLod(*device, false, "an LOD query in a waterfall over a table of one image mode"); - RunLod(*device, true, "an LOD query in a waterfall over a table of float and integer images"); + RunLod(*device, {1u, 1u, 1u, 1u}, false, "an LOD query in a waterfall over a table of one image mode"); + RunLod(*device, {0u, 1u, 0u, 1u}, true, "an LOD query in a waterfall over a table of float and integer images"); + RunLod(*device, {Textures, 0u, 0u, 0u}, false, "an LOD query in a waterfall over a null record 0 and integer images"); + RunLod(*device, {Textures, 0u, 1u, 0u}, true, "an LOD query in a waterfall over a null record 0 and float and integer images"); std::cout << "waterfall image tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 33633b9791..be633c5df7 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -336,10 +336,12 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con std::uint64_t entryBytes = TableEntryBytes; std::uint32_t entries = 0u; if (table.indexed) { - if (heap.SwizzleEnabled() || heap.AddTid() || heap.OutOfBounds() > 1u) throw std::runtime_error("bindless image table indexed through a V# with swizzling, ADD_TID or an unbounded range check is not implemented"); - entryBytes = heap.Stride(); - if (entryBytes < static_cast(table.entryOffset) + TableEntryBytes) throw std::runtime_error("bindless heap record is smaller than its descriptor"); - entries = heap.NumRecords(); + if (heap.NumRecords() != 0u) { + if (heap.SwizzleEnabled() || heap.AddTid() || heap.OutOfBounds() > 1u) throw std::runtime_error("bindless image table indexed through a V# with swizzling, ADD_TID or an unbounded range check is not implemented"); + entryBytes = heap.Stride(); + if (entryBytes < static_cast(table.entryOffset) + TableEntryBytes) throw std::runtime_error("bindless heap record is smaller than its descriptor"); + entries = heap.NumRecords(); + } } else { if (heapSize != 0u && (heapSize < table.entryOffset || (heapSize - table.entryOffset) % TableEntryBytes != 0u)) throw std::runtime_error("bindless heap contains a partial descriptor"); entries = heapSize > table.entryOffset ? static_cast(std::min((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits::max())) : 0u; @@ -415,8 +417,11 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con valid[i] = 1u; } if (table.laneKey && table.implicitDerivatives) { - for (std::size_t i = 1u; i < keys.size(); ++i) { - if (valid[i] != 0u && modes[i] != modes[0]) throw std::runtime_error("bindless image table keyed per lane holds images of different runtime modes under an implicit-derivative access, which is not implemented"); + std::optional boundMode; + for (std::size_t i = 0u; i < keys.size(); ++i) { + if (valid[i] == 0u) continue; + if (boundMode.has_value() && modes[i] != *boundMode) throw std::runtime_error("bindless image table keyed per lane holds images of different runtime modes under an implicit-derivative access, which is not implemented"); + boundMode = modes[i]; } } resolution.mapping.clear(); diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index fdab62a30f..13a4ff0dd7 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -43,7 +43,7 @@ - [8_SRGB and 8_8_SRGB loads](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) on a device that cannot sample R8_SRGB or R8G8_SRGB return the sRGB EOTF rounded to nearest even at bf16 precision, as adriadam10 measured on RDNA2 (gfx1035) in #866; the PS5's gfx1013 is not measured. - [Samples and gathers](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB. -- A [waterfall loop](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) over T#s is compiled as one access per lane only when each lane loaded its T# with `buffer_load_dword*` (IDXEN, immediate VOFFSET and SOFFSET) from one V# that the SRT walker evaluates: the access becomes a [bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) keyed by the lane's index, with the NonUniform decoration. The loop is recognised when every T# dword of an image read, sample, gather or query comes from `v_readlane_b32` or `v_readfirstlane_b32` and the EXEC its results are written under implies that each of them equals the lane's own VGPR. Waterfalls over other per-lane T#s, over S#s, around image stores or atomics, and over T# VGPRs that hold another descriptor in lanes the load skipped (in all eight dwords alike, directly or through the same masked writes and phis) still throw. A lane the load skipped whose T# VGPRs hold any other value reads zeros instead of using that value as a T#. Every lane runs the access with its own record, so a sample with implicit derivatives or an LOD query throws at dispatch when the records, and record 0 that lanes without a record use, need different runtime image modes. The table V# must be structured (stride at least the record offset plus 32 bytes, `OOB_SELECT` 0 or 1) without swizzling or `ADD_TID`, else the dispatch throws, and like other bindless tables it binds at most `BindlessTableSlots` records. None of this was measured on hardware. +- A [waterfall loop](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) over T#s is compiled as one access per lane only when each lane loaded its T# with `buffer_load_dword*` (IDXEN, immediate VOFFSET and SOFFSET) from one V# that the SRT walker evaluates: the access becomes a [bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) keyed by the lane's index, with the NonUniform decoration. The loop is recognised when every T# dword of an image read, sample, gather or query comes from `v_readlane_b32` or `v_readfirstlane_b32` and the EXEC its results are written under implies that each of them equals the lane's own VGPR. Waterfalls over other per-lane T#s, over S#s, around image stores or atomics, and over T# VGPRs that hold another descriptor in lanes the load skipped (in all eight dwords alike, directly or through the same masked writes and phis) still throw. A lane the load skipped whose T# VGPRs hold any other value reads zeros instead of using that value as a T#. Every lane runs the access with its own record, so a sample with implicit derivatives or an LOD query throws at dispatch when the bound records need different runtime image modes. Lanes whose record is null or outside the table take the zero path of the per-lane mode switch under record 0's mode, the first mode when record 0 is null, so around such a sample or query the derivatives of quad neighbours whose records need another mode are still undefined. A table V# with records must be structured (stride at least the record offset plus 32 bytes, `OOB_SELECT` 0 or 1) without swizzling or `ADD_TID`, else the dispatch throws. One with `num_records` 0, a null V# included, binds no record, and every lane that runs the access reads zeros: with no records every `OOB_SELECT` of the GFX10 table in Mesa [ac_descriptors.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/ac_descriptors.c) puts each load out of range. The table is bound whole, so a table V# with more than `BindlessTableSlots` (16) records throws at dispatch. None of this was measured on hardware. - [Shader MODE register access](../../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) reads the initial rounding fields from `GuestContext::floatMode`; missing metadata retains the legacy round-to-nearest-even assumption. Immediate rounding writes are accepted only when they preserve that state. State-changing or SGPR-sourced MODE writes throw until runtime mode tracking and affected arithmetic are implemented. Non-rounding register reads still throw; `s_setreg_b32` writes to other hardware registers are still ignored. Tests follow the public RDNA2/LLVM contract, not console measurements. - A [flat SRT read](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read. - Repeated `s_load`s of one [flat SRT](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own. From 3ce02f5928aef3bf6413a13460ae862e0da8ba14 Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:19:02 -0400 Subject: [PATCH 4/6] fix(recompiler): preserve per-lane table modes in disk resource plans The disk resource plan codec added on main must retain whether an indirect image uses a per-lane key, an indexed record stride, and implicit derivatives. Encode and decode those fields, update the aggregate field guard, and reject the older resource-plan format. Cover all eight flag combinations so a loaded plan cannot silently lose the stride or bound-mode checks. Co-Authored-By: Codex --- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 30 +++++++++++++++++++ core/shader/recompiler/ShaderDiskCache.cpp | 8 ++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 130508267e..32efe22386 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -358,6 +359,9 @@ void verifyEntryRoundTrip() { auto previousVersion = file; previousVersion[4] = static_cast(ShaderDiskCache::FormatVersion - 1u); require(ShaderDiskCache::DecodeEntry(previousVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the previous format version loads"); + auto upstreamVersion = file; + upstreamVersion[4] = std::byte{30}; + require(ShaderDiskCache::DecodeEntry(upstreamVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the upstream resource plan format loads"); auto incompatible = variant; incompatible.artifact.runtimeAbiVersion = RuntimeAbi::Version + 1u; const auto incompatibleFile = ShaderDiskCache::EncodeEntry(key, incompatible); @@ -1554,6 +1558,9 @@ void verifyPlanEntryDamage(const std::vector& key, const IrResourcePl auto previousFormat = file; previousFormat[4] = static_cast(ShaderDiskCache::FormatVersion - 1u); require(ShaderDiskCache::DecodePlanEntry(previousFormat, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry of the previous format loads"); + auto upstreamFormat = file; + upstreamFormat[4] = std::byte{30}; + require(ShaderDiskCache::DecodePlanEntry(upstreamFormat, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry with the upstream resource plan format loads"); auto otherSource = file; otherSource[8] ^= std::byte{1}; require(ShaderDiskCache::DecodePlanEntry(otherSource, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry of another source version loads"); @@ -1621,7 +1628,23 @@ void verifyMalformedPlans() { require(!ShaderDiskCache::EncodePlan(emptied, bytes), "a plan with an empty value slot was encoded"); } +void verifyIndirectImagePlanModes() { + std::set> tableEncodings; + for (std::uint32_t flags = 0u; flags < 8u; ++flags) { + auto plan = handBuiltPlan(); + auto& table = *plan.descriptorSources[1].indirectImage; + table.laneKey = (flags & 1u) != 0u; + table.indexed = (flags & 2u) != 0u; + table.implicitDerivatives = (flags & 4u) != 0u; + const auto bytes = encodedPlan(plan); + require(tableEncodings.insert(bytes).second, "indirect image table modes share a disk plan encoding"); + IrResourcePlan decoded; + require(ShaderDiskCache::DecodePlan(bytes, decoded) && samePlan(plan, decoded), "indirect image table modes changed in a disk plan round trip"); + } +} + void verifyPlanRoundTrip() { + verifyIndirectImagePlanModes(); const auto handBuilt = handBuiltPlan(); const auto handBytes = encodedPlan(handBuilt); IrResourcePlan handDecoded; @@ -1881,6 +1904,13 @@ void verifyDefaultDirectory(const char* self) { int main(int argc, char** argv) { try { + if (argc == 2 && std::string_view(argv[1]) == "--resource-plan-codec") { + verifyResultRoundTrip(); + verifyEntryRoundTrip(); + verifyIndirectImagePlanModes(); + std::cout << "resource plan codec tests passed\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--load") return runLoadingProcess(); if (argc == 2 && std::string_view(argv[1]) == "--no-failure-memo") return runWithoutFailureMemo(); if (argc == 2 && std::string_view(argv[1]).starts_with("--load-plan")) return runPlanLoadingProcess(argv[1]); diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 355e720e76..d8880fc672 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -147,7 +147,7 @@ static_assert(fieldCount() == 21, "IrResourcePlan changed: updat static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); static_assert(fieldCount() == 30, "MemoryInfo changed: update the plan encoder"); static_assert(fieldCount() == 3, "DescriptorSource changed: update the plan encoder"); -static_assert(fieldCount() == 7, "IndirectImage changed: update the plan encoder"); +static_assert(fieldCount() == 10, "IndirectImage changed: update the plan encoder"); static_assert(fieldCount() == 4, "ResourceBlock changed: update the plan encoder"); static_assert(fieldCount() == 2, "SrtRead changed: update the plan encoder"); static_assert(fieldCount() == 5, "UniformFill changed: update the plan encoder"); @@ -973,6 +973,9 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) { out.Value(table.keyArg); out.Value(table.entryOffset); out.Value(table.hasMaterial); + out.Value(table.laneKey); + out.Value(table.indexed); + out.Value(table.implicitDerivatives); }); writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) { out.Value(valueIndex(block.condition)); @@ -1115,6 +1118,9 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { in.Value(table.keyArg); in.Value(table.entryOffset); in.Value(table.hasMaterial); + in.Value(table.laneKey); + in.Value(table.indexed); + in.Value(table.implicitDerivatives); source.indirectImage = indirect ? std::optional(table) : std::nullopt; }); reader.List(plan.controlFlow, 28, [&](Reader& in, ResourceBlock& block) { From ea91d68090c69695975721de4b9c097181579ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Wed, 14 Oct 2026 13:24:31 +0200 Subject: [PATCH 5/6] test(agc): read the waterfall test SPIR-V through a span Co-Authored-By: Claude --- .../prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp index ac9d82217c..4de2a85f28 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp @@ -156,7 +156,7 @@ std::uint32_t TextureOf(std::uint32_t iteration, std::uint32_t key) { return key == Records - 1u - iteration ? Textures : (key + iteration) % Textures; } -bool DecoratesNonUniform(const std::vector& words) { +bool DecoratesNonUniform(std::span words) { for (std::size_t index = 5u; index < words.size();) { const auto count = words[index] >> 16u; if (count == 0u) return false; From 270eb0647cc7cd98d03b2e3541a80ed556d3df65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Wed, 14 Oct 2026 13:24:31 +0200 Subject: [PATCH 6/6] fix(recompiler): move the disk format to 32 after main took 31 Co-Authored-By: Claude --- core/shader/recompiler/ShaderDiskCache.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 327eb2f30a..c3a26c6d84 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 31; +inline constexpr std::uint32_t FormatVersion = 32; enum class LoadStatus { Loaded,