feat(agc): plan table columns whose descriptor spans two loads of a record

A descriptor whose dwords come from two scalar buffer loads of the same
record, where one load's offset carries an immediate addend, was refused
because the column wanted one offset expression and one buffer node for
every dword. The column now peels an immediate addend off each read's
offset and compares effective offsets (memory offset plus addend) and
the remaining base expressions, and accepts distinct buffer nodes that
are equivalent. Dword 0 keeps its offset expression, so columns that were
already planned are planned the same way.

Folding a second load into the first changes where its offset wraps:
at a dynamic offset of 0xfffffff0, a second load at +16 wraps to the
start of the buffer, while the folded read lands past 4 GiB. A column
folds a load only when its addend is no smaller than dword 0's and its
immediate is small, and it records dword 0's addend and the largest extra
addend. A record whose offset would carry past 32 bits between the two
loads is not read and becomes OffsetCarry poison.

PPSA26344 pixel shader 0x8000296000 reads T# dwords 0-3 with an
s_buffer_load_dwordx16 at key * 0x88 and dwords 4-7 with an
s_buffer_load_dwordx4 at 48 + key * 0x88.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -1648,6 +1648,57 @@ void verifyPointerImageTable() {
 #endif
 }
 
+void verifyPointerTableOffsetCarry() {
+    using namespace ShaderRecompiler;
+    namespace Abi = ImageTableAbi;
+    constexpr std::size_t Low = 0x10000;
+    constexpr std::uint64_t High = 0xffff0000ull;
+    constexpr std::size_t HighBytes = 0x20000;
+    constexpr std::uint64_t Reserved = High + HighBytes;
+    static TableTexture textures[2];
+    void* block = nullptr;
+#ifdef _WIN32
+    block = VirtualAlloc(nullptr, static_cast<SIZE_T>(Reserved), MEM_RESERVE, PAGE_NOACCESS);
+    require(block != nullptr && VirtualAlloc(block, Low, MEM_COMMIT, PAGE_READWRITE) == block && VirtualAlloc(static_cast<std::byte*>(block) + High, HighBytes, MEM_COMMIT, PAGE_READWRITE) == static_cast<std::byte*>(block) + High, "pointer table offset carry: cannot map the root");
+#else
+    block = mmap(nullptr, Reserved, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+    require(block != MAP_FAILED && mprotect(block, Low, PROT_READ | PROT_WRITE) == 0 && mprotect(static_cast<std::byte*>(block) + High, HighBytes, PROT_READ | PROT_WRITE) == 0, "pointer table offset carry: cannot map the root");
+#endif
+    auto* root = static_cast<std::uint32_t*>(block);
+    const auto base = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(root));
+    static std::array<std::uint32_t, 8> keys{};
+    static std::array<std::uint32_t, 4> output{};
+    const auto keysV = TableV(keys.data(), 0u, static_cast<std::uint32_t>(sizeof(keys)));
+    const auto outputV = TableV(output.data(), 0u, static_cast<std::uint32_t>(sizeof(output)));
+    std::copy(keysV.begin(), keysV.end(), root);
+    std::copy(outputV.begin(), outputV.end(), root + 4);
+    const auto at = [&](std::uint64_t offset, std::array<std::uint32_t, 8> words) {
+        std::copy(words.begin(), words.end(), root + offset / 4u);
+    };
+    at(0x100000010ull, TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D));
+    at(0x50u, TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D));
+    const std::array<std::uint32_t, 2> userData{static_cast<std::uint32_t>(base), static_cast<std::uint32_t>(base >> 32u)};
+    const std::vector<std::uint32_t> code{0xf4080200u, 0xfa000000u, 0xf4080700u, 0xfa000010u, 0xf4080800u, 0xfa000020u, 0x7e200500u, 0x8f108210u, 0xf4200444u,
+        0x20000000u, 0x8711ff11u, 0x000000ffu, 0x8f128511u, 0x8113ff12u, 0xffffff10u, 0x8114ff12u, 0xffffff20u, 0xf4080900u, 0x26000040u, 0xf4080a00u,
+        0x28000040u, 0xf09c0f08u, 0x01090000u, 0xe0700000u, 0x80070000u, 0xbf810000u};
+    auto request = TableRequest(code, userData);
+    request.shader.codeAddress = 0x36000u;
+    const auto plan = GetResourcePlan(request);
+    require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "pointer table offset carry: a T# read by two loads of its record was not planned as a table");
+    const auto& column = *plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
+    require(column.address && column.stride == 32u && column.addend == 0xffffff10u && column.offset == 0x40u && column.maxKey == 0xffu && column.foldedAddend == 0xffffff10u && column.foldedCarry == 16u, "pointer table offset carry: the column is wrong");
+    const auto compiled = CompileTable(request);
+    const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0);
+    require(snapshot.keys == 256u && snapshot.codes[7] == Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::OffsetCarry)), "pointer table offset carry: the record whose second load wraps past 32 bits is not OffsetCarry poison");
+    require((snapshot.codes[6] & Abi::PoisonFlag) == 0u && (snapshot.codes[8] & Abi::PoisonFlag) == 0u && compiled.capture->snapshot.tables.words.at(snapshot.codes[6]).dwords[0] == root[0x100000010ull / 4u] && compiled.capture->snapshot.tables.words.at(snapshot.codes[8]).dwords[0] == root[0x50u / 4u] && root[0x100000010ull / 4u] != root[0x50u / 4u], "pointer table offset carry: the records on either side of the carry were not resolved");
+    require(!TableTouched(compiled.regions, root + 0x100000030ull / 4u, 0x20u), "pointer table offset carry: the record whose second load wraps was read");
+#ifdef _WIN32
+    VirtualFree(block, 0, MEM_RELEASE);
+#else
+    munmap(block, Reserved);
+#endif
+}
+
 void verifyDescriptorPhis() {
     using namespace ShaderRecompiler;
     constexpr std::uint32_t Format8888UNorm = 56;
@@ -4802,6 +4853,7 @@ int main(int argc, char** argv) {
             verifyImageTableRegistration();
             verifyScalarAddressRule();
             verifyPointerImageTable();
+            verifyPointerTableOffsetCarry();
             std::cout << "Image table keys, maps, records and registration passed\n";
             return 0;
         }
@@ -4825,6 +4877,7 @@ int main(int argc, char** argv) {
         verifyImageTableRegistration();
         verifyScalarAddressRule();
         verifyPointerImageTable();
+        verifyPointerTableOffsetCarry();
         verifyDescriptorPhis();
         verifyUniformTableImage();
         verifyLaneSelectedDescriptors();

core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp

@@ -237,6 +237,13 @@ alignas(256) constexpr std::array<std::uint32_t, 25> ReadPointerCode{
     0xbf810000u,
 };
 
+alignas(256) constexpr std::array<std::uint32_t, 30> SplitRecordCode{
+    0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0x8f108202u, 0xf4200444u,
+    0x20000000u, 0x9312b011u, 0xf42c0802u, 0x24000000u, 0xf4280a02u, 0x24000020u, 0x360200bfu, 0x7e020d01u,
+    0x060202f0u, 0x100202ffu, 0x3c800000u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, 0x01090401u, 0x7e120202u,
+    0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,
+};
+
 alignas(256) constexpr std::array<std::uint32_t, 32> IndirectRecordCode{
     0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u,
     0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u, 0xf4240b02u, 0x24000008u, 0xf40c0916u, 0xfa000020u,
@@ -1099,6 +1106,29 @@ void RunPointerRecordTests(AgcDriver::VulkanDevice& device) {
     RequireRed(changed, 1, Lanes, texels[0], "float texels after the T# behind record 1's pointer changed");
 }
 
+void RunSplitRecordTests(AgcDriver::VulkanDevice& device) {
+    Fill();
+    const auto floatImage = Image(FloatTexels.data(), Format32Float);
+    const auto uintImage = Image(UintTexels.data(), Format32UInt);
+    const auto sintImage = Image(SintTexels.data(), Format32SInt);
+    const std::array<std::array<std::uint32_t, 4>, 3> images{floatImage, uintImage, sintImage};
+    const std::array<std::function<std::uint32_t(std::uint32_t)>, 3> texels{
+        [](std::uint32_t x) { return std::bit_cast<std::uint32_t>(FloatTexels[x]); },
+        [](std::uint32_t x) { return UintTexels[x]; },
+        [](std::uint32_t x) { return static_cast<std::uint32_t>(SintTexels[x]); },
+    };
+    const std::array<const char*, 3> names{"float", "uint", "sint"};
+
+    Table = {};
+    for (std::uint32_t record = 0; record < 3; ++record) SetRecord(record, PointClamp, images[record]);
+    Keys = {};
+    for (std::uint32_t group = 0; group < 3; ++group) Keys[group] = 2u - group;
+    const auto split = Run(device, SplitRecordCode, 3, Lanes, 64);
+    Require(!Faulted(split), "image table: a T# read by two loads of its record faulted:\n" + split.log);
+    Require(split.imageTableCount == 3u, "image table: a T# read by two loads of its record does not hold one element per distinct T# (" + std::to_string(split.imageTableCount) + ")");
+    for (std::uint32_t group = 0; group < 3; ++group) RequireRed(split, group, Lanes, texels[2u - group], std::string(names[2u - group]) + " texels of a T# read by two loads of its record");
+}
+
 void RunPointerStorageWriteTests(AgcDriver::VulkanDevice& device) {
     alignas(256) static std::array<std::uint32_t, 256u * 8u> records{};
     alignas(256) static std::array<std::uint32_t, 256u * 8u> otherRecords{};
@@ -1299,6 +1329,7 @@ int main() {
         RunStoreTests(*device);
         RunPointerTests(*device);
         RunPointerRecordTests(*device);
+        RunSplitRecordTests(*device);
         RunPointerStorageWriteTests(*device);
         RunGraphicsTests(*device);
         std::puts("image table tests passed");

core/shader/recompiler/ImageTableAbi.hpp

@@ -75,6 +75,7 @@ enum class PoisonReason : std::uint32_t {
     HeapWritten,
     DriverRejected,
     UnsupportedOperation,
+    OffsetCarry,
     Count
 };
 
@@ -101,6 +102,7 @@ constexpr std::string_view PoisonReasonName(PoisonReason reason) {
         case PoisonReason::HeapWritten: return "table written by its dispatch";
         case PoisonReason::DriverRejected: return "rejected by the driver";
         case PoisonReason::UnsupportedOperation: return "unsupported operation";
+        case PoisonReason::OffsetCarry: return "record offset carries past 32 bits";
         case PoisonReason::Count: break;
     }
     return "unknown";

core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp

@@ -42,9 +42,11 @@ struct TableColumn {
     bool indirect = false;
     std::uint32_t indirectOffset = 0;
     std::uint32_t indirectImmediate = 0;
+    std::uint32_t foldedAddend = 0;
+    std::uint32_t foldedCarry = 0;
 
     bool operator==(const TableColumn& other) const {
-        return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate;
+        return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry;
     }
 };
 

core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp

@@ -345,7 +345,7 @@ void reportTables() {
         static_cast<unsigned long long>(counters.narrowed.exchange(0, std::memory_order_relaxed)), static_cast<unsigned long long>(counters.narrowSkipped.exchange(0, std::memory_order_relaxed)), static_cast<double>(counters.nanoseconds.exchange(0, std::memory_order_relaxed)) / 1e6);
 }
 
-enum class RecordState : std::uint8_t { OutsideDomain, Read, Unmapped };
+enum class RecordState : std::uint8_t { OutsideDomain, Read, Unmapped, Carried };
 
 struct ColumnRecords {
     std::uint64_t base = 0;
@@ -643,6 +643,10 @@ private:
         for (std::uint32_t record = 0; record < records.keys; record++) {
             if (records.states[record] != RecordState::Read) continue;
             const auto offset = column.addend + record * column.stride;
+            if (offset >= column.foldedAddend && offset > 0xffffffffu - column.foldedCarry) {
+                records.states[record] = RecordState::Carried;
+                continue;
+            }
             const auto head = locate(offset, column.offset);
             const auto tail = locate(offset, column.offset + last);
             if (head.has_value() && tail.has_value() && (*head & 3u) == 0u && *tail > *head && *tail - *head == last && wholeAccessible(*head) && wholeAccessible(*tail)) {
@@ -730,6 +734,10 @@ private:
                     out.codes[key] = ImageTableAbi::PoisonCode(static_cast<std::uint32_t>(PoisonReason::Unmapped));
                     counters.poison[static_cast<std::size_t>(PoisonReason::Unmapped)].fetch_add(1, std::memory_order_relaxed);
                     break;
+                case RecordState::Carried:
+                    out.codes[key] = ImageTableAbi::PoisonCode(static_cast<std::uint32_t>(PoisonReason::OffsetCarry));
+                    counters.poison[static_cast<std::size_t>(PoisonReason::OffsetCarry)].fetch_add(1, std::memory_order_relaxed);
+                    break;
             }
             trace.poison++;
         }

core/shader/recompiler/Optimization/src/ResourceTracker.cpp

@@ -539,6 +539,17 @@ private:
         return KeyDomain{sourceIndex, memory->offset, std::max(4u, 1u << zeros)};
     }
 
+    static IrValue* SplitImmediateAddend(IrValue* value, std::uint32_t& addend) {
+        value = value->Resolve();
+        addend = 0;
+        if (value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) {
+            if (immediateU32(value->Argument(0), addend)) return value->Argument(1)->Resolve();
+            if (immediateU32(value->Argument(1), addend)) return value->Argument(0)->Resolve();
+        }
+        addend = 0;
+        return value;
+    }
+
     bool TryMakeTableColumn(IrValue& handle, bool sampler, bool r128, TableColumnPlan& plan) {
         const auto expected = sampler ? IrOpcode::GetSamplerResource : IrOpcode::GetImageResource;
         const std::uint32_t width = sampler ? 4u : 8u;
@@ -558,6 +569,10 @@ private:
         }
         IrValue* heapHandle = nullptr;
         IrValue* offset = nullptr;
+        IrValue* base = nullptr;
+        std::uint32_t baseAddend = 0;
+        std::uint32_t carry = 0;
+        bool folded = false;
         std::uint32_t first = 0;
         bool address = false;
         for (std::uint32_t dword = 0; dword < words; dword++) {
@@ -568,20 +583,30 @@ private:
             if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) {
                 return false;
             }
+            std::uint32_t readAddend = 0;
+            IrValue* readBase = SplitImmediateAddend(read->Argument(1), readAddend);
             if (dword == 0u) {
                 first = memory->offset;
                 address = addressRead;
-            } else if (addressRead != address || memory->offset != first + dword * static_cast<std::uint32_t>(sizeof(std::uint32_t))) {
+                base = readBase;
+                baseAddend = readAddend;
+            } else if (addressRead != address || memory->offset + readAddend != first + baseAddend + dword * static_cast<std::uint32_t>(sizeof(std::uint32_t))) {
                 return false;
+            } else if (readAddend != baseAddend) {
+                if (readAddend < baseAddend || static_cast<std::uint64_t>(memory->offset) + (readAddend - baseAddend) > 0x7fffffffu) {
+                    return false;
+                }
+                folded = true;
+                carry = std::max(carry, readAddend - baseAddend);
             }
             IrValue* currentHandle = read->Argument(0)->Resolve();
-            if (heapHandle != nullptr && currentHandle != heapHandle) {
+            if (heapHandle != nullptr && currentHandle != heapHandle && !EquivalentValue(m_program.Resources(), heapHandle, currentHandle)) {
                 return false;
             }
-            heapHandle = currentHandle;
+            if (heapHandle == nullptr) heapHandle = currentHandle;
             if (dword == 0u) {
                 offset = read->Argument(1)->Resolve();
-            } else if (!EquivalentValue(m_program.Resources(), offset, read->Argument(1))) {
+            } else if (!EquivalentValue(m_program.Resources(), base, readBase)) {
                 return false;
             }
             plan.memory.push_back(memoryIndex);
@@ -623,6 +648,8 @@ private:
         column.maxKey = possibleU32Bits(key);
         column.sampler = sampler;
         column.address = address;
+        column.foldedAddend = folded ? baseAddend : 0u;
+        column.foldedCarry = folded ? carry : 0u;
         const auto domain = masked ? std::optional<KeyDomain>{} : MatchKeyDomain(key);
         DescriptorSource columnSource = heapSource;
         columnSource.tableColumn = column;

core/shader/recompiler/ShaderDiskCache.cpp

@@ -107,8 +107,8 @@ static_assert(sizeof(IrResourcePlan) == 584, "IrResourcePlan changed: update Enc
 static_assert(sizeof(IrValue) == 152, "IrValue changed: update the plan value encoder");
 static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder");
 static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder");
-static_assert(sizeof(DescriptorSource) == 152, "DescriptorSource changed: update the plan encoder");
-static_assert(sizeof(TableColumn) == 72, "TableColumn changed: update the plan encoder");
+static_assert(sizeof(DescriptorSource) == 160, "DescriptorSource changed: update the plan encoder");
+static_assert(sizeof(TableColumn) == 80, "TableColumn changed: update the plan encoder");
 static_assert(sizeof(KeyDomain) == 16, "KeyDomain changed: update the plan encoder");
 static_assert(sizeof(ResourceBlock) == 80, "ResourceBlock changed: update the plan encoder");
 static_assert(sizeof(SrtRead) == 16, "SrtRead changed: update the plan encoder");
@@ -148,7 +148,7 @@ static_assert(fieldCount<IrResourcePlan>() == 21, "IrResourcePlan changed: updat
 static_assert(fieldCount<GuestRegister>() == 2, "GuestRegister changed: update the plan value encoder");
 static_assert(fieldCount<MemoryInfo>() == 30, "MemoryInfo changed: update the plan encoder");
 static_assert(fieldCount<DescriptorSource>() == 3, "DescriptorSource changed: update the plan encoder");
-static_assert(fieldCount<TableColumn>() == 13, "TableColumn changed: update the plan encoder");
+static_assert(fieldCount<TableColumn>() == 15, "TableColumn changed: update the plan encoder");
 static_assert(fieldCount<KeyDomain>() == 4, "KeyDomain changed: update the plan encoder");
 static_assert(fieldCount<ResourceBlock>() == 4, "ResourceBlock changed: update the plan encoder");
 static_assert(fieldCount<SrtRead>() == 2, "SrtRead changed: update the plan encoder");
@@ -1022,6 +1022,8 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) {
         out.Value(column.indirect);
         out.Value(column.indirectOffset);
         out.Value(column.indirectImmediate);
+        out.Value(column.foldedAddend);
+        out.Value(column.foldedCarry);
     });
     writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) {
         out.Value(valueIndex(block.condition));
@@ -1175,6 +1177,8 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) {
         in.Value(column.indirect);
         in.Value(column.indirectOffset);
         in.Value(column.indirectImmediate);
+        in.Value(column.foldedAddend);
+        in.Value(column.foldedCarry);
         source.tableColumn = table ? std::optional(std::move(column)) : std::nullopt;
     });
     reader.List(plan.controlFlow, 28, [&](Reader& in, ResourceBlock& block) {