diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 59cf1d71b5..8d8cba88c6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1648,6 +1648,57 @@ void verifyPointerImageTable() { #endif } +void verifyPointerTableOffsetCarry() { + using namespace ShaderRecompiler; + namespace Abi = ImageTableAbi; + constexpr std::size_t Low = 0x10000; + constexpr std::uint64_t High = 0xffff0000ull; + constexpr std::size_t HighBytes = 0x20000; + constexpr std::uint64_t Reserved = High + HighBytes; + static TableTexture textures[2]; + void* block = nullptr; +#ifdef _WIN32 + block = VirtualAlloc(nullptr, static_cast(Reserved), MEM_RESERVE, PAGE_NOACCESS); + require(block != nullptr && VirtualAlloc(block, Low, MEM_COMMIT, PAGE_READWRITE) == block && VirtualAlloc(static_cast(block) + High, HighBytes, MEM_COMMIT, PAGE_READWRITE) == static_cast(block) + High, "pointer table offset carry: cannot map the root"); +#else + block = mmap(nullptr, Reserved, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + require(block != MAP_FAILED && mprotect(block, Low, PROT_READ | PROT_WRITE) == 0 && mprotect(static_cast(block) + High, HighBytes, PROT_READ | PROT_WRITE) == 0, "pointer table offset carry: cannot map the root"); +#endif + auto* root = static_cast(block); + const auto base = static_cast(reinterpret_cast(root)); + static std::array keys{}; + static std::array output{}; + const auto keysV = TableV(keys.data(), 0u, static_cast(sizeof(keys))); + const auto outputV = TableV(output.data(), 0u, static_cast(sizeof(output))); + std::copy(keysV.begin(), keysV.end(), root); + std::copy(outputV.begin(), outputV.end(), root + 4); + const auto at = [&](std::uint64_t offset, std::array words) { + std::copy(words.begin(), words.end(), root + offset / 4u); + }; + at(0x100000010ull, TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D)); + at(0x50u, TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D)); + const std::array userData{static_cast(base), static_cast(base >> 32u)}; + const std::vector code{0xf4080200u, 0xfa000000u, 0xf4080700u, 0xfa000010u, 0xf4080800u, 0xfa000020u, 0x7e200500u, 0x8f108210u, 0xf4200444u, + 0x20000000u, 0x8711ff11u, 0x000000ffu, 0x8f128511u, 0x8113ff12u, 0xffffff10u, 0x8114ff12u, 0xffffff20u, 0xf4080900u, 0x26000040u, 0xf4080a00u, + 0x28000040u, 0xf09c0f08u, 0x01090000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; + auto request = TableRequest(code, userData); + request.shader.codeAddress = 0x36000u; + const auto plan = GetResourcePlan(request); + require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "pointer table offset carry: a T# read by two loads of its record was not planned as a table"); + const auto& column = *plan->descriptorSources.at(plan->info.images[0].source).tableColumn; + require(column.address && column.stride == 32u && column.addend == 0xffffff10u && column.offset == 0x40u && column.maxKey == 0xffu && column.foldedAddend == 0xffffff10u && column.foldedCarry == 16u, "pointer table offset carry: the column is wrong"); + const auto compiled = CompileTable(request); + const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0); + require(snapshot.keys == 256u && snapshot.codes[7] == Abi::PoisonCode(static_cast(Abi::PoisonReason::OffsetCarry)), "pointer table offset carry: the record whose second load wraps past 32 bits is not OffsetCarry poison"); + require((snapshot.codes[6] & Abi::PoisonFlag) == 0u && (snapshot.codes[8] & Abi::PoisonFlag) == 0u && compiled.capture->snapshot.tables.words.at(snapshot.codes[6]).dwords[0] == root[0x100000010ull / 4u] && compiled.capture->snapshot.tables.words.at(snapshot.codes[8]).dwords[0] == root[0x50u / 4u] && root[0x100000010ull / 4u] != root[0x50u / 4u], "pointer table offset carry: the records on either side of the carry were not resolved"); + require(!TableTouched(compiled.regions, root + 0x100000030ull / 4u, 0x20u), "pointer table offset carry: the record whose second load wraps was read"); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + munmap(block, Reserved); +#endif +} + void verifyDescriptorPhis() { using namespace ShaderRecompiler; constexpr std::uint32_t Format8888UNorm = 56; @@ -4802,6 +4853,7 @@ int main(int argc, char** argv) { verifyImageTableRegistration(); verifyScalarAddressRule(); verifyPointerImageTable(); + verifyPointerTableOffsetCarry(); std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } @@ -4825,6 +4877,7 @@ int main(int argc, char** argv) { verifyImageTableRegistration(); verifyScalarAddressRule(); verifyPointerImageTable(); + verifyPointerTableOffsetCarry(); verifyDescriptorPhis(); verifyUniformTableImage(); verifyLaneSelectedDescriptors(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp index 4fd0865828..43f1799617 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp @@ -237,6 +237,13 @@ alignas(256) constexpr std::array ReadPointerCode{ 0xbf810000u, }; +alignas(256) constexpr std::array SplitRecordCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0x8f108202u, 0xf4200444u, + 0x20000000u, 0x9312b011u, 0xf42c0802u, 0x24000000u, 0xf4280a02u, 0x24000020u, 0x360200bfu, 0x7e020d01u, + 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, 0x01090401u, 0x7e120202u, + 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + alignas(256) constexpr std::array IndirectRecordCode{ 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, 0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u, 0xf4240b02u, 0x24000008u, 0xf40c0916u, 0xfa000020u, @@ -1099,6 +1106,29 @@ void RunPointerRecordTests(AgcDriver::VulkanDevice& device) { RequireRed(changed, 1, Lanes, texels[0], "float texels after the T# behind record 1's pointer changed"); } +void RunSplitRecordTests(AgcDriver::VulkanDevice& device) { + Fill(); + const auto floatImage = Image(FloatTexels.data(), Format32Float); + const auto uintImage = Image(UintTexels.data(), Format32UInt); + const auto sintImage = Image(SintTexels.data(), Format32SInt); + const std::array, 3> images{floatImage, uintImage, sintImage}; + const std::array, 3> texels{ + [](std::uint32_t x) { return std::bit_cast(FloatTexels[x]); }, + [](std::uint32_t x) { return UintTexels[x]; }, + [](std::uint32_t x) { return static_cast(SintTexels[x]); }, + }; + const std::array names{"float", "uint", "sint"}; + + Table = {}; + for (std::uint32_t record = 0; record < 3; ++record) SetRecord(record, PointClamp, images[record]); + Keys = {}; + for (std::uint32_t group = 0; group < 3; ++group) Keys[group] = 2u - group; + const auto split = Run(device, SplitRecordCode, 3, Lanes, 64); + Require(!Faulted(split), "image table: a T# read by two loads of its record faulted:\n" + split.log); + Require(split.imageTableCount == 3u, "image table: a T# read by two loads of its record does not hold one element per distinct T# (" + std::to_string(split.imageTableCount) + ")"); + for (std::uint32_t group = 0; group < 3; ++group) RequireRed(split, group, Lanes, texels[2u - group], std::string(names[2u - group]) + " texels of a T# read by two loads of its record"); +} + void RunPointerStorageWriteTests(AgcDriver::VulkanDevice& device) { alignas(256) static std::array records{}; alignas(256) static std::array otherRecords{}; @@ -1299,6 +1329,7 @@ int main() { RunStoreTests(*device); RunPointerTests(*device); RunPointerRecordTests(*device); + RunSplitRecordTests(*device); RunPointerStorageWriteTests(*device); RunGraphicsTests(*device); std::puts("image table tests passed"); diff --git a/core/shader/recompiler/ImageTableAbi.hpp b/core/shader/recompiler/ImageTableAbi.hpp index 8a07ef7cda..9b434141d8 100644 --- a/core/shader/recompiler/ImageTableAbi.hpp +++ b/core/shader/recompiler/ImageTableAbi.hpp @@ -75,6 +75,7 @@ enum class PoisonReason : std::uint32_t { HeapWritten, DriverRejected, UnsupportedOperation, + OffsetCarry, Count }; @@ -101,6 +102,7 @@ constexpr std::string_view PoisonReasonName(PoisonReason reason) { case PoisonReason::HeapWritten: return "table written by its dispatch"; case PoisonReason::DriverRejected: return "rejected by the driver"; case PoisonReason::UnsupportedOperation: return "unsupported operation"; + case PoisonReason::OffsetCarry: return "record offset carries past 32 bits"; case PoisonReason::Count: break; } return "unknown"; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index dd2c624c31..c402bbee7e 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -42,9 +42,11 @@ struct TableColumn { bool indirect = false; std::uint32_t indirectOffset = 0; std::uint32_t indirectImmediate = 0; + std::uint32_t foldedAddend = 0; + std::uint32_t foldedCarry = 0; bool operator==(const TableColumn& other) const { - return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate; + return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry; } }; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 4ecd7e90f2..7353276efb 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -345,7 +345,7 @@ void reportTables() { static_cast(counters.narrowed.exchange(0, std::memory_order_relaxed)), static_cast(counters.narrowSkipped.exchange(0, std::memory_order_relaxed)), static_cast(counters.nanoseconds.exchange(0, std::memory_order_relaxed)) / 1e6); } -enum class RecordState : std::uint8_t { OutsideDomain, Read, Unmapped }; +enum class RecordState : std::uint8_t { OutsideDomain, Read, Unmapped, Carried }; struct ColumnRecords { std::uint64_t base = 0; @@ -643,6 +643,10 @@ private: for (std::uint32_t record = 0; record < records.keys; record++) { if (records.states[record] != RecordState::Read) continue; const auto offset = column.addend + record * column.stride; + if (offset >= column.foldedAddend && offset > 0xffffffffu - column.foldedCarry) { + records.states[record] = RecordState::Carried; + continue; + } const auto head = locate(offset, column.offset); const auto tail = locate(offset, column.offset + last); if (head.has_value() && tail.has_value() && (*head & 3u) == 0u && *tail > *head && *tail - *head == last && wholeAccessible(*head) && wholeAccessible(*tail)) { @@ -730,6 +734,10 @@ private: out.codes[key] = ImageTableAbi::PoisonCode(static_cast(PoisonReason::Unmapped)); counters.poison[static_cast(PoisonReason::Unmapped)].fetch_add(1, std::memory_order_relaxed); break; + case RecordState::Carried: + out.codes[key] = ImageTableAbi::PoisonCode(static_cast(PoisonReason::OffsetCarry)); + counters.poison[static_cast(PoisonReason::OffsetCarry)].fetch_add(1, std::memory_order_relaxed); + break; } trace.poison++; } diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 11e737fa97..6bce45abca 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -539,6 +539,17 @@ private: return KeyDomain{sourceIndex, memory->offset, std::max(4u, 1u << zeros)}; } + static IrValue* SplitImmediateAddend(IrValue* value, std::uint32_t& addend) { + value = value->Resolve(); + addend = 0; + if (value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) { + if (immediateU32(value->Argument(0), addend)) return value->Argument(1)->Resolve(); + if (immediateU32(value->Argument(1), addend)) return value->Argument(0)->Resolve(); + } + addend = 0; + return value; + } + bool TryMakeTableColumn(IrValue& handle, bool sampler, bool r128, TableColumnPlan& plan) { const auto expected = sampler ? IrOpcode::GetSamplerResource : IrOpcode::GetImageResource; const std::uint32_t width = sampler ? 4u : 8u; @@ -558,6 +569,10 @@ private: } IrValue* heapHandle = nullptr; IrValue* offset = nullptr; + IrValue* base = nullptr; + std::uint32_t baseAddend = 0; + std::uint32_t carry = 0; + bool folded = false; std::uint32_t first = 0; bool address = false; for (std::uint32_t dword = 0; dword < words; dword++) { @@ -568,20 +583,30 @@ private: if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) { return false; } + std::uint32_t readAddend = 0; + IrValue* readBase = SplitImmediateAddend(read->Argument(1), readAddend); if (dword == 0u) { first = memory->offset; address = addressRead; - } else if (addressRead != address || memory->offset != first + dword * static_cast(sizeof(std::uint32_t))) { + base = readBase; + baseAddend = readAddend; + } else if (addressRead != address || memory->offset + readAddend != first + baseAddend + dword * static_cast(sizeof(std::uint32_t))) { return false; + } else if (readAddend != baseAddend) { + if (readAddend < baseAddend || static_cast(memory->offset) + (readAddend - baseAddend) > 0x7fffffffu) { + return false; + } + folded = true; + carry = std::max(carry, readAddend - baseAddend); } IrValue* currentHandle = read->Argument(0)->Resolve(); - if (heapHandle != nullptr && currentHandle != heapHandle) { + if (heapHandle != nullptr && currentHandle != heapHandle && !EquivalentValue(m_program.Resources(), heapHandle, currentHandle)) { return false; } - heapHandle = currentHandle; + if (heapHandle == nullptr) heapHandle = currentHandle; if (dword == 0u) { offset = read->Argument(1)->Resolve(); - } else if (!EquivalentValue(m_program.Resources(), offset, read->Argument(1))) { + } else if (!EquivalentValue(m_program.Resources(), base, readBase)) { return false; } plan.memory.push_back(memoryIndex); @@ -623,6 +648,8 @@ private: column.maxKey = possibleU32Bits(key); column.sampler = sampler; column.address = address; + column.foldedAddend = folded ? baseAddend : 0u; + column.foldedCarry = folded ? carry : 0u; const auto domain = masked ? std::optional{} : MatchKeyDomain(key); DescriptorSource columnSource = heapSource; columnSource.tableColumn = column; diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index a0f5e958be..1858e4721d 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -107,8 +107,8 @@ static_assert(sizeof(IrResourcePlan) == 584, "IrResourcePlan changed: update Enc static_assert(sizeof(IrValue) == 152, "IrValue changed: update the plan value encoder"); static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder"); static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder"); -static_assert(sizeof(DescriptorSource) == 152, "DescriptorSource changed: update the plan encoder"); -static_assert(sizeof(TableColumn) == 72, "TableColumn changed: update the plan encoder"); +static_assert(sizeof(DescriptorSource) == 160, "DescriptorSource changed: update the plan encoder"); +static_assert(sizeof(TableColumn) == 80, "TableColumn changed: update the plan encoder"); static_assert(sizeof(KeyDomain) == 16, "KeyDomain changed: update the plan encoder"); static_assert(sizeof(ResourceBlock) == 80, "ResourceBlock changed: update the plan encoder"); static_assert(sizeof(SrtRead) == 16, "SrtRead changed: update the plan encoder"); @@ -148,7 +148,7 @@ static_assert(fieldCount() == 21, "IrResourcePlan changed: updat static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); static_assert(fieldCount() == 30, "MemoryInfo changed: update the plan encoder"); static_assert(fieldCount() == 3, "DescriptorSource changed: update the plan encoder"); -static_assert(fieldCount() == 13, "TableColumn changed: update the plan encoder"); +static_assert(fieldCount() == 15, "TableColumn changed: update the plan encoder"); static_assert(fieldCount() == 4, "KeyDomain changed: update the plan encoder"); static_assert(fieldCount() == 4, "ResourceBlock changed: update the plan encoder"); static_assert(fieldCount() == 2, "SrtRead changed: update the plan encoder"); @@ -1022,6 +1022,8 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) { out.Value(column.indirect); out.Value(column.indirectOffset); out.Value(column.indirectImmediate); + out.Value(column.foldedAddend); + out.Value(column.foldedCarry); }); writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) { out.Value(valueIndex(block.condition)); @@ -1175,6 +1177,8 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { in.Value(column.indirect); in.Value(column.indirectOffset); in.Value(column.indirectImmediate); + in.Value(column.foldedAddend); + in.Value(column.foldedCarry); source.tableColumn = table ? std::optional(std::move(column)) : std::nullopt; }); reader.List(plan.controlFlow, 28, [&](Reader& in, ResourceBlock& block) {