feat(shader): follow pointer chains from image-table records to the T#

This commit is contained in:
ANONYMOUSZED-beep authored and x committed 2026-10-18 20:49:04 +02:00
1 parent 376c040294
commit 486f3069ae
7 files changed
+213 -9

No files matched your search

@@ -1658,6 +1658,7 @@ IrResourcePlan handBuiltPlan() {
table.tableColumn = TableColumn{2u, 32u, 16u, 4u, 8u, 0xffu, false, true, {KeyDomain{3u, 4u, 16u, false}, KeyDomain{2u, 8u, 4u, true}}, false};
table.tableColumn->selected = true;
table.tableColumn->selection = 2u;
table.tableColumn->pointerHops = {PointerHop{0u, 0x10u}, PointerHop{8u, 0x20u}};
DescriptorSource heap;
heap.dwordCount = 4u;
heap.dwords[1] = &readConst(6);
@@ -2515,6 +2515,77 @@ void verifyChainedTableOffset() {
require(words.at(snapshot.codes[3]).dwords[0] == root[(0x60u + 3u * 32u) / 4u] && words.at(snapshot.codes[3]).dwords[7] == root[(0x60u + 3u * 32u) / 4u + 7u] && words.at(snapshot.codes[5]).dwords[0] == root[(0x60u + 5u * 32u) / 4u] && snapshot.codes[3] != snapshot.codes[5], "chained table offset: a record was read from the wrong offset");
}
void verifyPointerChainImageTable() {
using namespace ShaderRecompiler;
namespace Abi = ImageTableAbi;
constexpr std::uint32_t Entries = 256;
static TableTexture textures[2];
alignas(4096) static std::array<std::uint32_t, 16> root{};
alignas(4096) static std::array<std::uint32_t, Entries * 4u> records{};
alignas(4096) static std::array<std::array<std::uint32_t, 8>, 4> links{};
alignas(4096) static std::array<std::array<std::uint32_t, 16>, 3> descriptors{};
static std::array<std::uint32_t, 8> keys{};
static std::array<std::uint32_t, 4> output{};
const auto address = [](const void* pointer) { return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(pointer)); };
const auto keysV = TableV(keys.data(), 0u, static_cast<std::uint32_t>(sizeof(keys)));
const auto outputV = TableV(output.data(), 0u, static_cast<std::uint32_t>(sizeof(output)));
const auto recordsV = TableV(records.data(), 0u, static_cast<std::uint32_t>(sizeof(records)));
std::copy(keysV.begin(), keysV.end(), root.begin());
std::copy(outputV.begin(), outputV.end(), root.begin() + 4);
std::copy(recordsV.begin(), recordsV.end(), root.begin() + 12);
const auto store = [](std::uint32_t* at, std::uint64_t pointer) {
at[0] = static_cast<std::uint32_t>(pointer);
at[1] = static_cast<std::uint32_t>(pointer >> 32u);
};
const auto descriptor0 = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D);
const auto descriptor1 = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D);
std::copy(descriptor0.begin(), descriptor0.end(), descriptors[0].begin() + 8);
std::copy(descriptor0.begin(), descriptor0.end(), descriptors[1].begin() + 8);
std::copy(descriptor1.begin(), descriptor1.end(), descriptors[2].begin() + 8);
for (std::uint32_t record = 0; record < 3u; record++) {
store(records.data() + record * 4u + 2u, address(links[record].data()));
store(links[record].data() + 4u, address(descriptors[record].data()));
}
store(records.data() + 3u * 4u + 2u, address(links[3].data()));
const std::array<std::uint32_t, 2> userData{static_cast<std::uint32_t>(address(root.data())), static_cast<std::uint32_t>(address(root.data()) >> 32u)};
const std::vector<std::uint32_t> code{0xf4080200u, 0xfa000000u, 0xf4080700u, 0xfa000010u, 0xf4080800u, 0xfa000020u, 0xf4080100u, 0xfa000030u, 0x7e200500u,
0x8f108210u, 0xf4200444u, 0x20000000u, 0x8711ff11u, 0x000000ffu, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf4040b96u, 0xfa000010u, 0xf40c0917u,
0xfa000020u, 0xf09c0f08u, 0x01090000u, 0xe0700000u, 0x80070000u, 0xbf810000u};
auto request = TableRequest(code, userData);
request.shader.codeAddress = 0x37000u;
const auto plan = GetResourcePlan(request);
require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "pointer chain image table: a T# behind two pointers was not planned as a table");
const auto& column = *plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
require(column.indirect && !column.address && column.stride == 16u && column.addend == 0u && column.offset == 8u && column.dwordCount == 8u && column.maxKey == 0xffu, "pointer chain image table: the record column is wrong");
require(column.indirectOffset == 0u && column.indirectImmediate == 0x20u && column.pointerHops == std::vector<PointerHop>{{0u, 0x10u}}, "pointer chain image table: the pointer hops are wrong");
for (const auto& memory : plan->memoryInfo) {
require(memory.kind != ResourceKind::ScalarAddress || memory.planningOnly, "pointer chain image table: a load along the pointer chain still reads through BDA");
}
require(!plan->info.usesDma, "pointer chain image table: the shader still reads through BDA");
const auto compiled = CompileTable(request);
const auto& tables = compiled.capture->snapshot.tables;
const auto& snapshot = tables.tables.at(0);
require(snapshot.keys == Entries && snapshot.codes[0] == snapshot.codes[1] && snapshot.codes[0] != snapshot.codes[2] && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && (snapshot.codes[2] & Abi::PoisonFlag) == 0u, "pointer chain image table: the records behind two pointers do not resolve to one element per distinct T#");
require(tables.words.at(snapshot.codes[0]).dwords[0] == descriptor0[0] && tables.words.at(snapshot.codes[2]).dwords[0] == descriptor1[0] && tables.words.at(snapshot.codes[2]).dwords[1] == descriptor1[1], "pointer chain image table: the snapshot words are wrong");
const auto unmapped = Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::Unmapped));
require(snapshot.codes[3] == unmapped && snapshot.codes[4] == unmapped && snapshot.codes[255] == unmapped, "pointer chain image table: a record whose chain reaches a null pointer is not Unmapped poison");
for (std::uint32_t record = 0; record < 3u; record++) {
require(TableCovered(compiled.regions, links[record].data() + 4u, 8u) && TableCovered(compiled.regions, descriptors[record].data() + 8u, 32u), "pointer chain image table: the captured regions miss a link or a T#");
const auto link = address(links[record].data() + 4u);
require(std::ranges::any_of(tables.ranges, [&](const auto& range) { return range.first <= link && link + 8u <= range.first + range.second; }), "pointer chain image table: the snapshot ranges miss a link of the chain");
}
store(links[1].data() + 4u, address(descriptors[2].data()));
const auto relinked = CompileTable(request);
const auto& changed = relinked.capture->snapshot.tables.tables.at(0);
require(changed.codes[1] == changed.codes[2] && changed.codes[1] != changed.codes[0], "pointer chain image table: changing the second pointer of a record did not change its T#");
#if ANYPS5_ENABLE_SPIRV_TOOLS
static_cast<void>(ValidateAndOptimizeSpirv(compiled.result->spirv, request.target.vulkanVersion, request.target.spirvVersion));
#endif
store(links[1].data() + 4u, address(descriptors[1].data()));
}
void verifyDescriptorPhis() {
using namespace ShaderRecompiler;
constexpr std::uint32_t Format8888UNorm = 56;
@@ -6149,6 +6220,7 @@ int main(int argc, char** argv) {
verifyEvaluableHeapSamplerDirect();
verifySelectedRecordUniformIndex();
verifyChainedTableOffset();
verifyPointerChainImageTable();
std::cout << "Image table keys, maps, records and registration passed\n";
return 0;
}
@@ -6202,6 +6274,7 @@ int main(int argc, char** argv) {
verifyPointerImageTable();
verifyPointerTableOffsetCarry();
verifyChainedTableOffset();
verifyPointerChainImageTable();
verifyDescriptorPhis();
verifyUniformTableImage();
verifyLaneSelectedDescriptors();
@@ -278,7 +278,16 @@ alignas(256) constexpr std::array<std::uint32_t, 32> IndirectRecordCode{
0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,
};
alignas(256) constexpr std::array<std::uint32_t, 34> PointerChainRecordCode{
0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u,
0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u, 0xf4240b02u, 0x24000008u, 0xf4040b96u, 0xfa000010u,
0xf40c0917u, 0xfa000020u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x7e0402ffu,
0x3e000000u, 0xf09c0f08u, 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u,
0x80070408u, 0xbf810000u,
};
alignas(256) std::array<std::uint32_t, Records * 16> Indirect{};
alignas(256) std::array<std::uint32_t, Records * 8> Links{};
constexpr std::array<std::uint32_t, 4> PointClamp{0x92u, (4u * 256u) << 12u, 0u, 0u};
constexpr std::array<std::uint32_t, 4> LinearWrap{0u, (4u * 256u) << 12u, (1u << 20u) | (1u << 22u), 0u};
@@ -460,12 +469,13 @@ Outcome RunIndirectRecords(AgcDriver::VulkanDevice& device, const std::array<std
std::copy(output.begin(), output.end(), Srt.begin() + 8);
std::copy(PointClamp.begin(), PointClamp.end(), Srt.begin() + 12);
const std::span<const std::uint32_t> span(code);
const std::array<ShaderRecompiler::MemoryRegion, 5> memory{{
const std::array<ShaderRecompiler::MemoryRegion, 6> memory{{
{reinterpret_cast<std::uintptr_t>(span.data()), std::as_bytes(span)},
{reinterpret_cast<std::uintptr_t>(Srt.data()), std::as_bytes(std::span(Srt))},
{reinterpret_cast<std::uintptr_t>(Table.data()), std::as_bytes(std::span(Table))},
{reinterpret_cast<std::uintptr_t>(Keys.data()), std::as_bytes(std::span(Keys))},
{reinterpret_cast<std::uintptr_t>(Indirect.data()), std::as_bytes(std::span(Indirect))},
{reinterpret_cast<std::uintptr_t>(Links.data()), std::as_bytes(std::span(Links))},
}};
return Execute(device, span, Srt.data(), memory, groups, Lanes, 64);
}
@@ -1216,6 +1226,45 @@ void RunPointerRecordTests(AgcDriver::VulkanDevice& device) {
RequireRed(changed, 1, Lanes, texels[0], "float texels after the T# behind record 1's pointer changed");
}
void RunPointerChainRecordTests(AgcDriver::VulkanDevice& device) {
Fill();
const auto floatImage = Image(FloatTexels.data(), Format32Float);
const auto uintImage = Image(UintTexels.data(), Format32UInt);
const auto sintImage = Image(SintTexels.data(), Format32SInt);
const std::array<std::array<std::uint32_t, 4>, 3> images{floatImage, uintImage, sintImage};
const std::array<std::function<std::uint32_t(std::uint32_t)>, 3> texels{
[](std::uint32_t x) { return std::bit_cast<std::uint32_t>(FloatTexels[x]); },
[](std::uint32_t x) { return UintTexels[x]; },
[](std::uint32_t x) { return static_cast<std::uint32_t>(SintTexels[x]); },
};
const std::array<const char*, 3> names{"float", "uint", "sint"};
const auto store = [](std::uint32_t* at, const void* pointer) {
const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(pointer));
at[0] = static_cast<std::uint32_t>(address);
at[1] = static_cast<std::uint32_t>(address >> 32u);
};
Table = {};
Indirect = {};
Links = {};
Keys = {};
for (std::uint32_t group = 0; group < 3; ++group) Keys[group] = 2u - group;
for (std::uint32_t record = 0; record < 3; ++record) {
const std::uint32_t entry = 2u - record;
std::copy(images[record].begin(), images[record].end(), Indirect.data() + entry * 16u + 8u);
store(Links.data() + record * 8u + 4u, Indirect.data() + entry * 16u);
store(Table.data() + record * (Stride / 4u) + 2u, Links.data() + record * 8u);
}
const auto chained = RunIndirectRecords(device, PointerChainRecordCode, 3);
Require(!Faulted(chained), "image table: a T# behind two pointers from its record faulted:\n" + chained.log);
Require(chained.imageTableCount == 3u, "image table: T#s behind two pointers from their records do not hold one element per distinct T# (" + std::to_string(chained.imageTableCount) + ")");
for (std::uint32_t group = 0; group < 3; ++group) RequireRed(chained, group, Lanes, texels[2u - group], std::string(names[2u - group]) + " texels of a T# behind two pointers from its record");
store(Links.data() + 8u + 4u, Indirect.data() + 2u * 16u);
const auto relinked = RunIndirectRecords(device, PointerChainRecordCode, 3);
RequireRed(relinked, 1, Lanes, texels[0], "float texels after record 1's second pointer moved to another T#");
}
void RunSplitRecordTests(AgcDriver::VulkanDevice& device) {
Fill();
const auto floatImage = Image(FloatTexels.data(), Format32Float);
@@ -1441,6 +1490,7 @@ int main() {
RunSelectedStoreTests(*device);
RunPointerTests(*device);
RunPointerRecordTests(*device);
RunPointerChainRecordTests(*device);
RunSplitRecordTests(*device);
RunPointerStorageWriteTests(*device);
RunGraphicsTests(*device);
@@ -28,6 +28,13 @@ struct KeyDomain {
bool operator==(const KeyDomain& other) const = default;
};
struct PointerHop {
std::uint32_t offset = 0;
std::uint32_t immediate = 0;
bool operator==(const PointerHop& other) const = default;
};
struct TableColumn {
std::uint32_t heapSource = 0;
std::uint32_t stride = 0;
@@ -42,13 +49,14 @@ struct TableColumn {
bool indirect = false;
std::uint32_t indirectOffset = 0;
std::uint32_t indirectImmediate = 0;
std::vector<PointerHop> pointerHops;
std::uint32_t foldedAddend = 0;
std::uint32_t foldedCarry = 0;
bool selected = false;
std::uint32_t selection = 0;
bool operator==(const TableColumn& other) const {
return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection;
return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && pointerHops == other.pointerHops && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection;
}
};
@@ -818,7 +818,18 @@ private:
records.states[record] = RecordState::Unmapped;
continue;
}
const std::uint64_t pointer = (static_cast<std::uint64_t>(Read(records.base + *highWord)) << 32u) | Read(records.base + *lowWord);
std::uint64_t pointer = (static_cast<std::uint64_t>(Read(records.base + *highWord)) << 32u) | Read(records.base + *lowWord);
for (const auto& hop : column.pointerHops) {
const auto low = ImageTableAbi::ScalarAddressDword(pointer, hop.offset, hop.immediate);
const auto high = ImageTableAbi::ScalarAddressDword(pointer, hop.offset, hop.immediate + 4u);
if (!low.has_value() || !high.has_value() || (*low & 3u) != 0u || *high != *low + sizeof(std::uint32_t) || !AddressAccessible(*low) || !AddressAccessible(*high)) {
records.states[record] = RecordState::Unmapped;
break;
}
pointer = (static_cast<std::uint64_t>(Read(*high)) << 32u) | Read(*low);
tables.ranges.emplace_back(*low, 2u * sizeof(std::uint32_t));
}
if (records.states[record] != RecordState::Read) continue;
DescriptorValue value;
value.dwordCount = column.sampler ? 4u : 8u;
std::uint64_t first = std::numeric_limits<std::uint64_t>::max();
@@ -190,6 +190,8 @@ private:
IrValue* offset = nullptr;
std::vector<std::uint32_t> memory;
std::vector<const IrValue*> reads;
std::vector<std::uint32_t> hopMemory;
std::vector<const IrValue*> hopReads;
std::optional<KeyDomain> keyDomain;
};
@@ -688,15 +690,44 @@ private:
return true;
}
bool TryMakeIndirectColumn(IrValue& handle, IrValue& pointer, bool sampler, std::uint32_t words, std::uint32_t descriptorOffset, std::uint32_t descriptorImmediate, TableColumnPlan& plan) {
if (pointer.Opcode() != IrOpcode::GetAddressResource || pointer.ArgumentCount() != 2u) {
bool TryMakeIndirectColumn(IrValue& handle, IrValue& descriptorPointer, bool sampler, std::uint32_t words, std::uint32_t descriptorOffset, std::uint32_t descriptorImmediate, TableColumnPlan& plan) {
std::vector<PointerHop> hops;
IrValue* pointer = &descriptorPointer;
while (pointer->Opcode() == IrOpcode::GetAddressResource && pointer->ArgumentCount() == 2u) {
std::uint32_t memoryIndex = 0;
if (ScalarAddressMemory(*pointer->Argument(0)->Resolve(), memoryIndex) == nullptr) {
break;
}
PointerHop hop;
IrValue* next = nullptr;
for (std::uint32_t dword = 0; dword < 2u; dword++) {
IrValue* read = pointer->Argument(dword)->Resolve();
const MemoryInfo* memory = ScalarAddressMemory(*read, memoryIndex);
std::uint32_t readOffset = 0;
if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read) || !immediateU32(read->Argument(1), readOffset)) {
return false;
}
IrValue* readPointer = read->Argument(0)->Resolve();
if (dword == 0u) {
hop = PointerHop{readOffset, memory->offset};
next = readPointer;
} else if (readOffset != hop.offset || memory->offset != hop.immediate + static_cast<std::uint32_t>(sizeof(std::uint32_t)) || (readPointer != next && !EquivalentValue(m_program.Resources(), next, readPointer))) {
return false;
}
plan.hopMemory.push_back(memoryIndex);
plan.hopReads.push_back(read);
}
hops.insert(hops.begin(), hop);
pointer = next;
}
if (pointer->Opcode() != IrOpcode::GetAddressResource || pointer->ArgumentCount() != 2u) {
return false;
}
IrValue* bufferHandle = nullptr;
IrValue* offset = nullptr;
std::uint32_t first = 0;
for (std::uint32_t dword = 0; dword < 2u; dword++) {
IrValue* read = pointer.Argument(dword)->Resolve();
IrValue* read = pointer->Argument(dword)->Resolve();
std::uint32_t memoryIndex = 0;
const MemoryInfo* memory = ScalarReadMemory(*read, memoryIndex);
if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) {
@@ -732,6 +763,7 @@ private:
column.indirect = true;
column.indirectOffset = descriptorOffset;
column.indirectImmediate = descriptorImmediate;
column.pointerHops = std::move(hops);
const auto domain = MatchKeyDomain(key);
DescriptorSource columnSource = heapSource;
columnSource.tableColumn = column;
@@ -1438,6 +1470,27 @@ private:
}
}
}
for (bool marked = true; marked;) {
marked = false;
for (const auto& plan : m_tableColumns) {
for (std::size_t index = 0; index < plan.hopReads.size(); index++) {
const IrValue* read = plan.hopReads[index];
if (std::ranges::find(m_tablePlanningReads, read) != m_tablePlanningReads.end()) {
continue;
}
const bool planned = std::ranges::all_of(read->Uses(), [&](const IrValue* pointer) {
return pointer->Opcode() == IrOpcode::GetAddressResource && std::ranges::all_of(pointer->Uses(), [&](const IrValue* user) {
return std::ranges::find(m_tablePlanningReads, user) != m_tablePlanningReads.end();
});
});
if (planned) {
m_tablePlanningReads.push_back(read);
m_tablePlanningMemory.push_back(plan.hopMemory[index]);
marked = true;
}
}
}
}
m_memoryUsersIndexed = false;
m_memoryUsers.clear();
}
+11 -3
View File
@@ -108,8 +108,8 @@ static_assert(sizeof(IrResourcePlan) == 728, "IrResourcePlan changed: update Enc
static_assert(sizeof(IrValue) == 152, "IrValue changed: update the plan value encoder");
static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder");
static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder");
static_assert(sizeof(DescriptorSource) == 168, "DescriptorSource changed: update the plan encoder");
static_assert(sizeof(TableColumn) == 88, "TableColumn changed: update the plan encoder");
static_assert(sizeof(DescriptorSource) == 192, "DescriptorSource changed: update the plan encoder");
static_assert(sizeof(TableColumn) == 112, "TableColumn changed: update the plan encoder");
static_assert(sizeof(KeyDomain) == 16, "KeyDomain changed: update the plan encoder");
static_assert(sizeof(ResourceBlock) == 80, "ResourceBlock changed: update the plan encoder");
static_assert(sizeof(SrtRead) == 16, "SrtRead changed: update the plan encoder");
@@ -149,7 +149,7 @@ static_assert(fieldCount<IrResourcePlan>() == 24, "IrResourcePlan changed: updat
static_assert(fieldCount<GuestRegister>() == 2, "GuestRegister changed: update the plan value encoder");
static_assert(fieldCount<MemoryInfo>() == 31, "MemoryInfo changed: update the plan encoder");
static_assert(fieldCount<DescriptorSource>() == 3, "DescriptorSource changed: update the plan encoder");
static_assert(fieldCount<TableColumn>() == 17, "TableColumn changed: update the plan encoder");
static_assert(fieldCount<TableColumn>() == 18, "TableColumn changed: update the plan encoder");
static_assert(fieldCount<KeyDomain>() == 4, "KeyDomain changed: update the plan encoder");
static_assert(fieldCount<ResourceBlock>() == 4, "ResourceBlock changed: update the plan encoder");
static_assert(fieldCount<SrtRead>() == 2, "SrtRead changed: update the plan encoder");
@@ -1047,6 +1047,10 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) {
out.Value(column.indirect);
out.Value(column.indirectOffset);
out.Value(column.indirectImmediate);
out.List(column.pointerHops, [](Writer& hops, const PointerHop& hop) {
hops.Value(hop.offset);
hops.Value(hop.immediate);
});
out.Value(column.foldedAddend);
out.Value(column.foldedCarry);
out.Value(column.selected);
@@ -1204,6 +1208,10 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) {
in.Value(column.indirect);
in.Value(column.indirectOffset);
in.Value(column.indirectImmediate);
in.List(column.pointerHops, 8, [](Reader& hops, PointerHop& hop) {
hops.Value(hop.offset);
hops.Value(hop.immediate);
});
in.Value(column.foldedAddend);
in.Value(column.foldedCarry);
in.Value(column.selected);