mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-23 09:03:22 -05:00
Yunus Shaikh
bab404e7af
Since #6243, each read whose record offset the CPU evaluates gets its own selected table column, so a shader that reads one heap at many such keys plans one image per key. Past 64 keys that is refused with "image resource limit exceeded", or with "sampled image/sampler pair limit exceeded" when each read is sampled twice, where the shader planned within both limits before. After planning the table columns, the tracker now estimates the images and sampled pairs that collecting the program would add, keyed as AddImage and AddSampledPair key them, from the planned column sources and the direct handles. If either is over its limit, it plans again with the columns of one heap unselected, the heap with the most selected columns first, until both fit or no heap is left. Those reads then share one keyed column, as before #6243: the union of their key domains and the largest key. A shader within both limits plans exactly as before, from one planning pass. The estimate counts images exactly. A direct sampler is told apart by its handle's words and whether the sample adjusts, before the canonicalization collecting it applies, so the pair count can only be too high and errs toward merging.
1986 lines
89 KiB
C++
1986 lines
89 KiB
C++
#include "Optimization/ResourceTracker.hpp"
|
|
#include "Optimization/SrtWalker.hpp"
|
|
#include "Optimization/ResourceMaterializer.hpp"
|
|
#include "SpirvBackend/SpirvBufferFormat.hpp"
|
|
#include "IntermediateRepresentation/IrBuilder.hpp"
|
|
#include "Optimization/SrtWalker/SrtInstructionPredicates.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <bit>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <limits>
|
|
#include <map>
|
|
#include <span>
|
|
#include <string>
|
|
#include <tuple>
|
|
#include <unordered_map>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
namespace ShaderRecompiler {
|
|
namespace {
|
|
|
|
constexpr std::uint32_t samplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
|
|
constexpr std::uint32_t samplerDword3ReservedMask = 0x3ffff000u;
|
|
|
|
[[noreturn]] void fail(const std::string& message) {
|
|
throw std::runtime_error(message);
|
|
}
|
|
|
|
// Debug aid: APS5_TRACE_BDA=1 names the accesses that make a program address-based (see Collect).
|
|
bool bdaTraceEnabled() {
|
|
static const bool enabled = std::getenv("APS5_TRACE_BDA") != nullptr;
|
|
return enabled;
|
|
}
|
|
constexpr unsigned bdaTraceLimit = 8;
|
|
|
|
std::string formatHex32(std::uint32_t value) {
|
|
static constexpr char digits[] = "0123456789abcdef";
|
|
std::string hex(8u, '0');
|
|
for (std::uint32_t index = 0; index < 8u; index++) {
|
|
hex[7u - index] = digits[(value >> (index * 4u)) & 0xfu];
|
|
}
|
|
return hex;
|
|
}
|
|
|
|
std::string describeValueChain(const IrValue* value, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return "Immediate";
|
|
}
|
|
std::string text = std::string(IrOpcodeName(value->Opcode()));
|
|
if (value->Opcode() == IrOpcode::LoadAddressU32 || value->Opcode() == IrOpcode::ReadConstBuffer) {
|
|
text += "@pc=0x" + formatHex32(value->Flags<MemoryFlags>().pc);
|
|
}
|
|
if (depth == 0u || value->ArgumentCount() == 0u) {
|
|
return text;
|
|
}
|
|
text += "(";
|
|
for (std::uint32_t index = 0; index < value->ArgumentCount(); index++) {
|
|
if (index != 0u) {
|
|
text += ", ";
|
|
}
|
|
text += describeValueChain(value->Argument(index), depth - 1u);
|
|
}
|
|
text += ")";
|
|
return text;
|
|
}
|
|
|
|
std::uint32_t possibleU32Bits(const IrValue* value) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return value->Type() == IrType::U32 ? value->ImmediateU32() : std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
switch (value->Opcode()) {
|
|
case IrOpcode::BitwiseAnd32:
|
|
return possibleU32Bits(value->Argument(0)) & possibleU32Bits(value->Argument(1));
|
|
case IrOpcode::BitwiseOr32:
|
|
return possibleU32Bits(value->Argument(0)) | possibleU32Bits(value->Argument(1));
|
|
case IrOpcode::ShiftLeftLogical32: {
|
|
const IrValue* shift = value->Argument(1)->Resolve();
|
|
return shift->HasImmediate() && shift->Type() == IrType::U32 ? possibleU32Bits(value->Argument(0)) << (shift->ImmediateU32() & 31u) : std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
default:
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
}
|
|
|
|
std::uint32_t byteExtent(const MemoryInfo& memory) {
|
|
const auto bytes = std::max((memory.dataBits + 7u) / 8u, 1u);
|
|
const auto count = std::max(memory.dataDwords, 1u);
|
|
const auto end = static_cast<std::uint64_t>(memory.offset) + static_cast<std::uint64_t>(bytes) * count;
|
|
return end > std::numeric_limits<std::uint32_t>::max() ? std::numeric_limits<std::uint32_t>::max() : static_cast<std::uint32_t>(end);
|
|
}
|
|
|
|
class Tracker {
|
|
public:
|
|
explicit Tracker(IrProgram& program) : m_program(program), m_info(program.Resources().info), m_builder(program) {
|
|
m_info.buffers.clear();
|
|
m_info.images.clear();
|
|
m_info.samplers.clear();
|
|
m_info.sampledPairs.clear();
|
|
m_info.usesDma = false;
|
|
m_info.bdaWrites = false;
|
|
}
|
|
|
|
void Run() {
|
|
if (m_program.Resources().resourceTrackingComplete) {
|
|
fail("resources already tracked");
|
|
}
|
|
if (!m_program.Resources().srtPlanComplete) {
|
|
fail("SRT plan is not ready");
|
|
}
|
|
SplitDescriptorPhis();
|
|
PlanTableColumns();
|
|
std::unordered_map<const IrBlock*, std::uint32_t> blockIndices;
|
|
for (const IrBlock* block : m_program.BlockOrder()) {
|
|
blockIndices.emplace(block, static_cast<std::uint32_t>(blockIndices.size()));
|
|
}
|
|
m_blockSources.assign(blockIndices.size(), {});
|
|
m_blockImages.assign(blockIndices.size(), {});
|
|
for (auto& block : m_program.Blocks()) {
|
|
const auto found = blockIndices.find(block.get());
|
|
m_currentBlock = found == blockIndices.end() ? std::numeric_limits<std::uint32_t>::max() : found->second;
|
|
for (IrValue* inst : block->Instructions()) {
|
|
Collect(*inst);
|
|
}
|
|
}
|
|
m_currentBlock = std::numeric_limits<std::uint32_t>::max();
|
|
PlanResourceControlFlow();
|
|
if (m_bdaTraces > bdaTraceLimit) std::fprintf(stderr, "[bda] %u more address accesses in this program not shown\n", m_bdaTraces - bdaTraceLimit);
|
|
CheckTableStores();
|
|
AssignTables();
|
|
LinkImageAliases();
|
|
for (const auto& patch : m_handlePatches) {
|
|
patch.handle->SetFlags<std::uint32_t>(patch.resource);
|
|
}
|
|
for (const auto& patch : m_memoryPatches) {
|
|
auto& memory = m_program.Resources().memoryInfo[patch.index];
|
|
memory.resource = patch.resource;
|
|
if (patch.hasSampler) {
|
|
memory.sampler = patch.sampler;
|
|
}
|
|
}
|
|
std::vector<const IrValue*> referenced;
|
|
for (const auto& plan : m_tableColumns) {
|
|
const auto& column = *m_sources[plan.source].tableColumn;
|
|
IrValue* offset = column.selected ? &m_builder.Constant(column.addend) : plan.offset;
|
|
for (std::uint32_t argument = 0; argument < plan.handle->ArgumentCount(); argument++) {
|
|
plan.handle->ReplaceArgument(argument, offset);
|
|
}
|
|
KeepSourceAlive(*plan.handle, m_sources[column.heapSource], referenced);
|
|
if (column.selected) {
|
|
KeepSourceAlive(*plan.handle, m_sources[column.selection], referenced);
|
|
}
|
|
if (plan.keyDomain.has_value()) {
|
|
KeepSourceAlive(*plan.handle, m_sources[plan.keyDomain->source], referenced);
|
|
}
|
|
}
|
|
for (const auto index : m_tablePlanningMemory) {
|
|
m_program.Resources().memoryInfo[index].planningOnly = true;
|
|
}
|
|
std::erase_if(m_program.Metadata().dynamicReads, [&](IrValue* value) {
|
|
const IrValue* inst = value->Resolve();
|
|
return std::ranges::find(m_tablePlanningReads, inst) != m_tablePlanningReads.end();
|
|
});
|
|
m_program.Resources().descriptorSources = std::move(m_sources);
|
|
m_program.Resources().info = std::move(m_info);
|
|
m_program.Resources().resourceTrackingComplete = true;
|
|
}
|
|
|
|
private:
|
|
struct HandlePatch {
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t resource = 0;
|
|
};
|
|
|
|
struct MemoryPatch {
|
|
std::uint32_t index = 0;
|
|
std::uint32_t resource = 0;
|
|
std::uint32_t sampler = 0;
|
|
bool hasSampler = false;
|
|
};
|
|
|
|
struct TableColumnPlan {
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t source = 0;
|
|
IrValue* offset = nullptr;
|
|
std::vector<std::uint32_t> memory;
|
|
std::vector<const IrValue*> reads;
|
|
std::optional<KeyDomain> keyDomain;
|
|
};
|
|
|
|
void MakeSource(const IrValue& handle, std::uint32_t width, bool sampler, bool sampleAdjust, DescriptorSource& descriptor) {
|
|
if (handle.ArgumentCount() != width) {
|
|
fail(std::string(IrOpcodeName(handle.Opcode())) + " has " + std::to_string(handle.ArgumentCount()) + " descriptor dwords, expected " + std::to_string(width));
|
|
}
|
|
descriptor.dwordCount = width;
|
|
for (std::uint32_t i = 0; i < width; i++) {
|
|
descriptor.dwords[i] = handle.Argument(i)->Resolve();
|
|
}
|
|
if (sampleAdjust) {
|
|
descriptor.dwords[3] = canonicalizeSampleAdjustDword3(descriptor.dwords[3]);
|
|
}
|
|
const IrValue* dword0 = descriptor.dwords[0]->Resolve();
|
|
if (sampler && dword0->HasImmediate() && dword0->Type() == IrType::U32 && (dword0->ImmediateU32() & samplerBorderClampMask) == 0u) {
|
|
descriptor.dwords[3] = &m_builder.Constant(0u);
|
|
}
|
|
}
|
|
|
|
IrValue* canonicalizeSampleAdjustDword3(IrValue* value) {
|
|
for (;;) {
|
|
value = value->Resolve();
|
|
if ((possibleU32Bits(value) & ~samplerDword3ReservedMask) == 0u) {
|
|
return &m_builder.Constant(0u);
|
|
}
|
|
if (value->Opcode() != IrOpcode::BitwiseOr32) {
|
|
return value;
|
|
}
|
|
IrValue* left = value->Argument(0)->Resolve();
|
|
IrValue* right = value->Argument(1)->Resolve();
|
|
const bool leftReserved = (possibleU32Bits(left) & ~samplerDword3ReservedMask) == 0u;
|
|
const bool rightReserved = (possibleU32Bits(right) & ~samplerDword3ReservedMask) == 0u;
|
|
if (leftReserved) {
|
|
value = right;
|
|
} else if (rightReserved) {
|
|
value = left;
|
|
} else {
|
|
return value;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool ValidateSource(const DescriptorSource& descriptor, std::uint32_t& badDword) const {
|
|
constexpr SrtWalker walker;
|
|
for (std::uint32_t i = 0; i < descriptor.dwordCount; i++) {
|
|
badDword = i;
|
|
if (descriptor.dwords[i]->Resolve()->Type() != IrType::U32) {
|
|
return false;
|
|
}
|
|
if (!walker.ValidateRuntimeValue(m_program.Resources(), descriptor.dwords[i])) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
std::uint32_t InternSource(const DescriptorSource& descriptor) {
|
|
for (std::uint32_t candidate = 0; candidate < m_sources.size(); candidate++) {
|
|
const auto& current = m_sources[candidate];
|
|
if (current.dwordCount != descriptor.dwordCount || current.tableColumn != descriptor.tableColumn) {
|
|
continue;
|
|
}
|
|
bool same = true;
|
|
for (std::uint32_t i = 0; i < descriptor.dwordCount; i++) {
|
|
same = same && EquivalentValue(m_program.Resources(), current.dwords[i], descriptor.dwords[i]);
|
|
}
|
|
if (same) {
|
|
NoteBlockSource(candidate);
|
|
return candidate;
|
|
}
|
|
}
|
|
m_sources.push_back(descriptor);
|
|
NoteBlockSource(static_cast<std::uint32_t>(m_sources.size() - 1));
|
|
return static_cast<std::uint32_t>(m_sources.size() - 1);
|
|
}
|
|
|
|
void NoteBlockImage(std::uint32_t image) {
|
|
if (m_currentBlock >= m_blockImages.size()) {
|
|
return;
|
|
}
|
|
auto& images = m_blockImages[m_currentBlock];
|
|
if (std::ranges::find(images, image) == images.end()) {
|
|
images.push_back(image);
|
|
}
|
|
}
|
|
|
|
void NoteBlockSource(std::uint32_t source) {
|
|
if (m_currentBlock >= m_blockSources.size()) {
|
|
return;
|
|
}
|
|
auto& sources = m_blockSources[m_currentBlock];
|
|
if (std::ranges::find(sources, source) == sources.end()) {
|
|
sources.push_back(source);
|
|
}
|
|
}
|
|
|
|
void PlanResourceControlFlow() {
|
|
const auto& order = m_program.BlockOrder();
|
|
const auto& info = m_program.Metadata().blockInfo;
|
|
if (order.empty() || order.size() != info.size()) {
|
|
return;
|
|
}
|
|
std::unordered_map<const IrBlock*, std::uint32_t> indexOf;
|
|
std::unordered_map<std::uint32_t, std::uint32_t> idToIndex;
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
indexOf.emplace(order[index], index);
|
|
idToIndex.emplace(info[index].id, index);
|
|
}
|
|
std::vector<ResourceBlock> controlFlow(order.size());
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
auto& block = controlFlow[index];
|
|
const auto& terminator = info[index].terminator;
|
|
const auto target = [&](std::uint32_t id) { return idToIndex.find(id); };
|
|
const auto trueEdge = target(terminator.trueBlock);
|
|
const auto falseEdge = target(terminator.falseBlock);
|
|
if (terminator.kind == TerminatorKind::ConditionalBranch && !terminator.loopHeader && info[index].condition != nullptr && trueEdge != idToIndex.end() && falseEdge != idToIndex.end()) {
|
|
block.condition = info[index].condition;
|
|
block.successors = {trueEdge->second, falseEdge->second};
|
|
} else if (terminator.kind != TerminatorKind::Return) {
|
|
for (const IrBlock* successor : order[index]->Successors()) {
|
|
const auto found = indexOf.find(successor);
|
|
if (found == indexOf.end()) {
|
|
return;
|
|
}
|
|
block.successors.push_back(found->second);
|
|
}
|
|
}
|
|
block.sources = m_blockSources[index];
|
|
block.images = m_blockImages[index];
|
|
}
|
|
std::vector<std::uint8_t> unconditional(order.size());
|
|
for (std::uint32_t index = 0; index < order.size() && unconditional[index] == 0u;) {
|
|
unconditional[index] = 1u;
|
|
if (controlFlow[index].successors.size() != 1u) {
|
|
break;
|
|
}
|
|
index = controlFlow[index].successors.front();
|
|
}
|
|
std::vector<std::uint8_t> reached(m_sources.size());
|
|
std::vector<std::uint8_t> reachedImages(m_info.images.size());
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
if (unconditional[index] == 0u) {
|
|
continue;
|
|
}
|
|
for (const auto source : controlFlow[index].sources) {
|
|
reached[source] = 1u;
|
|
}
|
|
for (const auto image : controlFlow[index].images) {
|
|
reachedImages[image] = 1u;
|
|
}
|
|
}
|
|
bool guarded = false;
|
|
for (const auto& block : controlFlow) {
|
|
for (const auto source : block.sources) {
|
|
guarded = guarded || reached[source] == 0u;
|
|
}
|
|
for (const auto image : block.images) {
|
|
guarded = guarded || reachedImages[image] == 0u;
|
|
}
|
|
}
|
|
if (guarded) {
|
|
m_program.Resources().controlFlow = std::move(controlFlow);
|
|
}
|
|
}
|
|
|
|
static bool immediateU32(IrValue* value, std::uint32_t& result) {
|
|
value = value->Resolve();
|
|
if (!value->HasImmediate() || value->Type() != IrType::U32) {
|
|
return false;
|
|
}
|
|
result = value->ImmediateU32();
|
|
return true;
|
|
}
|
|
|
|
const MemoryInfo* ScalarReadMemory(const IrValue& read, std::uint32_t& index) const {
|
|
if (read.Opcode() != IrOpcode::ReadConstBuffer || read.ArgumentCount() != 2u) {
|
|
return nullptr;
|
|
}
|
|
index = read.Flags<MemoryFlags>().index;
|
|
if (index >= m_program.Resources().memoryInfo.size()) {
|
|
return nullptr;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[index];
|
|
return memory.kind == ResourceKind::ScalarBuffer && memory.dataBits == 32u && memory.dataDwords == 1u ? &memory : nullptr;
|
|
}
|
|
|
|
const MemoryInfo* ScalarAddressMemory(const IrValue& read, std::uint32_t& index) const {
|
|
if (read.Opcode() != IrOpcode::LoadAddressU32 || read.ArgumentCount() != 4u) {
|
|
return nullptr;
|
|
}
|
|
index = read.Flags<MemoryFlags>().index;
|
|
if (index >= m_program.Resources().memoryInfo.size()) {
|
|
return nullptr;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[index];
|
|
std::uint32_t high = 0;
|
|
const IrValue* active = read.Argument(3)->Resolve();
|
|
const bool always = active->HasImmediate() && active->Type() == IrType::U1 && active->ImmediateBool();
|
|
return memory.kind == ResourceKind::ScalarAddress && !memory.addressIsFull && memory.dataBits == 32u && memory.dataDwords == 1u && immediateU32(read.Argument(2), high) && high == 0u && always ? &memory : nullptr;
|
|
}
|
|
|
|
static bool AccessesMemory(IrOpcode op) {
|
|
return BufferAccessOf(op) != BufferAccess::None || AddressOpcodeInfoOf(op).access != AddressAccess::None || ImageOpcodeInfoOf(op).access != ImageAccess::None;
|
|
}
|
|
|
|
void IndexMemoryUsers() {
|
|
for (const auto& block : m_program.Blocks()) {
|
|
for (const IrValue* inst : block->Instructions()) {
|
|
if (AccessesMemory(inst->Opcode())) {
|
|
m_memoryUsers[inst->Flags<MemoryFlags>().index].push_back(inst);
|
|
}
|
|
}
|
|
}
|
|
m_memoryUsersIndexed = true;
|
|
}
|
|
|
|
bool MemoryIndexBelongsTo(std::uint32_t index, const IrValue& owner) const {
|
|
if (m_memoryUsersIndexed) {
|
|
const auto users = m_memoryUsers.find(index);
|
|
return users == m_memoryUsers.end() || std::ranges::all_of(users->second, [&](const IrValue* user) { return user == &owner; });
|
|
}
|
|
for (const auto& block : m_program.Blocks()) {
|
|
for (const IrValue* inst : block->Instructions()) {
|
|
if (!AccessesMemory(inst->Opcode()) || inst == &owner) {
|
|
continue;
|
|
}
|
|
if (inst->Flags<MemoryFlags>().index == index) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool MakeRuntimeSource(const IrValue& handle, IrOpcode expected, std::uint32_t width, std::uint32_t& source, DescriptorSource& descriptor) {
|
|
if (handle.Opcode() != expected) {
|
|
return false;
|
|
}
|
|
MakeSource(handle, width, false, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
if (!ValidateSource(descriptor, badDword)) {
|
|
return false;
|
|
}
|
|
source = InternSource(descriptor);
|
|
return true;
|
|
}
|
|
|
|
static bool MatchScale(IrValue* value, IrValue*& key, std::uint32_t& stride) {
|
|
value = value->Resolve();
|
|
if (value->ArgumentCount() != 2u) {
|
|
return false;
|
|
}
|
|
std::uint32_t immediate = 0;
|
|
if (value->Opcode() == IrOpcode::ShiftLeftLogical32) {
|
|
if (!immediateU32(value->Argument(1), immediate) || immediate >= 32u) {
|
|
return false;
|
|
}
|
|
key = value->Argument(0)->Resolve();
|
|
stride = 1u << immediate;
|
|
return true;
|
|
}
|
|
if (value->Opcode() != IrOpcode::IMul32) {
|
|
return false;
|
|
}
|
|
if (immediateU32(value->Argument(1), immediate)) {
|
|
key = value->Argument(0)->Resolve();
|
|
} else if (immediateU32(value->Argument(0), immediate)) {
|
|
key = value->Argument(1)->Resolve();
|
|
} else {
|
|
return false;
|
|
}
|
|
stride = immediate;
|
|
return true;
|
|
}
|
|
|
|
static bool MatchAffineOffset(IrValue* value, IrValue*& key, std::uint32_t& stride, std::uint32_t& addend) {
|
|
value = value->Resolve();
|
|
addend = 0;
|
|
bool matched = MatchScale(value, key, stride);
|
|
if (!matched && value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) {
|
|
if (immediateU32(value->Argument(0), addend)) {
|
|
matched = MatchScale(value->Argument(1), key, stride);
|
|
} else if (immediateU32(value->Argument(1), addend)) {
|
|
matched = MatchScale(value->Argument(0), key, stride);
|
|
}
|
|
}
|
|
return matched && stride >= 4u && stride % 4u == 0u && key->Type() == IrType::U32;
|
|
}
|
|
|
|
static bool MatchMaskedOffset(IrValue* value, IrValue*& key, std::uint32_t& stride) {
|
|
value = value->Resolve();
|
|
if (value->Opcode() != IrOpcode::BitwiseAnd32 || value->ArgumentCount() != 2u) {
|
|
return false;
|
|
}
|
|
std::uint32_t mask = 0;
|
|
IrValue* scaled = nullptr;
|
|
if (immediateU32(value->Argument(1), mask)) {
|
|
scaled = value->Argument(0);
|
|
} else if (immediateU32(value->Argument(0), mask)) {
|
|
scaled = value->Argument(1);
|
|
} else {
|
|
return false;
|
|
}
|
|
return MatchScale(scaled, key, stride) && std::has_single_bit(stride) && stride >= 4u && (mask & ~(stride - 1u)) != 0u && key->Type() == IrType::U32;
|
|
}
|
|
|
|
static bool ReadsThroughDescriptor(const IrValue* value) {
|
|
std::vector<const IrValue*> pending{value};
|
|
std::vector<const IrValue*> seen;
|
|
while (!pending.empty()) {
|
|
const IrValue* current = pending.back()->Resolve();
|
|
pending.pop_back();
|
|
if (std::find(seen.begin(), seen.end(), current) != seen.end()) continue;
|
|
seen.push_back(current);
|
|
if (current->Opcode() == IrOpcode::ReadConstBuffer || current->Opcode() == IrOpcode::LoadAddressU32) return true;
|
|
for (std::size_t index = 0; index < current->ArgumentCount(); index++) pending.push_back(current->Argument(index));
|
|
}
|
|
return false;
|
|
}
|
|
|
|
std::optional<KeyDomain> MatchKeyDomain(IrValue* key) {
|
|
key = key->Resolve();
|
|
DescriptorSource direct;
|
|
direct.dwordCount = 1u;
|
|
direct.dwords[0] = key;
|
|
std::uint32_t badDword = 0;
|
|
if (ValidateSource(direct, badDword) && !ReadsThroughDescriptor(key)) {
|
|
KeyDomain domain{InternSource(direct), 0u, 4u};
|
|
domain.value = true;
|
|
return domain;
|
|
}
|
|
std::uint32_t memoryIndex = 0;
|
|
const MemoryInfo* memory = ScalarReadMemory(*key, memoryIndex);
|
|
if (memory == nullptr) {
|
|
return std::nullopt;
|
|
}
|
|
DescriptorSource source;
|
|
std::uint32_t sourceIndex = 0;
|
|
if (!MakeRuntimeSource(*key->Argument(0)->Resolve(), IrOpcode::GetBufferResource, 4u, sourceIndex, source)) {
|
|
return std::nullopt;
|
|
}
|
|
IrValue* selector = nullptr;
|
|
std::uint32_t stride = 0;
|
|
std::uint32_t addend = 0;
|
|
if (MatchAffineOffset(key->Argument(1), selector, stride, addend)) {
|
|
return KeyDomain{sourceIndex, memory->offset + addend, stride};
|
|
}
|
|
const auto bits = possibleU32Bits(key->Argument(1));
|
|
const auto zeros = bits == 0u ? 31u : std::min<std::uint32_t>(static_cast<std::uint32_t>(std::countr_zero(bits)), 31u);
|
|
return KeyDomain{sourceIndex, memory->offset, std::max(4u, 1u << zeros)};
|
|
}
|
|
|
|
static IrValue* SplitImmediateAddend(IrValue* value, std::uint32_t& addend) {
|
|
value = value->Resolve();
|
|
addend = 0;
|
|
while (value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) {
|
|
std::uint32_t immediate = 0;
|
|
IrValue* rest = nullptr;
|
|
if (immediateU32(value->Argument(0), immediate)) rest = value->Argument(1)->Resolve();
|
|
else if (immediateU32(value->Argument(1), immediate)) rest = value->Argument(0)->Resolve();
|
|
else break;
|
|
addend += immediate;
|
|
value = rest;
|
|
}
|
|
return value;
|
|
}
|
|
|
|
bool TryMakeTableColumn(IrValue& handle, bool sampler, bool r128, TableColumnPlan& plan) {
|
|
const auto expected = sampler ? IrOpcode::GetSamplerResource : IrOpcode::GetImageResource;
|
|
const std::uint32_t width = sampler ? 4u : 8u;
|
|
if (handle.Opcode() != expected || handle.ArgumentCount() != width) {
|
|
return false;
|
|
}
|
|
std::uint32_t words = width;
|
|
if (!sampler && r128) {
|
|
bool zeroTail = true;
|
|
for (std::uint32_t dword = 4u; dword < 8u; dword++) {
|
|
std::uint32_t immediate = 0;
|
|
zeroTail = zeroTail && immediateU32(handle.Argument(dword), immediate) && immediate == 0u;
|
|
}
|
|
if (zeroTail) {
|
|
words = 4u;
|
|
}
|
|
}
|
|
IrValue* heapHandle = nullptr;
|
|
IrValue* offset = nullptr;
|
|
IrValue* base = nullptr;
|
|
std::uint32_t baseAddend = 0;
|
|
std::uint32_t carry = 0;
|
|
bool folded = false;
|
|
std::uint32_t first = 0;
|
|
bool address = false;
|
|
for (std::uint32_t dword = 0; dword < words; dword++) {
|
|
IrValue* read = handle.Argument(dword)->Resolve();
|
|
std::uint32_t memoryIndex = 0;
|
|
const MemoryInfo* memory = ScalarReadMemory(*read, memoryIndex);
|
|
const bool addressRead = memory == nullptr && (memory = ScalarAddressMemory(*read, memoryIndex)) != nullptr;
|
|
if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) {
|
|
return false;
|
|
}
|
|
std::uint32_t readAddend = 0;
|
|
IrValue* readBase = SplitImmediateAddend(read->Argument(1), readAddend);
|
|
if (dword == 0u) {
|
|
first = memory->offset;
|
|
address = addressRead;
|
|
base = readBase;
|
|
baseAddend = readAddend;
|
|
} else if (addressRead != address || memory->offset + readAddend != first + baseAddend + dword * static_cast<std::uint32_t>(sizeof(std::uint32_t))) {
|
|
return false;
|
|
} else if (readAddend != baseAddend) {
|
|
if (readAddend < baseAddend || static_cast<std::uint64_t>(memory->offset) + (readAddend - baseAddend) > 0x7fffffffu) {
|
|
return false;
|
|
}
|
|
folded = true;
|
|
carry = std::max(carry, readAddend - baseAddend);
|
|
}
|
|
IrValue* currentHandle = read->Argument(0)->Resolve();
|
|
if (heapHandle != nullptr && currentHandle != heapHandle && !EquivalentValue(m_program.Resources(), heapHandle, currentHandle)) {
|
|
return false;
|
|
}
|
|
if (heapHandle == nullptr) heapHandle = currentHandle;
|
|
if (dword == 0u) {
|
|
offset = read->Argument(1)->Resolve();
|
|
} else if (!EquivalentValue(m_program.Resources(), base, readBase)) {
|
|
return false;
|
|
}
|
|
plan.memory.push_back(memoryIndex);
|
|
plan.reads.push_back(read);
|
|
}
|
|
if (address) {
|
|
DescriptorSource direct;
|
|
MakeSource(handle, width, false, false, direct);
|
|
std::uint32_t badDword = 0;
|
|
if (ValidateSource(direct, badDword)) {
|
|
return false;
|
|
}
|
|
}
|
|
IrValue* key = nullptr;
|
|
std::uint32_t stride = 0;
|
|
std::uint32_t addend = 0;
|
|
bool masked = false;
|
|
std::uint32_t descriptorOffset = 0;
|
|
if (address && immediateU32(offset, descriptorOffset)) {
|
|
return TryMakeIndirectColumn(handle, *heapHandle, sampler, words, descriptorOffset, first, plan);
|
|
}
|
|
if (!MatchAffineOffset(offset, key, stride, addend)) {
|
|
if (!MatchMaskedOffset(offset, key, stride)) {
|
|
return false;
|
|
}
|
|
masked = true;
|
|
}
|
|
DescriptorSource heapSource;
|
|
std::uint32_t heapIndex = 0;
|
|
if (!MakeRuntimeSource(*heapHandle, address ? IrOpcode::GetAddressResource : IrOpcode::GetBufferResource, address ? 2u : 4u, heapIndex, heapSource)) {
|
|
return false;
|
|
}
|
|
TableColumn column;
|
|
column.heapSource = heapIndex;
|
|
column.stride = stride;
|
|
column.addend = addend;
|
|
column.offset = first;
|
|
column.dwordCount = words;
|
|
column.maxKey = possibleU32Bits(key);
|
|
column.sampler = sampler;
|
|
column.address = address;
|
|
column.foldedAddend = folded ? baseAddend : 0u;
|
|
column.foldedCarry = folded ? carry : 0u;
|
|
if (!address && !sampler && std::ranges::find(m_unselectedHandles, &handle) == m_unselectedHandles.end()) {
|
|
DescriptorSource direct;
|
|
MakeSource(handle, width, false, false, direct);
|
|
DescriptorSource selection;
|
|
selection.dwordCount = 1u;
|
|
selection.dwords[0] = offset;
|
|
std::uint32_t badDword = 0;
|
|
if (ValidateSource(direct, badDword) && ValidateSource(selection, badDword)) {
|
|
column.selected = true;
|
|
column.selection = InternSource(selection);
|
|
}
|
|
}
|
|
const auto domain = masked ? std::optional<KeyDomain>{} : MatchKeyDomain(key);
|
|
DescriptorSource columnSource = heapSource;
|
|
columnSource.tableColumn = column;
|
|
plan.handle = &handle;
|
|
plan.source = InternSource(columnSource);
|
|
plan.keyDomain = domain;
|
|
auto& shared = *m_sources[plan.source].tableColumn;
|
|
shared.maxKey = std::max(shared.maxKey, column.maxKey);
|
|
if (!domain.has_value()) {
|
|
shared.unkeyed = true;
|
|
shared.keyDomains.clear();
|
|
} else if (!shared.unkeyed && std::find(shared.keyDomains.begin(), shared.keyDomains.end(), *domain) == shared.keyDomains.end()) {
|
|
shared.keyDomains.push_back(*domain);
|
|
}
|
|
plan.offset = offset;
|
|
return true;
|
|
}
|
|
|
|
bool TryMakeIndirectColumn(IrValue& handle, IrValue& pointer, bool sampler, std::uint32_t words, std::uint32_t descriptorOffset, std::uint32_t descriptorImmediate, TableColumnPlan& plan) {
|
|
if (pointer.Opcode() != IrOpcode::GetAddressResource || pointer.ArgumentCount() != 2u) {
|
|
return false;
|
|
}
|
|
IrValue* bufferHandle = nullptr;
|
|
IrValue* offset = nullptr;
|
|
std::uint32_t first = 0;
|
|
for (std::uint32_t dword = 0; dword < 2u; dword++) {
|
|
IrValue* read = pointer.Argument(dword)->Resolve();
|
|
std::uint32_t memoryIndex = 0;
|
|
const MemoryInfo* memory = ScalarReadMemory(*read, memoryIndex);
|
|
if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) {
|
|
return false;
|
|
}
|
|
if (dword == 0u) {
|
|
first = memory->offset;
|
|
bufferHandle = read->Argument(0)->Resolve();
|
|
offset = read->Argument(1)->Resolve();
|
|
} else if (memory->offset != first + static_cast<std::uint32_t>(sizeof(std::uint32_t)) || read->Argument(0)->Resolve() != bufferHandle || !EquivalentValue(m_program.Resources(), offset, read->Argument(1))) {
|
|
return false;
|
|
}
|
|
}
|
|
IrValue* key = nullptr;
|
|
std::uint32_t stride = 0;
|
|
std::uint32_t addend = 0;
|
|
if (!MatchAffineOffset(offset, key, stride, addend)) {
|
|
return false;
|
|
}
|
|
DescriptorSource heapSource;
|
|
std::uint32_t heapIndex = 0;
|
|
if (!MakeRuntimeSource(*bufferHandle, IrOpcode::GetBufferResource, 4u, heapIndex, heapSource)) {
|
|
return false;
|
|
}
|
|
TableColumn column;
|
|
column.heapSource = heapIndex;
|
|
column.stride = stride;
|
|
column.addend = addend;
|
|
column.offset = first;
|
|
column.dwordCount = words;
|
|
column.maxKey = possibleU32Bits(key);
|
|
column.sampler = sampler;
|
|
column.indirect = true;
|
|
column.indirectOffset = descriptorOffset;
|
|
column.indirectImmediate = descriptorImmediate;
|
|
const auto domain = MatchKeyDomain(key);
|
|
DescriptorSource columnSource = heapSource;
|
|
columnSource.tableColumn = column;
|
|
plan.handle = &handle;
|
|
plan.source = InternSource(columnSource);
|
|
plan.keyDomain = domain;
|
|
auto& shared = *m_sources[plan.source].tableColumn;
|
|
shared.maxKey = std::max(shared.maxKey, column.maxKey);
|
|
if (!domain.has_value()) {
|
|
shared.unkeyed = true;
|
|
shared.keyDomains.clear();
|
|
} else if (!shared.unkeyed && std::find(shared.keyDomains.begin(), shared.keyDomains.end(), *domain) == shared.keyDomains.end()) {
|
|
shared.keyDomains.push_back(*domain);
|
|
}
|
|
plan.offset = offset;
|
|
return true;
|
|
}
|
|
|
|
const TableColumnPlan* FindTableColumn(const IrValue& handle) const {
|
|
const auto found = std::ranges::find_if(m_tableColumns, [&](const TableColumnPlan& plan) {
|
|
return plan.handle == &handle;
|
|
});
|
|
return found == m_tableColumns.end() ? nullptr : &*found;
|
|
}
|
|
|
|
bool PlanTableColumn(IrValue& handle, bool sampler, bool r128) {
|
|
if (FindTableColumn(handle) != nullptr) {
|
|
return true;
|
|
}
|
|
TableColumnPlan plan;
|
|
if (!TryMakeTableColumn(handle, sampler, r128, plan)) {
|
|
return false;
|
|
}
|
|
m_tableColumns.push_back(std::move(plan));
|
|
return true;
|
|
}
|
|
|
|
static constexpr std::uint32_t phiSearchDepth = 8u;
|
|
|
|
static bool SplittableImageRead(const IrValue& inst) {
|
|
switch (inst.Opcode()) {
|
|
case IrOpcode::ImageSampleRaw:
|
|
case IrOpcode::ImageGatherRaw:
|
|
case IrOpcode::ImageQueryLod:
|
|
case IrOpcode::ImageQueryDimensions:
|
|
case IrOpcode::ImageRead: return inst.Type() == IrType::U32x4;
|
|
default: return false;
|
|
}
|
|
}
|
|
|
|
void CollectDescriptorPhis(IrValue* value, std::vector<IrValue*>& phis, std::uint32_t depth) const {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return;
|
|
}
|
|
if (value->IsPhi()) {
|
|
if (ResolveInvariantPhi(m_program.Resources(), value) == nullptr && std::ranges::find(phis, value) == phis.end()) {
|
|
phis.push_back(value);
|
|
}
|
|
return;
|
|
}
|
|
if (depth != 0u && value->Opcode() == IrOpcode::ReadFirstLane && value->ArgumentCount() == 2u) {
|
|
CollectDescriptorPhis(value->Argument(0), phis, depth - 1u);
|
|
return;
|
|
}
|
|
if (depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) {
|
|
return;
|
|
}
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
CollectDescriptorPhis(value->Argument(index), phis, depth - 1u);
|
|
}
|
|
}
|
|
|
|
static IrValue* PhiIncoming(const IrValue& phi, const IrBlock* predecessor) {
|
|
for (std::size_t index = 0; index < phi.ArgumentCount(); index++) {
|
|
if (phi.PhiBlock(index) == predecessor) {
|
|
return phi.Argument(index);
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
IrValue* EmitBefore(IrValue& position, IrOpcode op, IrType type, std::span<IrValue* const> arguments, std::uint64_t flags = 0) {
|
|
IrValue& value = m_program.CreateValue(op, type, flags);
|
|
for (IrValue* argument : arguments) {
|
|
value.AddArgument(argument);
|
|
}
|
|
position.Parent()->InsertInstructionBefore(&position, &value);
|
|
return &value;
|
|
}
|
|
|
|
IrValue* EmitCopy(const IrValue& original, std::span<IrValue* const> arguments, IrValue& position) {
|
|
return EmitBefore(position, original.Opcode(), original.Type(), arguments, original.Flags<std::uint64_t>());
|
|
}
|
|
|
|
IrValue* Rematerialize(IrValue* value, IrValue& position, bool emit, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return value;
|
|
}
|
|
if (value->IsPhi()) {
|
|
IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value);
|
|
return invariant == nullptr || invariant->IsPhi() ? nullptr : Rematerialize(invariant, position, emit, depth);
|
|
}
|
|
if (depth == 0u) {
|
|
return nullptr;
|
|
}
|
|
const auto op = value->Opcode();
|
|
const bool register_ = op == IrOpcode::GetUserData;
|
|
const bool argumentless = (op == IrOpcode::GetShaderBase || op == IrOpcode::GetSrtResource) && value->ArgumentCount() == 0u;
|
|
const bool srtRead = op == IrOpcode::ReadConst && value->ArgumentCount() == 2u && value->Argument(0)->Resolve()->Opcode() == IrOpcode::GetSrtResource && value->Argument(1)->Resolve()->HasImmediate();
|
|
if (!register_ && !argumentless && !srtRead && !Detail::IsRuntimeUniformOp(op)) {
|
|
return nullptr;
|
|
}
|
|
std::vector<IrValue*> arguments;
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
IrValue* argument = register_ ? value->Argument(index) : Rematerialize(value->Argument(index), position, emit, depth - 1u);
|
|
if (argument == nullptr) {
|
|
return nullptr;
|
|
}
|
|
arguments.push_back(argument);
|
|
}
|
|
return emit ? EmitCopy(*value, arguments, position) : value;
|
|
}
|
|
|
|
IrValue* SubstituteEdge(IrValue* value, const IrBlock* predecessor, IrValue& position, bool emit, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
std::vector<IrValue*> phis;
|
|
CollectDescriptorPhis(value, phis, depth);
|
|
if (phis.empty()) {
|
|
return value;
|
|
}
|
|
if (value->IsPhi()) {
|
|
IrValue* incoming = PhiIncoming(*value, predecessor);
|
|
return incoming == nullptr ? nullptr : Rematerialize(incoming, position, emit, phiSearchDepth);
|
|
}
|
|
std::vector<IrValue*> arguments;
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
IrValue* argument = SubstituteEdge(value->Argument(index), predecessor, position, emit, depth - 1u);
|
|
if (argument == nullptr) {
|
|
return nullptr;
|
|
}
|
|
arguments.push_back(argument);
|
|
}
|
|
return emit ? EmitCopy(*value, arguments, position) : value;
|
|
}
|
|
|
|
IrValue& EdgeSelector(IrBlock& block, const std::vector<bool>& edges) {
|
|
for (const auto& selector : m_edgeSelectors) {
|
|
if (selector.block == &block && selector.edges == edges) {
|
|
return *selector.value;
|
|
}
|
|
}
|
|
IrValue& phi = m_program.CreateValue(IrOpcode::Phi, IrType::Bool);
|
|
const auto& predecessors = block.Predecessors();
|
|
for (std::size_t index = 0; index < predecessors.size(); index++) {
|
|
phi.AddPhiOperand(predecessors[index], &m_builder.ConstantBool(edges[index]));
|
|
}
|
|
block.InsertInstructionBefore(nullptr, &phi);
|
|
m_edgeSelectors.push_back({&block, edges, &phi});
|
|
return phi;
|
|
}
|
|
|
|
void SplitDescriptorPhi(IrValue& inst) {
|
|
const auto info = ImageOpcodeInfoOf(inst.Opcode());
|
|
const std::uint32_t handleCount = info.needsSampler ? 2u : 1u;
|
|
if (inst.ArgumentCount() < handleCount || inst.Parent() == nullptr) {
|
|
return;
|
|
}
|
|
std::array<IrValue*, 2> handles {};
|
|
std::array<bool, 2> split {};
|
|
std::vector<IrValue*> phis;
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
IrValue* handle = inst.Argument(slot)->Resolve();
|
|
const auto expected = slot == 0u ? IrOpcode::GetImageResource : IrOpcode::GetSamplerResource;
|
|
if (handle->Opcode() != expected || handle->ArgumentCount() != (slot == 0u ? 8u : 4u)) {
|
|
return;
|
|
}
|
|
handles[slot] = handle;
|
|
std::vector<IrValue*> handlePhis;
|
|
for (std::size_t dword = 0; dword < handle->ArgumentCount(); dword++) {
|
|
CollectDescriptorPhis(handle->Argument(dword), handlePhis, phiSearchDepth);
|
|
}
|
|
split[slot] = !handlePhis.empty();
|
|
for (IrValue* phi : handlePhis) {
|
|
if (std::ranges::find(phis, phi) == phis.end()) {
|
|
phis.push_back(phi);
|
|
}
|
|
}
|
|
}
|
|
if (phis.empty()) {
|
|
return;
|
|
}
|
|
if (!SplitDescriptorEdges(inst, handles, split, handleCount, phis)) {
|
|
SplitDescriptorWeb(inst, handles, split, handleCount);
|
|
}
|
|
}
|
|
|
|
bool SplitDescriptorEdges(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount, const std::vector<IrValue*>& phis) {
|
|
IrBlock* block = phis.front()->Parent();
|
|
if (block == nullptr) {
|
|
return false;
|
|
}
|
|
const auto& predecessors = block->Predecessors();
|
|
if (predecessors.size() < 2u) {
|
|
return false;
|
|
}
|
|
for (const IrValue* phi : phis) {
|
|
if (phi->Parent() != block || phi->PhiBlockCount() != predecessors.size()) {
|
|
return false;
|
|
}
|
|
for (const IrBlock* predecessor : predecessors) {
|
|
if (PhiIncoming(*phi, predecessor) == nullptr) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<std::uint32_t> edgeArm(predecessors.size());
|
|
std::vector<const IrBlock*> arms;
|
|
for (std::size_t edge = 0; edge < predecessors.size(); edge++) {
|
|
std::uint32_t arm = 0;
|
|
for (; arm < arms.size(); arm++) {
|
|
const bool same = std::ranges::all_of(phis, [&](const IrValue* phi) {
|
|
return EquivalentValue(m_program.Resources(), PhiIncoming(*phi, predecessors[edge]), PhiIncoming(*phi, arms[arm]));
|
|
});
|
|
if (same) {
|
|
break;
|
|
}
|
|
}
|
|
if (arm == arms.size()) {
|
|
arms.push_back(predecessors[edge]);
|
|
}
|
|
edgeArm[edge] = arm;
|
|
}
|
|
for (const IrBlock* arm : arms) {
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) {
|
|
if (SubstituteEdge(handles[slot]->Argument(dword), arm, inst, false, phiSearchDepth) == nullptr) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<IrValue*> conditions;
|
|
for (std::uint32_t arm = 0; arm + 1u < arms.size(); arm++) {
|
|
std::vector<bool> edges(edgeArm.size());
|
|
for (std::size_t edge = 0; edge < edgeArm.size(); edge++) {
|
|
edges[edge] = edgeArm[edge] == arm;
|
|
}
|
|
conditions.push_back(&EdgeSelector(*block, edges));
|
|
}
|
|
ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) {
|
|
return SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth);
|
|
});
|
|
return true;
|
|
}
|
|
|
|
template <typename Dword>
|
|
void ReplaceWithArmCopies(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount, std::span<IrValue* const> conditions, Dword&& dword) {
|
|
std::vector<IrValue*> copies;
|
|
const auto flags = inst.Flags<MemoryFlags>();
|
|
const auto memory = m_program.Resources().memoryInfo.at(flags.index);
|
|
for (std::uint32_t arm = 0; arm <= conditions.size(); arm++) {
|
|
std::vector<IrValue*> arguments(inst.Arguments().begin(), inst.Arguments().end());
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
if (!split[slot]) {
|
|
continue;
|
|
}
|
|
std::vector<IrValue*> dwords;
|
|
for (std::size_t index = 0; index < handles[slot]->ArgumentCount(); index++) {
|
|
dwords.push_back(dword(arm, slot, index));
|
|
}
|
|
arguments[slot] = EmitCopy(*handles[slot], dwords, inst);
|
|
}
|
|
auto copyFlags = flags;
|
|
if (arm != 0u) {
|
|
copyFlags.index = static_cast<std::uint32_t>(m_program.Resources().memoryInfo.size());
|
|
m_program.Resources().memoryInfo.push_back(memory);
|
|
}
|
|
std::uint64_t rawFlags = 0;
|
|
std::memcpy(&rawFlags, ©Flags, sizeof(copyFlags));
|
|
copies.push_back(EmitBefore(inst, inst.Opcode(), inst.Type(), arguments, rawFlags));
|
|
}
|
|
|
|
IrValue* result = copies.back();
|
|
if (copies.size() > 1u) {
|
|
std::array<IrValue*, 4> components {};
|
|
for (std::uint32_t component = 0; component < components.size(); component++) {
|
|
const auto extract = [&](IrValue* vector) {
|
|
const std::array<IrValue*, 2> arguments {vector, &m_builder.Constant(component)};
|
|
return EmitBefore(inst, IrOpcode::CompositeExtractU32x4, IrType::U32, arguments);
|
|
};
|
|
IrValue* selected = extract(copies.back());
|
|
for (std::uint32_t arm = static_cast<std::uint32_t>(copies.size()) - 1u; arm-- > 0u;) {
|
|
const std::array<IrValue*, 3> arguments {conditions[arm], extract(copies[arm]), selected};
|
|
selected = EmitBefore(inst, IrOpcode::SelectU32, IrType::U32, arguments);
|
|
}
|
|
components[component] = selected;
|
|
}
|
|
result = EmitBefore(inst, IrOpcode::CompositeConstructU32x4, IrType::U32x4, components);
|
|
}
|
|
inst.ReplaceAllUsesWith(result);
|
|
inst.Invalidate();
|
|
inst.Parent()->RemoveInstruction(&inst);
|
|
}
|
|
|
|
static bool HoldsProgramCounter(IrValue* value, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->Opcode() == IrOpcode::GetShaderBase) {
|
|
return true;
|
|
}
|
|
if (value->HasImmediate() || depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) {
|
|
return false;
|
|
}
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
if (HoldsProgramCounter(value->Argument(index), depth - 1u)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
struct DescriptorWeb {
|
|
static constexpr std::size_t maxStates = 512u;
|
|
static constexpr std::size_t maxArms = 64u;
|
|
std::vector<std::vector<IrValue*>> arms;
|
|
std::map<std::vector<IrValue*>, IrValue*> selectors;
|
|
std::vector<std::pair<IrBlock*, IrValue*>> phis;
|
|
std::vector<std::tuple<IrBlock*, IrValue*, IrValue*>> lanes;
|
|
std::unordered_map<IrValue*, IrValue*> resolved;
|
|
std::unordered_map<const IrBlock*, std::size_t> order;
|
|
};
|
|
|
|
IrValue* WebSelector(std::vector<IrValue*> tuple, DescriptorWeb& web, IrValue& position) {
|
|
for (IrValue*& value : tuple) {
|
|
value = value->Resolve();
|
|
if (!value->IsPhi()) {
|
|
continue;
|
|
}
|
|
const auto [entry, inserted] = web.resolved.try_emplace(value, value);
|
|
if (inserted) {
|
|
IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value);
|
|
entry->second = invariant == nullptr ? value : invariant->Resolve();
|
|
}
|
|
value = entry->second;
|
|
}
|
|
if (const auto known = web.selectors.find(tuple); known != web.selectors.end()) {
|
|
return known->second;
|
|
}
|
|
const auto phi = std::ranges::find_if(tuple, [](const IrValue* value) { return value->IsPhi(); });
|
|
const auto lane = std::ranges::find_if(tuple, [](const IrValue* value) { return value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::ReadFirstLane; });
|
|
if (phi == tuple.end() && lane != tuple.end()) {
|
|
return LaneSelector(tuple, **lane, web, position);
|
|
}
|
|
if (phi == tuple.end()) {
|
|
for (IrValue* value : tuple) {
|
|
if (HoldsProgramCounter(value, phiSearchDepth) || Rematerialize(value, position, false, phiSearchDepth) == nullptr) {
|
|
return nullptr;
|
|
}
|
|
}
|
|
const auto same = std::ranges::find_if(web.arms, [&](const std::vector<IrValue*>& arm) {
|
|
return std::ranges::equal(tuple, arm, [&](IrValue* left, IrValue* right) { return EquivalentValue(m_program.Resources(), left, right); });
|
|
});
|
|
if (same != web.arms.end()) {
|
|
return &m_builder.Constant(static_cast<std::uint32_t>(same - web.arms.begin()));
|
|
}
|
|
if (web.arms.size() == DescriptorWeb::maxArms) {
|
|
return nullptr;
|
|
}
|
|
web.arms.push_back(std::move(tuple));
|
|
return &m_builder.Constant(static_cast<std::uint32_t>(web.arms.size() - 1u));
|
|
}
|
|
if (web.order.empty()) {
|
|
const auto& order = m_program.BlockOrder();
|
|
for (std::size_t index = 0; index < order.size(); index++) {
|
|
web.order.emplace(order[index], index);
|
|
}
|
|
}
|
|
IrBlock* block = nullptr;
|
|
std::size_t latest = 0;
|
|
for (const IrValue* value : tuple) {
|
|
if (!value->IsPhi()) {
|
|
continue;
|
|
}
|
|
const auto found = web.order.find(value->Parent());
|
|
if (found == web.order.end()) {
|
|
return nullptr;
|
|
}
|
|
if (block == nullptr || found->second > latest) {
|
|
block = value->Parent();
|
|
latest = found->second;
|
|
}
|
|
}
|
|
if (block == nullptr || web.selectors.size() == DescriptorWeb::maxStates) {
|
|
return nullptr;
|
|
}
|
|
const auto& predecessors = block->Predecessors();
|
|
if (predecessors.size() < 2u) {
|
|
return nullptr;
|
|
}
|
|
for (const IrValue* value : tuple) {
|
|
if (value->IsPhi() && value->Parent() == block && value->PhiBlockCount() != predecessors.size()) {
|
|
return nullptr;
|
|
}
|
|
}
|
|
IrValue& selector = m_program.CreateValue(IrOpcode::Phi, IrType::U32);
|
|
web.selectors.emplace(tuple, &selector);
|
|
web.phis.emplace_back(block, &selector);
|
|
for (IrBlock* predecessor : predecessors) {
|
|
std::vector<IrValue*> incoming;
|
|
for (IrValue* value : tuple) {
|
|
IrValue* edge = value->IsPhi() && value->Parent() == block ? PhiIncoming(*value, predecessor) : value;
|
|
if (edge == nullptr) {
|
|
return nullptr;
|
|
}
|
|
incoming.push_back(edge);
|
|
}
|
|
IrValue* edgeSelector = WebSelector(std::move(incoming), web, position);
|
|
if (edgeSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
selector.AddPhiOperand(predecessor, edgeSelector);
|
|
}
|
|
return &selector;
|
|
}
|
|
|
|
IrValue* LaneSelector(const std::vector<IrValue*>& tuple, IrValue& first, DescriptorWeb& web, IrValue& position) {
|
|
IrBlock* block = first.Parent();
|
|
const auto op = first.Opcode();
|
|
const bool select = op == IrOpcode::SelectU32;
|
|
if (block == nullptr || first.ArgumentCount() != (select ? 3u : 2u) || web.selectors.size() == DescriptorWeb::maxStates) {
|
|
return nullptr;
|
|
}
|
|
IrValue* shared = first.Argument(select ? 0u : 1u);
|
|
std::vector<IrValue*> left;
|
|
std::vector<IrValue*> right;
|
|
for (IrValue* value : tuple) {
|
|
if (value->Opcode() == op && value->Parent() == block && value->ArgumentCount() == first.ArgumentCount() && EquivalentValue(m_program.Resources(), value->Argument(select ? 0u : 1u), shared)) {
|
|
left.push_back(value->Argument(select ? 1u : 0u));
|
|
if (select) {
|
|
right.push_back(value->Argument(2u));
|
|
}
|
|
} else if (value->HasImmediate() || (select && !value->IsPhi() && value->Opcode() != IrOpcode::ReadFirstLane)) {
|
|
left.push_back(value);
|
|
if (select) {
|
|
right.push_back(value);
|
|
}
|
|
} else {
|
|
return nullptr;
|
|
}
|
|
}
|
|
auto& instructions = block->Instructions();
|
|
auto latest = instructions.end();
|
|
for (auto it = instructions.begin(); it != instructions.end(); ++it) {
|
|
if (std::ranges::find(tuple, *it) != tuple.end()) {
|
|
latest = it;
|
|
}
|
|
}
|
|
if (latest == instructions.end()) {
|
|
return nullptr;
|
|
}
|
|
IrValue* before = std::next(latest) == instructions.end() ? nullptr : *std::next(latest);
|
|
IrValue* leftSelector = WebSelector(std::move(left), web, position);
|
|
if (leftSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
IrValue& node = m_program.CreateValue(op, IrType::U32, first.Flags<std::uint64_t>());
|
|
if (select) {
|
|
IrValue* rightSelector = WebSelector(std::move(right), web, position);
|
|
if (rightSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
node.AddArgument(shared);
|
|
node.AddArgument(leftSelector);
|
|
node.AddArgument(rightSelector);
|
|
} else {
|
|
node.AddArgument(leftSelector);
|
|
node.AddArgument(shared);
|
|
}
|
|
web.selectors.emplace(tuple, &node);
|
|
web.lanes.emplace_back(block, before, &node);
|
|
return &node;
|
|
}
|
|
|
|
void SplitDescriptorWeb(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount) {
|
|
std::vector<IrValue*> tuple;
|
|
std::array<std::size_t, 2> offsets {};
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
offsets[slot] = tuple.size();
|
|
for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) {
|
|
tuple.push_back(handles[slot]->Argument(dword));
|
|
}
|
|
}
|
|
DescriptorWeb web;
|
|
IrValue* selector = WebSelector(tuple, web, inst);
|
|
const bool select = selector != nullptr && web.arms.size() > 1u;
|
|
if (select) {
|
|
for (const auto& [block, phi] : web.phis) {
|
|
block->InsertInstructionBefore(nullptr, phi);
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
if (before == nullptr) {
|
|
block->AppendInstruction(node);
|
|
} else {
|
|
block->InsertInstructionBefore(before, node);
|
|
}
|
|
}
|
|
} else {
|
|
for (const auto& [block, phi] : web.phis) {
|
|
for (std::size_t index = 0; index < phi->ArgumentCount(); index++) {
|
|
phi->ReplaceArgument(index, nullptr);
|
|
}
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
for (std::size_t index = 0; index < node->ArgumentCount(); index++) {
|
|
node->ReplaceArgument(index, nullptr);
|
|
}
|
|
}
|
|
for (const auto& [block, phi] : web.phis) {
|
|
phi->Invalidate();
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
node->Invalidate();
|
|
}
|
|
}
|
|
if (selector == nullptr || web.arms.empty()) {
|
|
return;
|
|
}
|
|
|
|
std::vector<IrValue*> conditions;
|
|
for (std::uint32_t arm = 0; arm + 1u < web.arms.size(); arm++) {
|
|
const std::array<IrValue*, 2> compare {selector, &m_builder.Constant(arm)};
|
|
conditions.push_back(EmitBefore(inst, IrOpcode::IEqual32, IrType::Bool, compare));
|
|
}
|
|
ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) {
|
|
return Rematerialize(web.arms[arm][offsets[slot] + dword], inst, true, phiSearchDepth);
|
|
});
|
|
}
|
|
|
|
void SplitDescriptorPhis() {
|
|
std::vector<IrValue*> candidates;
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
if (SplittableImageRead(*inst)) {
|
|
candidates.push_back(inst);
|
|
}
|
|
}
|
|
}
|
|
for (IrValue* inst : candidates) {
|
|
SplitDescriptorPhi(*inst);
|
|
}
|
|
}
|
|
|
|
void PlanTableColumns() {
|
|
static const bool enabled = std::getenv("APS5_NO_IMAGE_TABLES") == nullptr;
|
|
if (!enabled) {
|
|
return;
|
|
}
|
|
const auto sources = m_sources;
|
|
const auto columns = m_tableColumns;
|
|
const auto reads = m_tablePlanningReads;
|
|
const auto memory = m_tablePlanningMemory;
|
|
PlanTableColumnPass();
|
|
while (!PlannedWithinLimits()) {
|
|
const auto group = LargestSelectedGroup();
|
|
if (group.empty()) {
|
|
return;
|
|
}
|
|
m_unselectedHandles.insert(m_unselectedHandles.end(), group.begin(), group.end());
|
|
m_sources = sources;
|
|
m_tableColumns = columns;
|
|
m_tablePlanningReads = reads;
|
|
m_tablePlanningMemory = memory;
|
|
PlanTableColumnPass();
|
|
}
|
|
}
|
|
|
|
bool PlannedWithinLimits() {
|
|
std::vector<ImageResource> images;
|
|
std::vector<std::pair<std::uint32_t, std::uint32_t>> pairs;
|
|
std::vector<const IrValue*> directImages;
|
|
std::vector<std::pair<const IrValue*, bool>> directSamplers;
|
|
const auto direct = std::numeric_limits<std::uint32_t>::max() / 2u;
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode());
|
|
if (inst->Opcode() == IrOpcode::ImageBvhIntersectRay || imageInfo.access == ImageAccess::None || imageInfo.resourceClass == ImageResourceClass::None || inst->ArgumentCount() == 0u) {
|
|
continue;
|
|
}
|
|
const auto flags = inst->Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
continue;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[flags.index];
|
|
if (memory.kind != ResourceKind::Image || memory.planningOnly || IsTablePlanningMemory(flags.index)) {
|
|
continue;
|
|
}
|
|
const IrValue* handle = inst->Argument(0)->Resolve();
|
|
const TableColumnPlan* column = FindTableColumn(*handle);
|
|
std::uint32_t source = 0;
|
|
if (column != nullptr) {
|
|
source = column->source;
|
|
} else {
|
|
const auto found = std::ranges::find_if(directImages, [&](const IrValue* other) { return SameHandle(*other, *handle); });
|
|
source = direct + static_cast<std::uint32_t>(found - directImages.begin());
|
|
if (found == directImages.end()) directImages.push_back(handle);
|
|
}
|
|
const auto key = ImageKey(source, memory, inst->Opcode());
|
|
const auto image = std::ranges::find_if(images, [&](const ImageResource& other) { return SameImageKey(other, key); });
|
|
const auto imageIndex = static_cast<std::uint32_t>(image - images.begin());
|
|
if (image == images.end()) images.push_back(key);
|
|
if (!imageInfo.needsSampler || inst->ArgumentCount() < 2u) {
|
|
continue;
|
|
}
|
|
const IrValue* samplerHandle = inst->Argument(1)->Resolve();
|
|
const TableColumnPlan* samplerColumn = FindTableColumn(*samplerHandle);
|
|
std::uint32_t sampler = 0;
|
|
if (samplerColumn != nullptr) {
|
|
sampler = samplerColumn->source;
|
|
} else {
|
|
const bool sampleAdjust = (memory.imageSampleFlags & RdnaImageSampleFlagAdjust) != 0;
|
|
const auto found = std::ranges::find_if(directSamplers, [&](const auto& other) { return other.second == sampleAdjust && SameHandle(*other.first, *samplerHandle); });
|
|
sampler = direct + static_cast<std::uint32_t>(found - directSamplers.begin());
|
|
if (found == directSamplers.end()) directSamplers.emplace_back(samplerHandle, sampleAdjust);
|
|
}
|
|
if (std::ranges::find(pairs, std::pair{imageIndex, sampler}) == pairs.end()) pairs.emplace_back(imageIndex, sampler);
|
|
}
|
|
}
|
|
return images.size() <= ShaderInfo::MaxImages && pairs.size() <= ShaderInfo::MaxSampledPairs;
|
|
}
|
|
|
|
std::vector<const IrValue*> LargestSelectedGroup() const {
|
|
const auto unselected = [](TableColumn column) {
|
|
column.selected = false;
|
|
column.selection = 0u;
|
|
return column;
|
|
};
|
|
std::vector<std::vector<std::uint32_t>> groups;
|
|
for (const auto& plan : m_tableColumns) {
|
|
const auto& column = m_sources[plan.source].tableColumn;
|
|
if (!column.has_value()) {
|
|
continue;
|
|
}
|
|
const auto group = std::ranges::find_if(groups, [&](const std::vector<std::uint32_t>& members) { return unselected(*m_sources[members.front()].tableColumn) == unselected(*column); });
|
|
if (group == groups.end()) {
|
|
groups.push_back({plan.source});
|
|
} else if (std::ranges::find(*group, plan.source) == group->end()) {
|
|
group->push_back(plan.source);
|
|
}
|
|
}
|
|
const std::vector<std::uint32_t>* largest = nullptr;
|
|
for (const auto& group : groups) {
|
|
const bool selected = std::ranges::any_of(group, [&](std::uint32_t source) { return m_sources[source].tableColumn->selected; });
|
|
if (selected && group.size() > 1u && (largest == nullptr || group.size() > largest->size())) {
|
|
largest = &group;
|
|
}
|
|
}
|
|
std::vector<const IrValue*> handles;
|
|
if (largest == nullptr) {
|
|
return handles;
|
|
}
|
|
for (const auto& plan : m_tableColumns) {
|
|
if (std::ranges::find(*largest, plan.source) != largest->end()) {
|
|
handles.push_back(plan.handle);
|
|
}
|
|
}
|
|
return handles;
|
|
}
|
|
|
|
void PlanTableColumnPass() {
|
|
IndexMemoryUsers();
|
|
const auto directImages = DirectImageHandles();
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode());
|
|
if (imageInfo.access == ImageAccess::None || inst->ArgumentCount() == 0u) {
|
|
continue;
|
|
}
|
|
const auto flags = inst->Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
continue;
|
|
}
|
|
const auto memory = m_program.Resources().memoryInfo[flags.index];
|
|
IrValue& image = *inst->Argument(0)->Resolve();
|
|
if (std::ranges::none_of(directImages, [&](const IrValue* handle) { return SameHandle(*handle, image); })) {
|
|
static_cast<void>(PlanTableColumn(image, false, memory.imageR128));
|
|
}
|
|
if (imageInfo.needsSampler && inst->ArgumentCount() > 1u && !DirectSampler(*inst->Argument(1)->Resolve())) {
|
|
static_cast<void>(PlanTableColumn(*inst->Argument(1)->Resolve(), true, false));
|
|
}
|
|
}
|
|
}
|
|
for (const auto& plan : m_tableColumns) {
|
|
for (std::size_t index = 0; index < plan.reads.size(); index++) {
|
|
const IrValue* read = plan.reads[index];
|
|
const bool planned = std::ranges::all_of(read->Uses(), [&](const IrValue* user) {
|
|
return FindTableColumn(*user) != nullptr;
|
|
});
|
|
if (planned && std::ranges::find(m_tablePlanningReads, read) == m_tablePlanningReads.end()) {
|
|
m_tablePlanningReads.push_back(read);
|
|
m_tablePlanningMemory.push_back(plan.memory[index]);
|
|
}
|
|
}
|
|
}
|
|
m_memoryUsersIndexed = false;
|
|
m_memoryUsers.clear();
|
|
}
|
|
|
|
void KeepSourceAlive(IrValue& position, const DescriptorSource& source, std::vector<const IrValue*>& referenced) {
|
|
for (std::uint32_t dword = 0; dword < source.dwordCount; dword++) {
|
|
IrValue* value = source.dwords[dword]->Resolve();
|
|
if (value->HasImmediate() || std::ranges::find(referenced, value) != referenced.end()) {
|
|
continue;
|
|
}
|
|
referenced.push_back(value);
|
|
const std::array<IrValue*, 1> arguments{value};
|
|
EmitBefore(position, IrOpcode::ReferenceU32, IrType::Void, arguments);
|
|
}
|
|
}
|
|
|
|
bool DirectSampler(const IrValue& handle) {
|
|
if (handle.Opcode() != IrOpcode::GetSamplerResource || handle.ArgumentCount() != 4u || FindTableColumn(handle) != nullptr) {
|
|
return false;
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(handle, 4u, true, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
return ValidateSource(descriptor, badDword);
|
|
}
|
|
|
|
bool DirectImage(const IrValue& handle) {
|
|
if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u) {
|
|
return false;
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(handle, 8u, false, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
return ValidateSource(descriptor, badDword);
|
|
}
|
|
|
|
std::vector<const IrValue*> DirectImageHandles() {
|
|
std::vector<const IrValue*> handles;
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode());
|
|
if (imageInfo.access == ImageAccess::None || inst->ArgumentCount() == 0u) {
|
|
continue;
|
|
}
|
|
const auto flags = inst->Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
continue;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[flags.index];
|
|
const bool tableUnsupported = imageInfo.resourceClass == ImageResourceClass::Storage || memory.imagePacked || memory.imageByElements != 0u;
|
|
const IrValue* handle = inst->Argument(0)->Resolve();
|
|
if (tableUnsupported && DirectImage(*handle)) {
|
|
handles.push_back(handle);
|
|
}
|
|
}
|
|
}
|
|
return handles;
|
|
}
|
|
|
|
bool SameHandle(const IrValue& left, const IrValue& right) const {
|
|
if (&left == &right) {
|
|
return true;
|
|
}
|
|
if (left.Opcode() != right.Opcode() || left.ArgumentCount() != right.ArgumentCount()) {
|
|
return false;
|
|
}
|
|
for (std::size_t index = 0; index < left.ArgumentCount(); index++) {
|
|
if (!EquivalentValue(m_program.Resources(), left.Argument(index), right.Argument(index))) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool IsTablePlanningMemory(std::uint32_t index) const {
|
|
return std::ranges::find(m_tablePlanningMemory, index) != m_tablePlanningMemory.end();
|
|
}
|
|
|
|
bool TableSource(std::uint32_t source) const {
|
|
return source < m_sources.size() && m_sources[source].tableColumn.has_value();
|
|
}
|
|
|
|
bool SameBufferSource(const IrValue& handle, std::uint32_t source) const {
|
|
const auto& current = m_sources[source];
|
|
if (current.tableColumn.has_value() || current.dwordCount != 4u || handle.ArgumentCount() != 4u) {
|
|
return false;
|
|
}
|
|
for (std::uint32_t dword = 0; dword < 4u; dword++) {
|
|
if (!EquivalentValue(m_program.Resources(), current.dwords[dword], handle.Argument(dword))) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool SelectionWritten(const TableColumn& column) const {
|
|
const auto written = [&](const auto& matches) {
|
|
return std::any_of(m_info.buffers.begin(), m_info.buffers.end(), [&](const auto& buffer) { return (buffer.written || buffer.atomic) && matches(buffer.source); });
|
|
};
|
|
std::vector<const IrValue*> pending{m_sources[column.selection].dwords[0]};
|
|
std::vector<const IrValue*> seen;
|
|
while (!pending.empty()) {
|
|
const IrValue* current = pending.back()->Resolve();
|
|
pending.pop_back();
|
|
if (std::ranges::find(seen, current) != seen.end()) {
|
|
continue;
|
|
}
|
|
seen.push_back(current);
|
|
const bool pointer = current->Opcode() == IrOpcode::LoadAddressU32;
|
|
const bool buffer = current->Opcode() == IrOpcode::ReadConstBuffer && current->ArgumentCount() != 0u;
|
|
if ((pointer || buffer) && m_info.bdaWrites) {
|
|
return true;
|
|
}
|
|
if (pointer && written([](std::uint32_t) { return true; })) {
|
|
return true;
|
|
}
|
|
if (buffer) {
|
|
const IrValue* handle = current->Argument(0)->Resolve();
|
|
if (written([&](std::uint32_t source) { return SameBufferSource(*handle, source); })) {
|
|
return true;
|
|
}
|
|
}
|
|
for (std::size_t index = 0; index < current->ArgumentCount(); index++) {
|
|
pending.push_back(current->Argument(index));
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void CheckTableStores() {
|
|
for (auto& source : m_sources) {
|
|
if (source.tableColumn.has_value() && source.tableColumn->selected && SelectionWritten(*source.tableColumn)) {
|
|
source.tableColumn->selected = false;
|
|
source.tableColumn->keyDomains.clear();
|
|
}
|
|
if (!source.tableColumn.has_value() || source.tableColumn->keyDomains.empty()) {
|
|
continue;
|
|
}
|
|
auto& column = *source.tableColumn;
|
|
const bool written = std::any_of(column.keyDomains.begin(), column.keyDomains.end(), [&](const KeyDomain& domain) {
|
|
return std::any_of(m_info.buffers.begin(), m_info.buffers.end(), [&](const auto& buffer) { return (buffer.written || buffer.atomic) && buffer.source == domain.source; });
|
|
});
|
|
if (written) {
|
|
column.keyDomains.clear();
|
|
}
|
|
}
|
|
}
|
|
|
|
void AssignTables() {
|
|
std::uint32_t next = 0;
|
|
for (auto& image : m_info.images) {
|
|
if (!TableSource(image.source)) {
|
|
continue;
|
|
}
|
|
image.table = next++;
|
|
if (image.atomic || image.packed || image.byElements != 0u) {
|
|
image.tableOperation = TableOperation::Unsupported;
|
|
}
|
|
}
|
|
for (auto& sampler : m_info.samplers) {
|
|
if (TableSource(sampler.source)) {
|
|
sampler.table = next++;
|
|
}
|
|
}
|
|
}
|
|
|
|
void GetHandle(IrValue* value, IrOpcode expected, std::uint32_t width, IrValue*& handle, std::uint32_t& source, bool sampler = false, bool sampleAdjust = false) {
|
|
handle = value->Resolve();
|
|
if (handle->Opcode() != expected) {
|
|
fail("memory operation requires " + std::string(IrOpcodeName(expected)));
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(*handle, width, sampler, sampleAdjust, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
if (!ValidateSource(descriptor, badDword)) {
|
|
fail(std::string(IrOpcodeName(expected)) + " dword " + std::to_string(badDword) + " is not a valid runtime value; chain: " + describeValueChain(descriptor.dwords[badDword], 8u));
|
|
}
|
|
source = InternSource(descriptor);
|
|
}
|
|
|
|
bool TakeGpuDescriptor(IrValue& inst, std::uint32_t memoryIndex) {
|
|
const IrValue* handle = inst.Argument(0)->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) {
|
|
return false;
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(*handle, 4u, false, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
auto& memory = m_program.Resources().memoryInfo[memoryIndex];
|
|
const auto access = BufferAccessOf(inst.Opcode());
|
|
if (ValidateSource(descriptor, badDword)) {
|
|
const auto source = InternSource(descriptor);
|
|
const auto resource = AddBuffer(source, memory, inst.Opcode(), inst.Flags<MemoryFlags>().pc);
|
|
if (resource == std::numeric_limits<std::uint32_t>::max()) fail("buffer resource limit exceeded");
|
|
AddMemoryPatch(memoryIndex, resource, 0u, false);
|
|
memory.gpuDescriptor = false;
|
|
m_info.usesFaultBuffer = m_info.usesFaultBuffer || access == BufferAccess::Atomic;
|
|
return true;
|
|
}
|
|
memory.gpuDescriptor = true;
|
|
m_info.usesDma = true;
|
|
m_info.bdaWrites = m_info.bdaWrites || access == BufferAccess::Write || access == BufferAccess::Atomic;
|
|
return true;
|
|
}
|
|
|
|
void ValidateAddressHandle(IrValue* value) const {
|
|
const IrValue* handle = value->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetAddressResource) {
|
|
fail("address operation requires GetAddressResource");
|
|
}
|
|
if (handle->ArgumentCount() != 2) {
|
|
fail("GetAddressResource must have two address dwords");
|
|
}
|
|
}
|
|
|
|
std::uint32_t AddBuffer(std::uint32_t source, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
for (std::uint32_t i = 0; i < m_info.buffers.size(); i++) {
|
|
if (m_info.buffers[i].source == source) {
|
|
Merge(m_info.buffers[i], memory, op, pc);
|
|
return i;
|
|
}
|
|
}
|
|
BufferResource resource;
|
|
resource.source = source;
|
|
resource.firstUsePc = pc;
|
|
Merge(resource, memory, op, pc);
|
|
m_info.buffers.push_back(resource);
|
|
return static_cast<std::uint32_t>(m_info.buffers.size() - 1);
|
|
}
|
|
|
|
static void Merge(BufferResource& resource, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto access = BufferAccessOf(op);
|
|
const bool atomic = access == BufferAccess::Atomic;
|
|
const bool write = access == BufferAccess::Write || atomic;
|
|
resource.firstUsePc = std::min(resource.firstUsePc, pc);
|
|
resource.maxByteExtent = std::max(resource.maxByteExtent, byteExtent(memory));
|
|
resource.read = resource.read || !write || atomic;
|
|
resource.written = resource.written || write;
|
|
resource.atomic = resource.atomic || atomic;
|
|
resource.formatted = resource.formatted || memory.formatted;
|
|
resource.descriptorFormatted = resource.descriptorFormatted || (memory.formatted && !memory.typed);
|
|
if (memory.formatted && memory.typed) {
|
|
const auto format = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat));
|
|
if (format.byteSize == 0u) fail("typed buffer instruction has an invalid format");
|
|
resource.typedAlignment = std::max(resource.typedAlignment, static_cast<std::uint8_t>(std::min(format.byteSize, 4u)));
|
|
}
|
|
if (memory.formatted && !memory.typed && !write) resource.formattedReadMask |= (1u << std::min(memory.dataDwords, 4u)) - 1u;
|
|
resource.scalar = resource.scalar || op == IrOpcode::ReadConstBuffer || memory.kind == ResourceKind::ScalarBuffer;
|
|
}
|
|
|
|
static ImageResource ImageKey(std::uint32_t source, const MemoryInfo& memory, IrOpcode op) {
|
|
ImageResource image;
|
|
image.source = source;
|
|
image.resourceClass = ImageOpcodeInfoOf(op).resourceClass;
|
|
image.dimension = memory.imageDimension;
|
|
image.mipMode = image.resourceClass == ImageResourceClass::Storage && memory.imageHasMip ? ImageMipMode::DynamicStorage : ImageMipMode::None;
|
|
image.depthCompare = (memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0;
|
|
image.r128 = memory.imageR128;
|
|
image.packed = memory.imagePacked;
|
|
image.byElements = memory.imageByElements;
|
|
image.byComponents = memory.imageByElements != 0u ? memory.dataDwords / memory.imageByElements : 0u;
|
|
return image;
|
|
}
|
|
|
|
static bool SameImageKey(const ImageResource& image, const ImageResource& key) {
|
|
return image.source == key.source && image.resourceClass == key.resourceClass && image.dimension == key.dimension && image.mipMode == key.mipMode && image.depthCompare == key.depthCompare && image.r128 == key.r128 && image.packed == key.packed && image.byElements == key.byElements && image.byComponents == key.byComponents;
|
|
}
|
|
|
|
std::uint32_t AddImage(std::uint32_t source, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto key = ImageKey(source, memory, op);
|
|
for (std::uint32_t i = 0; i < m_info.images.size(); i++) {
|
|
auto& image = m_info.images[i];
|
|
if (SameImageKey(image, key)) {
|
|
Merge(image, memory, op, pc);
|
|
return i;
|
|
}
|
|
}
|
|
if (m_info.images.size() >= ShaderInfo::MaxImages) {
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
ImageResource image = key;
|
|
image.singleMipPlaneBiasCompatible = m_program.Resources().stage == IrShaderStage::Pixel;
|
|
image.firstUsePc = pc;
|
|
Merge(image, memory, op, pc);
|
|
m_info.images.push_back(image);
|
|
return static_cast<std::uint32_t>(m_info.images.size() - 1);
|
|
}
|
|
|
|
static void Merge(ImageResource& image, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto access = ImageOpcodeInfoOf(op).access;
|
|
const bool atomic = access == ImageAccess::Atomic;
|
|
const bool write = access == ImageAccess::Write || atomic;
|
|
const bool atomic64 = IsImageAtomic64Opcode(op);
|
|
if ((image.read || image.written) && image.atomic64 != atomic64) {
|
|
throw std::runtime_error("an image accessed by 64-bit atomics is also accessed in another way");
|
|
}
|
|
image.atomic64 = atomic64;
|
|
image.firstUsePc = std::min(image.firstUsePc, pc);
|
|
image.read = image.read || !write || atomic;
|
|
image.written = image.written || write;
|
|
image.atomic = image.atomic || atomic;
|
|
image.srgbDecodeCompatible = image.srgbDecodeCompatible && !ImageOpcodeInfoOf(op).needsSampler;
|
|
image.fmaskCompatible = image.fmaskCompatible && op == IrOpcode::ImageRead && memory.dataBits == 32u;
|
|
image.singleMipPlaneBiasCompatible = image.singleMipPlaneBiasCompatible && op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagBias && memory.dataBits == 32u && !memory.imageR128 && !memory.imagePacked && memory.imageByElements == 0u;
|
|
image.depthBitsCompatible = image.depthBitsCompatible && memory.dataBits == 32u;
|
|
image.constantSwizzleCompatible = image.constantSwizzleCompatible && (op == IrOpcode::ImageSampleRaw || op == IrOpcode::ImageGatherRaw);
|
|
const bool cubeFaceLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u;
|
|
const bool cubeFaceSample = op == IrOpcode::ImageSampleRaw && (memory.imageSampleFlags == 0u || memory.imageSampleFlags == RdnaImageSampleFlagLevelZero || memory.imageSampleFlags == RdnaImageSampleFlagLod);
|
|
image.cubeFaceCompatible = image.cubeFaceCompatible && (cubeFaceLoad || cubeFaceSample) && !memory.imagePacked && memory.imageByElements == 0u;
|
|
const bool flatVolumeLoad = op == IrOpcode::ImageRead && memory.imageSampleFlags == 0u;
|
|
const bool flatVolumeSample = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLevelZero;
|
|
const bool flatVolumeQuery = op == IrOpcode::ImageQueryDimensions;
|
|
image.flatVolumeCompatible = image.flatVolumeCompatible && (flatVolumeLoad || flatVolumeSample || flatVolumeQuery) && !memory.imagePacked && memory.imageByElements == 0u;
|
|
const bool flatLineLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u;
|
|
const auto lineSampleFlags = memory.imageSampleFlags & ~RdnaImageSampleFlagCompare;
|
|
const bool flatLineSample = op == IrOpcode::ImageSampleRaw && (lineSampleFlags == RdnaImageSampleFlagLevelZero || lineSampleFlags == 0u);
|
|
const bool flatLineExplicitLod = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLod;
|
|
const bool flatLineLevelZero = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLevelZero;
|
|
const bool flatLineAccess = image.dimension == RdnaImageDimension::Dim2DArray ? flatLineLevelZero || flatLineExplicitLod : flatLineLoad || flatLineSample || flatLineExplicitLod;
|
|
image.flatLineCompatible = image.flatLineCompatible && flatLineAccess && !memory.imagePacked && memory.imageByElements == 0u;
|
|
image.flatVolumeLineCompatible = image.flatVolumeLineCompatible && (flatLineLoad || flatLineSample) && !memory.imagePacked && memory.imageByElements == 0u;
|
|
if ((memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0u) {
|
|
constexpr auto unsupported = RdnaImageSampleFlagLod | RdnaImageSampleFlagDerivative;
|
|
if (op == IrOpcode::ImageGatherRaw || (memory.imageSampleFlags & unsupported) != 0u) image.emulatedCompare |= EmulatedCompare::Unsupported;
|
|
if ((memory.imageSampleFlags & RdnaImageSampleFlagLevelZero) == 0u) image.emulatedCompare |= EmulatedCompare::RequiresSingleLevel;
|
|
}
|
|
}
|
|
|
|
std::uint32_t AddSampler(std::uint32_t source, std::uint32_t pc) {
|
|
for (std::uint32_t i = 0; i < m_info.samplers.size(); i++) {
|
|
if (m_info.samplers[i].source == source) {
|
|
m_info.samplers[i].firstUsePc = std::min(m_info.samplers[i].firstUsePc, pc);
|
|
return i;
|
|
}
|
|
}
|
|
if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) {
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
m_info.samplers.push_back({source, pc});
|
|
return static_cast<std::uint32_t>(m_info.samplers.size() - 1);
|
|
}
|
|
|
|
void AddSampledPair(std::uint32_t image, std::uint32_t sampler, std::uint32_t pc) {
|
|
for (auto& pair : m_info.sampledPairs) {
|
|
if (pair.image == image && pair.sampler == sampler) {
|
|
pair.firstUsePc = std::min(pair.firstUsePc, pc);
|
|
return;
|
|
}
|
|
}
|
|
if (m_info.sampledPairs.size() >= ShaderInfo::MaxSampledPairs) {
|
|
fail("sampled image/sampler pair limit exceeded");
|
|
}
|
|
m_info.sampledPairs.push_back({image, sampler, pc});
|
|
}
|
|
|
|
void AddHandlePatch(IrValue* handle, std::uint32_t resource) {
|
|
for (const auto& patch : m_handlePatches) {
|
|
if (patch.handle == handle) {
|
|
if (patch.resource != resource) {
|
|
fail(std::string(IrOpcodeName(handle->Opcode())) + " is reused with incompatible resource classes");
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
m_handlePatches.push_back({handle, resource});
|
|
}
|
|
|
|
void AddMemoryPatch(std::uint32_t index, std::uint32_t resource, std::uint32_t sampler, bool hasSampler) {
|
|
for (auto& patch : m_memoryPatches) {
|
|
if (patch.index != index) {
|
|
continue;
|
|
}
|
|
if (patch.resource != resource || (hasSampler && patch.hasSampler && patch.sampler != sampler)) {
|
|
fail("memory metadata is reused with incompatible resources");
|
|
}
|
|
if (hasSampler) {
|
|
patch.sampler = sampler;
|
|
patch.hasSampler = true;
|
|
}
|
|
return;
|
|
}
|
|
m_memoryPatches.push_back({index, resource, sampler, hasSampler});
|
|
}
|
|
|
|
void Collect(IrValue& inst) {
|
|
const auto op = inst.Opcode();
|
|
if (op == IrOpcode::ImageBvhIntersectRay) {
|
|
m_info.usesDma = true;
|
|
return;
|
|
}
|
|
const auto buffer = BufferAccessOf(op);
|
|
const auto addressInfo = AddressOpcodeInfoOf(op);
|
|
const auto imageInfo = ImageOpcodeInfoOf(op);
|
|
if (buffer == BufferAccess::None && addressInfo.access == AddressAccess::None && imageInfo.access == ImageAccess::None) {
|
|
return;
|
|
}
|
|
const auto flags = inst.Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
fail("memory metadata index " + std::to_string(flags.index) + " is out of range");
|
|
}
|
|
if (inst.ArgumentCount() == 0) {
|
|
fail("memory operation has no resource handle");
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[flags.index];
|
|
if (memory.planningOnly || IsTablePlanningMemory(flags.index)) {
|
|
return;
|
|
}
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t source = 0;
|
|
std::uint32_t resource = 0;
|
|
|
|
if (buffer != BufferAccess::None) {
|
|
if (!TakeGpuDescriptor(inst, flags.index)) fail("buffer operation requires a four-dword runtime V#");
|
|
return;
|
|
}
|
|
if (addressInfo.access != AddressAccess::None) {
|
|
if (!IsAddressResourceKind(memory.kind)) {
|
|
fail("address operation has invalid resource kind");
|
|
}
|
|
if (memory.kind == ResourceKind::Scratch) {
|
|
handle = inst.Argument(0)->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetScratchResource || handle->ArgumentCount() != 0) {
|
|
fail("scratch operation requires GetScratchResource");
|
|
}
|
|
if (m_info.scratchDwords == 0) {
|
|
fail("scratch operation requires a nonzero AGC per-thread size");
|
|
}
|
|
return;
|
|
}
|
|
ValidateAddressHandle(inst.Argument(0));
|
|
// Debug aid: APS5_TRACE_BDA=1 names every access that makes the program address-based
|
|
// (a raw scalar load the SRT walker left in place, or a flat/global access), so the
|
|
// reason a stage takes the BDA path can be read off without a shader dump.
|
|
// The first few accesses of a program are printed (a Bink kernel has hundreds); Run
|
|
// reports how many more there were.
|
|
if (bdaTraceEnabled() && ++m_bdaTraces <= bdaTraceLimit) {
|
|
const auto* kind = memory.kind == ResourceKind::ScalarAddress ? "scalar address" : memory.kind == ResourceKind::Global ? "global" : "flat";
|
|
const IrValue* offset = inst.ArgumentCount() > 1 ? inst.Argument(1)->Resolve() : nullptr;
|
|
const bool immediateOffset = offset != nullptr && offset->HasImmediate();
|
|
std::fprintf(stderr, "[bda] %s at pc 0x%08x: %s access, offset %s%s\n", std::string(IrOpcodeName(op)).c_str(), flags.pc, kind, immediateOffset ? "immediate" : "dynamic", memory.kind == ResourceKind::ScalarAddress && !immediateOffset ? " (a register offset is not planned by the SRT walker)" : "");
|
|
}
|
|
m_info.usesDma = true;
|
|
m_info.bdaWrites = m_info.bdaWrites || addressInfo.access == AddressAccess::Write || addressInfo.access == AddressAccess::Atomic;
|
|
return;
|
|
}
|
|
|
|
if (memory.kind != ResourceKind::Image || imageInfo.resourceClass == ImageResourceClass::None) {
|
|
fail("image operation has invalid resource kind");
|
|
}
|
|
handle = inst.Argument(0)->Resolve();
|
|
const TableColumnPlan* column = FindTableColumn(*handle);
|
|
if (column != nullptr) {
|
|
source = column->source;
|
|
} else {
|
|
GetHandle(inst.Argument(0), IrOpcode::GetImageResource, 8, handle, source);
|
|
}
|
|
resource = AddImage(source, memory, op, flags.pc);
|
|
NoteBlockImage(resource);
|
|
if (resource == std::numeric_limits<std::uint32_t>::max()) {
|
|
fail("image resource limit exceeded");
|
|
}
|
|
AddHandlePatch(handle, resource);
|
|
std::uint32_t sampler = 0;
|
|
if (imageInfo.needsSampler) {
|
|
if (inst.ArgumentCount() < 2) {
|
|
fail("sampled image operation has no sampler handle");
|
|
}
|
|
IrValue* samplerHandle = inst.Argument(1)->Resolve();
|
|
std::uint32_t samplerSource = 0;
|
|
const TableColumnPlan* samplerColumn = FindTableColumn(*samplerHandle);
|
|
if (samplerColumn != nullptr) {
|
|
samplerSource = samplerColumn->source;
|
|
} else {
|
|
const bool sampleAdjust = (memory.imageSampleFlags & RdnaImageSampleFlagAdjust) != 0;
|
|
GetHandle(inst.Argument(1), IrOpcode::GetSamplerResource, 4, samplerHandle, samplerSource, true, sampleAdjust);
|
|
}
|
|
sampler = AddSampler(samplerSource, flags.pc);
|
|
if (sampler == std::numeric_limits<std::uint32_t>::max()) {
|
|
fail("sampler resource limit exceeded");
|
|
}
|
|
AddHandlePatch(samplerHandle, sampler);
|
|
AddSampledPair(resource, sampler, flags.pc);
|
|
}
|
|
AddMemoryPatch(flags.index, resource, sampler, imageInfo.needsSampler);
|
|
}
|
|
|
|
const DescriptorSource* Source(std::uint32_t source) const {
|
|
return source < m_sources.size() ? &m_sources[source] : nullptr;
|
|
}
|
|
|
|
void LinkImageAliases() {
|
|
for (auto& buffer : m_info.buffers) {
|
|
const DescriptorSource* bufferSource = Source(buffer.source);
|
|
if (bufferSource == nullptr || bufferSource->dwordCount != 4) {
|
|
continue;
|
|
}
|
|
for (std::uint32_t image = 0; image < m_info.images.size(); image++) {
|
|
const DescriptorSource* imageSource = Source(m_info.images[image].source);
|
|
if (imageSource == nullptr || imageSource->dwordCount != 8 || imageSource->tableColumn.has_value()) {
|
|
continue;
|
|
}
|
|
bool alias = true;
|
|
for (std::uint32_t dword = 0; dword < 4; dword++) {
|
|
alias = alias && EquivalentValue(m_program.Resources(), bufferSource->dwords[dword], imageSource->dwords[dword]);
|
|
}
|
|
if (alias) {
|
|
buffer.imageAlias = image;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
IrProgram& m_program;
|
|
ShaderInfo m_info;
|
|
IrBuilder m_builder;
|
|
std::vector<DescriptorSource> m_sources;
|
|
std::vector<std::vector<std::uint32_t>> m_blockSources;
|
|
std::vector<std::vector<std::uint32_t>> m_blockImages;
|
|
std::uint32_t m_currentBlock = std::numeric_limits<std::uint32_t>::max();
|
|
std::vector<HandlePatch> m_handlePatches;
|
|
std::vector<MemoryPatch> m_memoryPatches;
|
|
// APS5_TRACE_BDA: address accesses seen by Collect (the first bdaTraceLimit are printed).
|
|
unsigned m_bdaTraces = 0;
|
|
std::unordered_map<std::uint32_t, std::vector<const IrValue*>> m_memoryUsers;
|
|
bool m_memoryUsersIndexed = false;
|
|
std::vector<TableColumnPlan> m_tableColumns;
|
|
std::vector<const IrValue*> m_tablePlanningReads;
|
|
std::vector<std::uint32_t> m_tablePlanningMemory;
|
|
std::vector<const IrValue*> m_unselectedHandles;
|
|
struct EdgeSelectorEntry {
|
|
IrBlock* block = nullptr;
|
|
std::vector<bool> edges;
|
|
IrValue* value = nullptr;
|
|
};
|
|
std::vector<EdgeSelectorEntry> m_edgeSelectors;
|
|
};
|
|
|
|
}
|
|
|
|
void ResourceTracker::Track(IrProgram& program) const {
|
|
Tracker(program).Run();
|
|
}
|
|
|
|
}
|