mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-22 08:33:12 -05:00
Dean Galvin
dbe16fb28c
A program with an atomic on a V# it specializes was compiled as address-based (usesDma) so that the atomic's unaligned-address check could call the fault function. The atomic itself goes through the element's storage buffer and nothing in such a program reads the address tables. In the driver an address-based build leases all registered guest memory and binds every V# of the program in place inside a host import, so none of its elements is staged in device memory, the atomic one included. The atomic now asks for the fault buffer alone (usesFaultBuffer) and records the fault through RecordBdaFaultWords, which has both forms. The fault record, the stop of the invocation and the atomic are unchanged. The 64-bit fault address needs shaderInt64, as the scalar buffer load does. A tessellation-control program with such an atomic is refused as before. Test: agc_shader_memory compiles a buffer_atomic_inc on a specialized V# and requires no address table binding, one fault buffer, one call of the fault function, logical addressing and the atomic element flag; without shaderInt64 the program is refused.
1514 lines
67 KiB
C++
1514 lines
67 KiB
C++
#include "Optimization/ResourceTracker.hpp"
|
|
#include "Optimization/SrtWalker.hpp"
|
|
#include "Optimization/ResourceMaterializer.hpp"
|
|
#include "SpirvBackend/SpirvBufferFormat.hpp"
|
|
#include "IntermediateRepresentation/IrBuilder.hpp"
|
|
#include "Optimization/SrtWalker/SrtInstructionPredicates.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <limits>
|
|
#include <map>
|
|
#include <span>
|
|
#include <string>
|
|
#include <tuple>
|
|
#include <unordered_map>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
namespace ShaderRecompiler {
|
|
namespace {
|
|
|
|
constexpr std::uint32_t samplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
|
|
constexpr std::uint32_t samplerDword3ReservedMask = 0x3ffff000u;
|
|
|
|
[[noreturn]] void fail(const std::string& message) {
|
|
throw std::runtime_error(message);
|
|
}
|
|
|
|
// Debug aid: APS5_TRACE_BDA=1 names the accesses that make a program address-based (see Collect).
|
|
bool bdaTraceEnabled() {
|
|
static const bool enabled = std::getenv("APS5_TRACE_BDA") != nullptr;
|
|
return enabled;
|
|
}
|
|
constexpr unsigned bdaTraceLimit = 8;
|
|
|
|
std::string formatHex32(std::uint32_t value) {
|
|
static constexpr char digits[] = "0123456789abcdef";
|
|
std::string hex(8u, '0');
|
|
for (std::uint32_t index = 0; index < 8u; index++) {
|
|
hex[7u - index] = digits[(value >> (index * 4u)) & 0xfu];
|
|
}
|
|
return hex;
|
|
}
|
|
|
|
std::string describeValueChain(const IrValue* value, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return "Immediate";
|
|
}
|
|
std::string text = std::string(IrOpcodeName(value->Opcode()));
|
|
if (value->Opcode() == IrOpcode::LoadAddressU32 || value->Opcode() == IrOpcode::ReadConstBuffer) {
|
|
text += "@pc=0x" + formatHex32(value->Flags<MemoryFlags>().pc);
|
|
}
|
|
if (depth == 0u || value->ArgumentCount() == 0u) {
|
|
return text;
|
|
}
|
|
text += "(";
|
|
for (std::uint32_t index = 0; index < value->ArgumentCount(); index++) {
|
|
if (index != 0u) {
|
|
text += ", ";
|
|
}
|
|
text += describeValueChain(value->Argument(index), depth - 1u);
|
|
}
|
|
text += ")";
|
|
return text;
|
|
}
|
|
|
|
std::uint32_t possibleU32Bits(const IrValue* value) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return value->Type() == IrType::U32 ? value->ImmediateU32() : std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
switch (value->Opcode()) {
|
|
case IrOpcode::BitwiseAnd32:
|
|
return possibleU32Bits(value->Argument(0)) & possibleU32Bits(value->Argument(1));
|
|
case IrOpcode::BitwiseOr32:
|
|
return possibleU32Bits(value->Argument(0)) | possibleU32Bits(value->Argument(1));
|
|
case IrOpcode::ShiftLeftLogical32: {
|
|
const IrValue* shift = value->Argument(1)->Resolve();
|
|
return shift->HasImmediate() && shift->Type() == IrType::U32 ? possibleU32Bits(value->Argument(0)) << (shift->ImmediateU32() & 31u) : std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
default:
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
}
|
|
|
|
std::uint32_t byteExtent(const MemoryInfo& memory) {
|
|
const auto bytes = std::max((memory.dataBits + 7u) / 8u, 1u);
|
|
const auto count = std::max(memory.dataDwords, 1u);
|
|
const auto end = static_cast<std::uint64_t>(memory.offset) + static_cast<std::uint64_t>(bytes) * count;
|
|
return end > std::numeric_limits<std::uint32_t>::max() ? std::numeric_limits<std::uint32_t>::max() : static_cast<std::uint32_t>(end);
|
|
}
|
|
|
|
class Tracker {
|
|
public:
|
|
explicit Tracker(IrProgram& program) : m_program(program), m_info(program.Resources().info), m_builder(program) {
|
|
m_info.buffers.clear();
|
|
m_info.images.clear();
|
|
m_info.samplers.clear();
|
|
m_info.sampledPairs.clear();
|
|
m_info.usesDma = false;
|
|
m_info.bdaWrites = false;
|
|
}
|
|
|
|
void Run() {
|
|
if (m_program.Resources().resourceTrackingComplete) {
|
|
fail("resources already tracked");
|
|
}
|
|
if (!m_program.Resources().srtPlanComplete) {
|
|
fail("SRT plan is not ready");
|
|
}
|
|
SplitDescriptorPhis();
|
|
PlanIndirectImages();
|
|
std::unordered_map<const IrBlock*, std::uint32_t> blockIndices;
|
|
for (const IrBlock* block : m_program.BlockOrder()) {
|
|
blockIndices.emplace(block, static_cast<std::uint32_t>(blockIndices.size()));
|
|
}
|
|
m_blockSources.assign(blockIndices.size(), {});
|
|
m_blockImages.assign(blockIndices.size(), {});
|
|
for (auto& block : m_program.Blocks()) {
|
|
const auto found = blockIndices.find(block.get());
|
|
m_currentBlock = found == blockIndices.end() ? std::numeric_limits<std::uint32_t>::max() : found->second;
|
|
for (IrValue* inst : block->Instructions()) {
|
|
Collect(*inst);
|
|
}
|
|
}
|
|
m_currentBlock = std::numeric_limits<std::uint32_t>::max();
|
|
PlanResourceControlFlow();
|
|
if (m_bdaTraces > bdaTraceLimit) std::fprintf(stderr, "[bda] %u more address accesses in this program not shown\n", m_bdaTraces - bdaTraceLimit);
|
|
LinkImageAliases();
|
|
for (const auto& patch : m_handlePatches) {
|
|
patch.handle->SetFlags<std::uint32_t>(patch.resource);
|
|
}
|
|
for (const auto& patch : m_memoryPatches) {
|
|
auto& memory = m_program.Resources().memoryInfo[patch.index];
|
|
memory.resource = patch.resource;
|
|
if (patch.hasSampler) {
|
|
memory.sampler = patch.sampler;
|
|
}
|
|
}
|
|
for (const auto& plan : m_indirectImages) {
|
|
plan.handle->ReplaceArgument(0, plan.key);
|
|
for (std::uint32_t dword = 0; dword < 4u; dword++) {
|
|
plan.handle->ReplaceArgument(dword + 1u, plan.roots[dword]);
|
|
}
|
|
for (std::uint32_t dword = 5u; dword < plan.roots.size(); dword++) {
|
|
plan.handle->ReplaceArgument(dword, plan.key);
|
|
}
|
|
for (const auto index : plan.memory) {
|
|
m_program.Resources().memoryInfo[index].planningOnly = true;
|
|
}
|
|
}
|
|
std::erase_if(m_program.Metadata().dynamicReads, [&](IrValue* value) {
|
|
const IrValue* inst = value->Resolve();
|
|
return std::ranges::any_of(m_indirectImages, [&](const IndirectImagePlan& plan) {
|
|
return std::ranges::find(plan.reads, inst) != plan.reads.end();
|
|
});
|
|
});
|
|
m_program.Resources().descriptorSources = std::move(m_sources);
|
|
m_program.Resources().info = std::move(m_info);
|
|
m_program.Resources().resourceTrackingComplete = true;
|
|
}
|
|
|
|
private:
|
|
struct HandlePatch {
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t resource = 0;
|
|
};
|
|
|
|
struct MemoryPatch {
|
|
std::uint32_t index = 0;
|
|
std::uint32_t resource = 0;
|
|
std::uint32_t sampler = 0;
|
|
bool hasSampler = false;
|
|
};
|
|
|
|
struct IndirectImagePlan {
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t source = 0;
|
|
IrValue* key = nullptr;
|
|
std::array<IrValue*, 8> roots {};
|
|
std::array<std::uint32_t, 8> memory {};
|
|
std::array<const IrValue*, 8> reads {};
|
|
};
|
|
|
|
void MakeSource(const IrValue& handle, std::uint32_t width, bool sampler, bool sampleAdjust, DescriptorSource& descriptor) {
|
|
if (handle.ArgumentCount() != width) {
|
|
fail(std::string(IrOpcodeName(handle.Opcode())) + " has " + std::to_string(handle.ArgumentCount()) + " descriptor dwords, expected " + std::to_string(width));
|
|
}
|
|
descriptor.dwordCount = width;
|
|
for (std::uint32_t i = 0; i < width; i++) {
|
|
descriptor.dwords[i] = handle.Argument(i)->Resolve();
|
|
}
|
|
if (sampleAdjust) {
|
|
descriptor.dwords[3] = canonicalizeSampleAdjustDword3(descriptor.dwords[3]);
|
|
}
|
|
const IrValue* dword0 = descriptor.dwords[0]->Resolve();
|
|
if (sampler && dword0->HasImmediate() && dword0->Type() == IrType::U32 && (dword0->ImmediateU32() & samplerBorderClampMask) == 0u) {
|
|
descriptor.dwords[3] = &m_builder.Constant(0u);
|
|
}
|
|
}
|
|
|
|
IrValue* canonicalizeSampleAdjustDword3(IrValue* value) {
|
|
for (;;) {
|
|
value = value->Resolve();
|
|
if (value->Opcode() != IrOpcode::BitwiseOr32) {
|
|
return value;
|
|
}
|
|
IrValue* left = value->Argument(0)->Resolve();
|
|
IrValue* right = value->Argument(1)->Resolve();
|
|
const bool leftReserved = (possibleU32Bits(left) & ~samplerDword3ReservedMask) == 0u;
|
|
const bool rightReserved = (possibleU32Bits(right) & ~samplerDword3ReservedMask) == 0u;
|
|
if (leftReserved && rightReserved) {
|
|
return &m_builder.Constant(0u);
|
|
}
|
|
if (leftReserved) {
|
|
value = right;
|
|
} else if (rightReserved) {
|
|
value = left;
|
|
} else {
|
|
return value;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool ValidateSource(const DescriptorSource& descriptor, std::uint32_t& badDword) const {
|
|
constexpr SrtWalker walker;
|
|
for (std::uint32_t i = 0; i < descriptor.dwordCount; i++) {
|
|
badDword = i;
|
|
if (descriptor.dwords[i]->Resolve()->Type() != IrType::U32) {
|
|
return false;
|
|
}
|
|
if (!walker.ValidateRuntimeValue(m_program.Resources(), descriptor.dwords[i])) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
std::uint32_t InternSource(const DescriptorSource& descriptor) {
|
|
for (std::uint32_t candidate = 0; candidate < m_sources.size(); candidate++) {
|
|
const auto& current = m_sources[candidate];
|
|
if (current.dwordCount != descriptor.dwordCount || current.indirectImage != descriptor.indirectImage) {
|
|
continue;
|
|
}
|
|
bool same = true;
|
|
for (std::uint32_t i = 0; i < descriptor.dwordCount; i++) {
|
|
same = same && EquivalentValue(m_program.Resources(), current.dwords[i], descriptor.dwords[i]);
|
|
}
|
|
if (same) {
|
|
NoteBlockSource(candidate);
|
|
return candidate;
|
|
}
|
|
}
|
|
m_sources.push_back(descriptor);
|
|
NoteBlockSource(static_cast<std::uint32_t>(m_sources.size() - 1));
|
|
return static_cast<std::uint32_t>(m_sources.size() - 1);
|
|
}
|
|
|
|
void NoteBlockImage(std::uint32_t image) {
|
|
if (m_currentBlock >= m_blockImages.size()) {
|
|
return;
|
|
}
|
|
auto& images = m_blockImages[m_currentBlock];
|
|
if (std::ranges::find(images, image) == images.end()) {
|
|
images.push_back(image);
|
|
}
|
|
}
|
|
|
|
void NoteBlockSource(std::uint32_t source) {
|
|
if (m_currentBlock >= m_blockSources.size()) {
|
|
return;
|
|
}
|
|
auto& sources = m_blockSources[m_currentBlock];
|
|
if (std::ranges::find(sources, source) == sources.end()) {
|
|
sources.push_back(source);
|
|
}
|
|
}
|
|
|
|
void PlanResourceControlFlow() {
|
|
const auto& order = m_program.BlockOrder();
|
|
const auto& info = m_program.Metadata().blockInfo;
|
|
if (order.empty() || order.size() != info.size()) {
|
|
return;
|
|
}
|
|
std::unordered_map<const IrBlock*, std::uint32_t> indexOf;
|
|
std::unordered_map<std::uint32_t, std::uint32_t> idToIndex;
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
indexOf.emplace(order[index], index);
|
|
idToIndex.emplace(info[index].id, index);
|
|
}
|
|
std::vector<ResourceBlock> controlFlow(order.size());
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
auto& block = controlFlow[index];
|
|
const auto& terminator = info[index].terminator;
|
|
const auto target = [&](std::uint32_t id) { return idToIndex.find(id); };
|
|
const auto trueEdge = target(terminator.trueBlock);
|
|
const auto falseEdge = target(terminator.falseBlock);
|
|
if (terminator.kind == TerminatorKind::ConditionalBranch && !terminator.loopHeader && info[index].condition != nullptr && trueEdge != idToIndex.end() && falseEdge != idToIndex.end()) {
|
|
block.condition = info[index].condition;
|
|
block.successors = {trueEdge->second, falseEdge->second};
|
|
} else if (terminator.kind != TerminatorKind::Return) {
|
|
for (const IrBlock* successor : order[index]->Successors()) {
|
|
const auto found = indexOf.find(successor);
|
|
if (found == indexOf.end()) {
|
|
return;
|
|
}
|
|
block.successors.push_back(found->second);
|
|
}
|
|
}
|
|
block.sources = m_blockSources[index];
|
|
block.images = m_blockImages[index];
|
|
}
|
|
std::vector<std::uint8_t> unconditional(order.size());
|
|
for (std::uint32_t index = 0; index < order.size() && unconditional[index] == 0u;) {
|
|
unconditional[index] = 1u;
|
|
if (controlFlow[index].successors.size() != 1u) {
|
|
break;
|
|
}
|
|
index = controlFlow[index].successors.front();
|
|
}
|
|
std::vector<std::uint8_t> reached(m_sources.size());
|
|
std::vector<std::uint8_t> reachedImages(m_info.images.size());
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
if (unconditional[index] == 0u) {
|
|
continue;
|
|
}
|
|
for (const auto source : controlFlow[index].sources) {
|
|
reached[source] = 1u;
|
|
}
|
|
for (const auto image : controlFlow[index].images) {
|
|
reachedImages[image] = 1u;
|
|
}
|
|
}
|
|
bool guarded = false;
|
|
for (const auto& block : controlFlow) {
|
|
for (const auto source : block.sources) {
|
|
guarded = guarded || reached[source] == 0u;
|
|
}
|
|
for (const auto image : block.images) {
|
|
guarded = guarded || reachedImages[image] == 0u;
|
|
}
|
|
}
|
|
if (guarded) {
|
|
m_program.Resources().controlFlow = std::move(controlFlow);
|
|
}
|
|
}
|
|
|
|
static bool immediateU32(IrValue* value, std::uint32_t& result) {
|
|
value = value->Resolve();
|
|
if (!value->HasImmediate() || value->Type() != IrType::U32) {
|
|
return false;
|
|
}
|
|
result = value->ImmediateU32();
|
|
return true;
|
|
}
|
|
|
|
static bool usesOnly(const IrValue& value, std::span<const IrValue* const> users) {
|
|
return !value.Uses().empty() && std::ranges::all_of(value.Uses(), [&](const IrValue* user) {
|
|
return std::ranges::find(users, user) != users.end();
|
|
});
|
|
}
|
|
|
|
const MemoryInfo* ScalarReadMemory(const IrValue& read, std::uint32_t& index) const {
|
|
if (read.Opcode() != IrOpcode::ReadConstBuffer || read.ArgumentCount() != 2u) {
|
|
return nullptr;
|
|
}
|
|
index = read.Flags<MemoryFlags>().index;
|
|
if (index >= m_program.Resources().memoryInfo.size()) {
|
|
return nullptr;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[index];
|
|
return memory.kind == ResourceKind::ScalarBuffer && memory.dataBits == 32u && memory.dataDwords == 1u ? &memory : nullptr;
|
|
}
|
|
|
|
static bool AccessesMemory(IrOpcode op) {
|
|
return BufferAccessOf(op) != BufferAccess::None || AddressOpcodeInfoOf(op).access != AddressAccess::None || ImageOpcodeInfoOf(op).access != ImageAccess::None;
|
|
}
|
|
|
|
void IndexMemoryUsers() {
|
|
for (const auto& block : m_program.Blocks()) {
|
|
for (const IrValue* inst : block->Instructions()) {
|
|
if (AccessesMemory(inst->Opcode())) {
|
|
m_memoryUsers[inst->Flags<MemoryFlags>().index].push_back(inst);
|
|
}
|
|
}
|
|
}
|
|
m_memoryUsersIndexed = true;
|
|
}
|
|
|
|
bool MemoryIndexBelongsTo(std::uint32_t index, const IrValue& owner) const {
|
|
if (m_memoryUsersIndexed) {
|
|
const auto users = m_memoryUsers.find(index);
|
|
return users == m_memoryUsers.end() || std::ranges::all_of(users->second, [&](const IrValue* user) { return user == &owner; });
|
|
}
|
|
for (const auto& block : m_program.Blocks()) {
|
|
for (const IrValue* inst : block->Instructions()) {
|
|
if (!AccessesMemory(inst->Opcode()) || inst == &owner) {
|
|
continue;
|
|
}
|
|
if (inst->Flags<MemoryFlags>().index == index) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool MakeRuntimeBufferSource(const IrValue& handle, std::uint32_t& source, DescriptorSource& descriptor) {
|
|
if (handle.Opcode() != IrOpcode::GetBufferResource) {
|
|
return false;
|
|
}
|
|
MakeSource(handle, 4u, false, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
if (!ValidateSource(descriptor, badDword)) {
|
|
return false;
|
|
}
|
|
source = InternSource(descriptor);
|
|
return true;
|
|
}
|
|
|
|
bool MatchMaterialOffset(IrValue* value, IrValue*& selector, std::uint32_t& stride, std::uint32_t& offset) const {
|
|
value = value->Resolve();
|
|
offset = 0;
|
|
IrValue* candidate = value;
|
|
if (candidate->Opcode() == IrOpcode::IAdd32 && candidate->ArgumentCount() == 2u) {
|
|
std::uint32_t immediate = 0;
|
|
if (immediateU32(candidate->Argument(0), immediate)) {
|
|
value = candidate->Argument(1)->Resolve();
|
|
} else if (immediateU32(candidate->Argument(1), immediate)) {
|
|
value = candidate->Argument(0)->Resolve();
|
|
} else {
|
|
return false;
|
|
}
|
|
offset = immediate;
|
|
}
|
|
IrValue* multiply = value;
|
|
if (multiply->Opcode() != IrOpcode::IMul32 || multiply->ArgumentCount() != 2u) {
|
|
return false;
|
|
}
|
|
if (immediateU32(multiply->Argument(0), stride)) {
|
|
selector = multiply->Argument(1)->Resolve();
|
|
} else if (immediateU32(multiply->Argument(1), stride)) {
|
|
selector = multiply->Argument(0)->Resolve();
|
|
} else {
|
|
return false;
|
|
}
|
|
return stride != 0u && selector->Opcode() == IrOpcode::ReadFirstLane;
|
|
}
|
|
|
|
// The table entry offset `key * 32` (a shift or a multiply), optionally plus an immediate.
|
|
static bool MatchTableOffset(IrValue* value, IrValue*& key, std::uint32_t& entryOffset) {
|
|
value = value->Resolve();
|
|
entryOffset = 0;
|
|
if (value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) {
|
|
std::uint32_t immediate = 0;
|
|
if (immediateU32(value->Argument(0), immediate)) {
|
|
value = value->Argument(1)->Resolve();
|
|
} else if (immediateU32(value->Argument(1), immediate)) {
|
|
value = value->Argument(0)->Resolve();
|
|
} else {
|
|
return false;
|
|
}
|
|
entryOffset = immediate;
|
|
}
|
|
if (value->ArgumentCount() != 2u) {
|
|
return false;
|
|
}
|
|
std::uint32_t scale = 0;
|
|
if (value->Opcode() == IrOpcode::ShiftLeftLogical32) {
|
|
if (!immediateU32(value->Argument(1), scale) || scale != 5u) {
|
|
return false;
|
|
}
|
|
key = value->Argument(0)->Resolve();
|
|
return true;
|
|
}
|
|
if (value->Opcode() != IrOpcode::IMul32) {
|
|
return false;
|
|
}
|
|
if (immediateU32(value->Argument(1), scale)) {
|
|
key = value->Argument(0)->Resolve();
|
|
} else if (immediateU32(value->Argument(0), scale)) {
|
|
key = value->Argument(1)->Resolve();
|
|
} else {
|
|
return false;
|
|
}
|
|
return scale == 32u;
|
|
}
|
|
|
|
// A T# whose eight dwords are scalar reads of one V# (the table) at `entryOffset + key * 32`
|
|
// with a wave-uniform runtime key: a bindless image table. The key stays an ordinary value
|
|
// (the SPIR-V selects the bound slot from it); the eight reads become planning-only.
|
|
bool TryMakeTableImage(IrValue& handle, IndirectImagePlan& plan) {
|
|
if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u) {
|
|
return false;
|
|
}
|
|
|
|
std::array<IrValue*, 8> heapReads {};
|
|
IrValue* heapHandle = nullptr;
|
|
IrValue* heapOffset = nullptr;
|
|
std::uint32_t immediateOffset = 0;
|
|
for (std::uint32_t dword = 0; dword < heapReads.size(); dword++) {
|
|
heapReads[dword] = handle.Argument(dword)->Resolve();
|
|
std::uint32_t memoryIndex = 0;
|
|
const MemoryInfo* memory = ScalarReadMemory(*heapReads[dword], memoryIndex);
|
|
if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *heapReads[dword])) {
|
|
return false;
|
|
}
|
|
if (dword == 0u) {
|
|
immediateOffset = memory->offset;
|
|
} else if (memory->offset != immediateOffset + dword * sizeof(std::uint32_t)) {
|
|
return false;
|
|
}
|
|
IrValue* currentHandle = heapReads[dword]->Argument(0)->Resolve();
|
|
if (heapHandle != nullptr && currentHandle != heapHandle) {
|
|
return false;
|
|
}
|
|
heapHandle = currentHandle;
|
|
if (dword == 0u) {
|
|
heapOffset = heapReads[dword]->Argument(1)->Resolve();
|
|
} else if (!EquivalentValue(m_program.Resources(), heapOffset, heapReads[dword]->Argument(1))) {
|
|
return false;
|
|
}
|
|
plan.memory[dword] = memoryIndex;
|
|
plan.reads[dword] = heapReads[dword];
|
|
}
|
|
|
|
IrValue* key = nullptr;
|
|
std::uint32_t entryOffset = 0;
|
|
if (!MatchTableOffset(heapOffset, key, entryOffset) || key->Type() != IrType::U32) {
|
|
return false;
|
|
}
|
|
entryOffset += immediateOffset;
|
|
|
|
const std::array<const IrValue*, 1> imageUsers {&handle};
|
|
for (const auto* read : heapReads) {
|
|
if (!usesOnly(*read, imageUsers)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
DescriptorSource heapSource;
|
|
std::uint32_t heapSourceIndex = 0;
|
|
if (!MakeRuntimeBufferSource(*heapHandle, heapSourceIndex, heapSource)) {
|
|
return false;
|
|
}
|
|
|
|
DescriptorSource::IndirectImage table;
|
|
table.heapSource = heapSourceIndex;
|
|
table.materialSource = heapSourceIndex;
|
|
table.entryOffset = entryOffset;
|
|
DescriptorSource materialSource = heapSource;
|
|
std::uint32_t materialMemoryIndex = 0;
|
|
const MemoryInfo* materialMemory = ScalarReadMemory(*key, materialMemoryIndex);
|
|
if (materialMemory != nullptr && MemoryIndexBelongsTo(materialMemoryIndex, *key)) {
|
|
IrValue* selector = nullptr;
|
|
std::uint32_t selectorStride = 0;
|
|
std::uint32_t selectorOffset = 0;
|
|
DescriptorSource candidateSource;
|
|
std::uint32_t candidateIndex = 0;
|
|
if (MatchMaterialOffset(key->Argument(1), selector, selectorStride, selectorOffset) && MakeRuntimeBufferSource(*key->Argument(0)->Resolve(), candidateIndex, candidateSource)) {
|
|
table.hasMaterial = true;
|
|
table.materialSource = candidateIndex;
|
|
table.selectorStride = selectorStride;
|
|
table.selectorOffset = selectorOffset + materialMemory->offset;
|
|
materialSource = candidateSource;
|
|
}
|
|
}
|
|
|
|
DescriptorSource imageSource;
|
|
imageSource.dwordCount = 8u;
|
|
std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin());
|
|
std::copy(materialSource.dwords.begin(), materialSource.dwords.begin() + 4u, imageSource.dwords.begin() + 4u);
|
|
imageSource.indirectImage = table;
|
|
|
|
plan.handle = &handle;
|
|
plan.source = InternSource(imageSource);
|
|
plan.key = key;
|
|
plan.roots = imageSource.dwords;
|
|
return true;
|
|
}
|
|
|
|
const IndirectImagePlan* FindIndirectImage(const IrValue& handle) const {
|
|
const auto found = std::ranges::find_if(m_indirectImages, [&](const IndirectImagePlan& plan) {
|
|
return plan.handle == &handle;
|
|
});
|
|
return found == m_indirectImages.end() ? nullptr : &*found;
|
|
}
|
|
|
|
bool IsIndirectPlanningMemory(std::uint32_t index) const {
|
|
return std::ranges::any_of(m_indirectImages, [&](const IndirectImagePlan& plan) {
|
|
return std::ranges::find(plan.memory, index) != plan.memory.end();
|
|
});
|
|
}
|
|
|
|
static constexpr std::uint32_t phiSearchDepth = 8u;
|
|
|
|
static bool SplittableImageRead(const IrValue& inst) {
|
|
switch (inst.Opcode()) {
|
|
case IrOpcode::ImageSampleRaw:
|
|
case IrOpcode::ImageGatherRaw:
|
|
case IrOpcode::ImageQueryLod:
|
|
case IrOpcode::ImageQueryDimensions:
|
|
case IrOpcode::ImageRead: return inst.Type() == IrType::U32x4;
|
|
default: return false;
|
|
}
|
|
}
|
|
|
|
void CollectDescriptorPhis(IrValue* value, std::vector<IrValue*>& phis, std::uint32_t depth) const {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return;
|
|
}
|
|
if (value->IsPhi()) {
|
|
if (ResolveInvariantPhi(m_program.Resources(), value) == nullptr && std::ranges::find(phis, value) == phis.end()) {
|
|
phis.push_back(value);
|
|
}
|
|
return;
|
|
}
|
|
if (depth != 0u && value->Opcode() == IrOpcode::ReadFirstLane && value->ArgumentCount() == 2u) {
|
|
CollectDescriptorPhis(value->Argument(0), phis, depth - 1u);
|
|
return;
|
|
}
|
|
if (depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) {
|
|
return;
|
|
}
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
CollectDescriptorPhis(value->Argument(index), phis, depth - 1u);
|
|
}
|
|
}
|
|
|
|
static IrValue* PhiIncoming(const IrValue& phi, const IrBlock* predecessor) {
|
|
for (std::size_t index = 0; index < phi.ArgumentCount(); index++) {
|
|
if (phi.PhiBlock(index) == predecessor) {
|
|
return phi.Argument(index);
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
IrValue* EmitBefore(IrValue& position, IrOpcode op, IrType type, std::span<IrValue* const> arguments, std::uint64_t flags = 0) {
|
|
IrValue& value = m_program.CreateValue(op, type, flags);
|
|
for (IrValue* argument : arguments) {
|
|
value.AddArgument(argument);
|
|
}
|
|
position.Parent()->InsertInstructionBefore(&position, &value);
|
|
return &value;
|
|
}
|
|
|
|
IrValue* EmitCopy(const IrValue& original, std::span<IrValue* const> arguments, IrValue& position) {
|
|
return EmitBefore(position, original.Opcode(), original.Type(), arguments, original.Flags<std::uint64_t>());
|
|
}
|
|
|
|
IrValue* Rematerialize(IrValue* value, IrValue& position, bool emit, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return value;
|
|
}
|
|
if (value->IsPhi()) {
|
|
IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value);
|
|
return invariant == nullptr || invariant->IsPhi() ? nullptr : Rematerialize(invariant, position, emit, depth);
|
|
}
|
|
if (depth == 0u) {
|
|
return nullptr;
|
|
}
|
|
const auto op = value->Opcode();
|
|
const bool register_ = op == IrOpcode::GetUserData;
|
|
const bool argumentless = (op == IrOpcode::GetShaderBase || op == IrOpcode::GetSrtResource) && value->ArgumentCount() == 0u;
|
|
const bool srtRead = op == IrOpcode::ReadConst && value->ArgumentCount() == 2u && value->Argument(0)->Resolve()->Opcode() == IrOpcode::GetSrtResource && value->Argument(1)->Resolve()->HasImmediate();
|
|
if (!register_ && !argumentless && !srtRead && !Detail::IsRuntimeUniformOp(op)) {
|
|
return nullptr;
|
|
}
|
|
std::vector<IrValue*> arguments;
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
IrValue* argument = register_ ? value->Argument(index) : Rematerialize(value->Argument(index), position, emit, depth - 1u);
|
|
if (argument == nullptr) {
|
|
return nullptr;
|
|
}
|
|
arguments.push_back(argument);
|
|
}
|
|
return emit ? EmitCopy(*value, arguments, position) : value;
|
|
}
|
|
|
|
IrValue* SubstituteEdge(IrValue* value, const IrBlock* predecessor, IrValue& position, bool emit, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
std::vector<IrValue*> phis;
|
|
CollectDescriptorPhis(value, phis, depth);
|
|
if (phis.empty()) {
|
|
return value;
|
|
}
|
|
if (value->IsPhi()) {
|
|
IrValue* incoming = PhiIncoming(*value, predecessor);
|
|
return incoming == nullptr ? nullptr : Rematerialize(incoming, position, emit, phiSearchDepth);
|
|
}
|
|
std::vector<IrValue*> arguments;
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
IrValue* argument = SubstituteEdge(value->Argument(index), predecessor, position, emit, depth - 1u);
|
|
if (argument == nullptr) {
|
|
return nullptr;
|
|
}
|
|
arguments.push_back(argument);
|
|
}
|
|
return emit ? EmitCopy(*value, arguments, position) : value;
|
|
}
|
|
|
|
IrValue& EdgeSelector(IrBlock& block, const std::vector<bool>& edges) {
|
|
for (const auto& selector : m_edgeSelectors) {
|
|
if (selector.block == &block && selector.edges == edges) {
|
|
return *selector.value;
|
|
}
|
|
}
|
|
IrValue& phi = m_program.CreateValue(IrOpcode::Phi, IrType::Bool);
|
|
const auto& predecessors = block.Predecessors();
|
|
for (std::size_t index = 0; index < predecessors.size(); index++) {
|
|
phi.AddPhiOperand(predecessors[index], &m_builder.ConstantBool(edges[index]));
|
|
}
|
|
block.InsertInstructionBefore(nullptr, &phi);
|
|
m_edgeSelectors.push_back({&block, edges, &phi});
|
|
return phi;
|
|
}
|
|
|
|
void SplitDescriptorPhi(IrValue& inst) {
|
|
const auto info = ImageOpcodeInfoOf(inst.Opcode());
|
|
const std::uint32_t handleCount = info.needsSampler ? 2u : 1u;
|
|
if (inst.ArgumentCount() < handleCount || inst.Parent() == nullptr) {
|
|
return;
|
|
}
|
|
std::array<IrValue*, 2> handles {};
|
|
std::array<bool, 2> split {};
|
|
std::vector<IrValue*> phis;
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
IrValue* handle = inst.Argument(slot)->Resolve();
|
|
const auto expected = slot == 0u ? IrOpcode::GetImageResource : IrOpcode::GetSamplerResource;
|
|
if (handle->Opcode() != expected || handle->ArgumentCount() != (slot == 0u ? 8u : 4u)) {
|
|
return;
|
|
}
|
|
handles[slot] = handle;
|
|
std::vector<IrValue*> handlePhis;
|
|
for (std::size_t dword = 0; dword < handle->ArgumentCount(); dword++) {
|
|
CollectDescriptorPhis(handle->Argument(dword), handlePhis, phiSearchDepth);
|
|
}
|
|
split[slot] = !handlePhis.empty();
|
|
for (IrValue* phi : handlePhis) {
|
|
if (std::ranges::find(phis, phi) == phis.end()) {
|
|
phis.push_back(phi);
|
|
}
|
|
}
|
|
}
|
|
if (phis.empty()) {
|
|
return;
|
|
}
|
|
if (!SplitDescriptorEdges(inst, handles, split, handleCount, phis)) {
|
|
SplitDescriptorWeb(inst, handles, split, handleCount);
|
|
}
|
|
}
|
|
|
|
bool SplitDescriptorEdges(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount, const std::vector<IrValue*>& phis) {
|
|
IrBlock* block = phis.front()->Parent();
|
|
if (block == nullptr) {
|
|
return false;
|
|
}
|
|
const auto& predecessors = block->Predecessors();
|
|
if (predecessors.size() < 2u) {
|
|
return false;
|
|
}
|
|
for (const IrValue* phi : phis) {
|
|
if (phi->Parent() != block || phi->PhiBlockCount() != predecessors.size()) {
|
|
return false;
|
|
}
|
|
for (const IrBlock* predecessor : predecessors) {
|
|
if (PhiIncoming(*phi, predecessor) == nullptr) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<std::uint32_t> edgeArm(predecessors.size());
|
|
std::vector<const IrBlock*> arms;
|
|
for (std::size_t edge = 0; edge < predecessors.size(); edge++) {
|
|
std::uint32_t arm = 0;
|
|
for (; arm < arms.size(); arm++) {
|
|
const bool same = std::ranges::all_of(phis, [&](const IrValue* phi) {
|
|
return EquivalentValue(m_program.Resources(), PhiIncoming(*phi, predecessors[edge]), PhiIncoming(*phi, arms[arm]));
|
|
});
|
|
if (same) {
|
|
break;
|
|
}
|
|
}
|
|
if (arm == arms.size()) {
|
|
arms.push_back(predecessors[edge]);
|
|
}
|
|
edgeArm[edge] = arm;
|
|
}
|
|
for (const IrBlock* arm : arms) {
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) {
|
|
if (SubstituteEdge(handles[slot]->Argument(dword), arm, inst, false, phiSearchDepth) == nullptr) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<IrValue*> conditions;
|
|
for (std::uint32_t arm = 0; arm + 1u < arms.size(); arm++) {
|
|
std::vector<bool> edges(edgeArm.size());
|
|
for (std::size_t edge = 0; edge < edgeArm.size(); edge++) {
|
|
edges[edge] = edgeArm[edge] == arm;
|
|
}
|
|
conditions.push_back(&EdgeSelector(*block, edges));
|
|
}
|
|
ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) {
|
|
return SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth);
|
|
});
|
|
return true;
|
|
}
|
|
|
|
template <typename Dword>
|
|
void ReplaceWithArmCopies(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount, std::span<IrValue* const> conditions, Dword&& dword) {
|
|
std::vector<IrValue*> copies;
|
|
const auto flags = inst.Flags<MemoryFlags>();
|
|
const auto memory = m_program.Resources().memoryInfo.at(flags.index);
|
|
for (std::uint32_t arm = 0; arm <= conditions.size(); arm++) {
|
|
std::vector<IrValue*> arguments(inst.Arguments().begin(), inst.Arguments().end());
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
if (!split[slot]) {
|
|
continue;
|
|
}
|
|
std::vector<IrValue*> dwords;
|
|
for (std::size_t index = 0; index < handles[slot]->ArgumentCount(); index++) {
|
|
dwords.push_back(dword(arm, slot, index));
|
|
}
|
|
arguments[slot] = EmitCopy(*handles[slot], dwords, inst);
|
|
}
|
|
auto copyFlags = flags;
|
|
if (arm != 0u) {
|
|
copyFlags.index = static_cast<std::uint32_t>(m_program.Resources().memoryInfo.size());
|
|
m_program.Resources().memoryInfo.push_back(memory);
|
|
}
|
|
std::uint64_t rawFlags = 0;
|
|
std::memcpy(&rawFlags, ©Flags, sizeof(copyFlags));
|
|
copies.push_back(EmitBefore(inst, inst.Opcode(), inst.Type(), arguments, rawFlags));
|
|
}
|
|
|
|
IrValue* result = copies.back();
|
|
if (copies.size() > 1u) {
|
|
std::array<IrValue*, 4> components {};
|
|
for (std::uint32_t component = 0; component < components.size(); component++) {
|
|
const auto extract = [&](IrValue* vector) {
|
|
const std::array<IrValue*, 2> arguments {vector, &m_builder.Constant(component)};
|
|
return EmitBefore(inst, IrOpcode::CompositeExtractU32x4, IrType::U32, arguments);
|
|
};
|
|
IrValue* selected = extract(copies.back());
|
|
for (std::uint32_t arm = static_cast<std::uint32_t>(copies.size()) - 1u; arm-- > 0u;) {
|
|
const std::array<IrValue*, 3> arguments {conditions[arm], extract(copies[arm]), selected};
|
|
selected = EmitBefore(inst, IrOpcode::SelectU32, IrType::U32, arguments);
|
|
}
|
|
components[component] = selected;
|
|
}
|
|
result = EmitBefore(inst, IrOpcode::CompositeConstructU32x4, IrType::U32x4, components);
|
|
}
|
|
inst.ReplaceAllUsesWith(result);
|
|
inst.Invalidate();
|
|
inst.Parent()->RemoveInstruction(&inst);
|
|
}
|
|
|
|
static bool HoldsProgramCounter(IrValue* value, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->Opcode() == IrOpcode::GetShaderBase) {
|
|
return true;
|
|
}
|
|
if (value->HasImmediate() || depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) {
|
|
return false;
|
|
}
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
if (HoldsProgramCounter(value->Argument(index), depth - 1u)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
struct DescriptorWeb {
|
|
static constexpr std::size_t maxStates = 512u;
|
|
static constexpr std::size_t maxArms = 64u;
|
|
std::vector<std::vector<IrValue*>> arms;
|
|
std::map<std::vector<IrValue*>, IrValue*> selectors;
|
|
std::vector<std::pair<IrBlock*, IrValue*>> phis;
|
|
std::vector<std::tuple<IrBlock*, IrValue*, IrValue*>> lanes;
|
|
std::unordered_map<IrValue*, IrValue*> resolved;
|
|
std::unordered_map<const IrBlock*, std::size_t> order;
|
|
};
|
|
|
|
IrValue* WebSelector(std::vector<IrValue*> tuple, DescriptorWeb& web, IrValue& position) {
|
|
for (IrValue*& value : tuple) {
|
|
value = value->Resolve();
|
|
if (!value->IsPhi()) {
|
|
continue;
|
|
}
|
|
const auto [entry, inserted] = web.resolved.try_emplace(value, value);
|
|
if (inserted) {
|
|
IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value);
|
|
entry->second = invariant == nullptr ? value : invariant->Resolve();
|
|
}
|
|
value = entry->second;
|
|
}
|
|
if (const auto known = web.selectors.find(tuple); known != web.selectors.end()) {
|
|
return known->second;
|
|
}
|
|
const auto phi = std::ranges::find_if(tuple, [](const IrValue* value) { return value->IsPhi(); });
|
|
const auto lane = std::ranges::find_if(tuple, [](const IrValue* value) { return value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::ReadFirstLane; });
|
|
if (phi == tuple.end() && lane != tuple.end()) {
|
|
return LaneSelector(tuple, **lane, web, position);
|
|
}
|
|
if (phi == tuple.end()) {
|
|
for (IrValue* value : tuple) {
|
|
if (HoldsProgramCounter(value, phiSearchDepth) || Rematerialize(value, position, false, phiSearchDepth) == nullptr) {
|
|
return nullptr;
|
|
}
|
|
}
|
|
const auto same = std::ranges::find_if(web.arms, [&](const std::vector<IrValue*>& arm) {
|
|
return std::ranges::equal(tuple, arm, [&](IrValue* left, IrValue* right) { return EquivalentValue(m_program.Resources(), left, right); });
|
|
});
|
|
if (same != web.arms.end()) {
|
|
return &m_builder.Constant(static_cast<std::uint32_t>(same - web.arms.begin()));
|
|
}
|
|
if (web.arms.size() == DescriptorWeb::maxArms) {
|
|
return nullptr;
|
|
}
|
|
web.arms.push_back(std::move(tuple));
|
|
return &m_builder.Constant(static_cast<std::uint32_t>(web.arms.size() - 1u));
|
|
}
|
|
if (web.order.empty()) {
|
|
const auto& order = m_program.BlockOrder();
|
|
for (std::size_t index = 0; index < order.size(); index++) {
|
|
web.order.emplace(order[index], index);
|
|
}
|
|
}
|
|
IrBlock* block = nullptr;
|
|
std::size_t latest = 0;
|
|
for (const IrValue* value : tuple) {
|
|
if (!value->IsPhi()) {
|
|
continue;
|
|
}
|
|
const auto found = web.order.find(value->Parent());
|
|
if (found == web.order.end()) {
|
|
return nullptr;
|
|
}
|
|
if (block == nullptr || found->second > latest) {
|
|
block = value->Parent();
|
|
latest = found->second;
|
|
}
|
|
}
|
|
if (block == nullptr || web.selectors.size() == DescriptorWeb::maxStates) {
|
|
return nullptr;
|
|
}
|
|
const auto& predecessors = block->Predecessors();
|
|
if (predecessors.size() < 2u) {
|
|
return nullptr;
|
|
}
|
|
for (const IrValue* value : tuple) {
|
|
if (value->IsPhi() && value->Parent() == block && value->PhiBlockCount() != predecessors.size()) {
|
|
return nullptr;
|
|
}
|
|
}
|
|
IrValue& selector = m_program.CreateValue(IrOpcode::Phi, IrType::U32);
|
|
web.selectors.emplace(tuple, &selector);
|
|
web.phis.emplace_back(block, &selector);
|
|
for (IrBlock* predecessor : predecessors) {
|
|
std::vector<IrValue*> incoming;
|
|
for (IrValue* value : tuple) {
|
|
IrValue* edge = value->IsPhi() && value->Parent() == block ? PhiIncoming(*value, predecessor) : value;
|
|
if (edge == nullptr) {
|
|
return nullptr;
|
|
}
|
|
incoming.push_back(edge);
|
|
}
|
|
IrValue* edgeSelector = WebSelector(std::move(incoming), web, position);
|
|
if (edgeSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
selector.AddPhiOperand(predecessor, edgeSelector);
|
|
}
|
|
return &selector;
|
|
}
|
|
|
|
IrValue* LaneSelector(const std::vector<IrValue*>& tuple, IrValue& first, DescriptorWeb& web, IrValue& position) {
|
|
IrBlock* block = first.Parent();
|
|
const auto op = first.Opcode();
|
|
const bool select = op == IrOpcode::SelectU32;
|
|
if (block == nullptr || first.ArgumentCount() != (select ? 3u : 2u) || web.selectors.size() == DescriptorWeb::maxStates) {
|
|
return nullptr;
|
|
}
|
|
IrValue* shared = first.Argument(select ? 0u : 1u);
|
|
std::vector<IrValue*> left;
|
|
std::vector<IrValue*> right;
|
|
for (IrValue* value : tuple) {
|
|
if (value->Opcode() == op && value->Parent() == block && value->ArgumentCount() == first.ArgumentCount() && EquivalentValue(m_program.Resources(), value->Argument(select ? 0u : 1u), shared)) {
|
|
left.push_back(value->Argument(select ? 1u : 0u));
|
|
if (select) {
|
|
right.push_back(value->Argument(2u));
|
|
}
|
|
} else if (value->HasImmediate() || (select && !value->IsPhi() && value->Opcode() != IrOpcode::ReadFirstLane)) {
|
|
left.push_back(value);
|
|
if (select) {
|
|
right.push_back(value);
|
|
}
|
|
} else {
|
|
return nullptr;
|
|
}
|
|
}
|
|
auto& instructions = block->Instructions();
|
|
auto latest = instructions.end();
|
|
for (auto it = instructions.begin(); it != instructions.end(); ++it) {
|
|
if (std::ranges::find(tuple, *it) != tuple.end()) {
|
|
latest = it;
|
|
}
|
|
}
|
|
if (latest == instructions.end()) {
|
|
return nullptr;
|
|
}
|
|
IrValue* before = std::next(latest) == instructions.end() ? nullptr : *std::next(latest);
|
|
IrValue* leftSelector = WebSelector(std::move(left), web, position);
|
|
if (leftSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
IrValue& node = m_program.CreateValue(op, IrType::U32, first.Flags<std::uint64_t>());
|
|
if (select) {
|
|
IrValue* rightSelector = WebSelector(std::move(right), web, position);
|
|
if (rightSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
node.AddArgument(shared);
|
|
node.AddArgument(leftSelector);
|
|
node.AddArgument(rightSelector);
|
|
} else {
|
|
node.AddArgument(leftSelector);
|
|
node.AddArgument(shared);
|
|
}
|
|
web.selectors.emplace(tuple, &node);
|
|
web.lanes.emplace_back(block, before, &node);
|
|
return &node;
|
|
}
|
|
|
|
void SplitDescriptorWeb(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount) {
|
|
std::vector<IrValue*> tuple;
|
|
std::array<std::size_t, 2> offsets {};
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
offsets[slot] = tuple.size();
|
|
for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) {
|
|
tuple.push_back(handles[slot]->Argument(dword));
|
|
}
|
|
}
|
|
DescriptorWeb web;
|
|
IrValue* selector = WebSelector(tuple, web, inst);
|
|
const bool select = selector != nullptr && web.arms.size() > 1u;
|
|
if (select) {
|
|
for (const auto& [block, phi] : web.phis) {
|
|
block->InsertInstructionBefore(nullptr, phi);
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
if (before == nullptr) {
|
|
block->AppendInstruction(node);
|
|
} else {
|
|
block->InsertInstructionBefore(before, node);
|
|
}
|
|
}
|
|
} else {
|
|
for (const auto& [block, phi] : web.phis) {
|
|
for (std::size_t index = 0; index < phi->ArgumentCount(); index++) {
|
|
phi->ReplaceArgument(index, nullptr);
|
|
}
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
for (std::size_t index = 0; index < node->ArgumentCount(); index++) {
|
|
node->ReplaceArgument(index, nullptr);
|
|
}
|
|
}
|
|
for (const auto& [block, phi] : web.phis) {
|
|
phi->Invalidate();
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
node->Invalidate();
|
|
}
|
|
}
|
|
if (selector == nullptr || web.arms.empty()) {
|
|
return;
|
|
}
|
|
|
|
std::vector<IrValue*> conditions;
|
|
for (std::uint32_t arm = 0; arm + 1u < web.arms.size(); arm++) {
|
|
const std::array<IrValue*, 2> compare {selector, &m_builder.Constant(arm)};
|
|
conditions.push_back(EmitBefore(inst, IrOpcode::IEqual32, IrType::Bool, compare));
|
|
}
|
|
ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) {
|
|
return Rematerialize(web.arms[arm][offsets[slot] + dword], inst, true, phiSearchDepth);
|
|
});
|
|
}
|
|
|
|
void SplitDescriptorPhis() {
|
|
std::vector<IrValue*> candidates;
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
if (SplittableImageRead(*inst)) {
|
|
candidates.push_back(inst);
|
|
}
|
|
}
|
|
}
|
|
for (IrValue* inst : candidates) {
|
|
SplitDescriptorPhi(*inst);
|
|
}
|
|
}
|
|
|
|
void PlanIndirectImages() {
|
|
// Kill switch: APS5_NO_BINDLESS_IMAGES=1 leaves table-loaded T#s unplanned (GetHandle then
|
|
// rejects them as before).
|
|
static const bool enabled = std::getenv("APS5_NO_BINDLESS_IMAGES") == nullptr;
|
|
if (!enabled) {
|
|
return;
|
|
}
|
|
IndexMemoryUsers();
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode());
|
|
if (imageInfo.access == ImageAccess::None || inst->ArgumentCount() == 0u) {
|
|
continue;
|
|
}
|
|
IrValue* handle = inst->Argument(0)->Resolve();
|
|
const IndirectImagePlan* planned = FindIndirectImage(*handle);
|
|
IndirectImagePlan plan;
|
|
if (planned == nullptr && !TryMakeTableImage(*handle, plan)) {
|
|
continue;
|
|
}
|
|
// A store through a table would mark every bound slot pending write-back.
|
|
if (imageInfo.resourceClass == ImageResourceClass::Storage) {
|
|
ResourceMaterializer::CountBindlessRejection(BindlessRejection::Storage);
|
|
fail("bindless storage image tables are unsupported");
|
|
}
|
|
if (planned == nullptr) {
|
|
m_indirectImages.push_back(std::move(plan));
|
|
}
|
|
}
|
|
}
|
|
m_memoryUsersIndexed = false;
|
|
m_memoryUsers.clear();
|
|
}
|
|
|
|
void GetHandle(IrValue* value, IrOpcode expected, std::uint32_t width, IrValue*& handle, std::uint32_t& source, bool sampler = false, bool sampleAdjust = false) {
|
|
handle = value->Resolve();
|
|
if (handle->Opcode() != expected) {
|
|
fail("memory operation requires " + std::string(IrOpcodeName(expected)));
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(*handle, width, sampler, sampleAdjust, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
if (expected == IrOpcode::GetImageResource) {
|
|
for (; badDword < descriptor.dwordCount; badDword++) {
|
|
const IrValue* value2 = descriptor.dwords[badDword]->Resolve();
|
|
if (value2->Opcode() == IrOpcode::ReadConstBuffer) {
|
|
fail(std::string(IrOpcodeName(expected)) + " dword " + std::to_string(badDword) + " is not a valid runtime value; chain: " + describeValueChain(descriptor.dwords[badDword], 8u));
|
|
}
|
|
}
|
|
badDword = 0;
|
|
}
|
|
if (!ValidateSource(descriptor, badDword)) {
|
|
fail(std::string(IrOpcodeName(expected)) + " dword " + std::to_string(badDword) + " is not a valid runtime value; chain: " + describeValueChain(descriptor.dwords[badDword], 8u));
|
|
}
|
|
source = InternSource(descriptor);
|
|
}
|
|
|
|
bool TakeGpuDescriptor(IrValue& inst, std::uint32_t memoryIndex) {
|
|
const IrValue* handle = inst.Argument(0)->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) {
|
|
return false;
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(*handle, 4u, false, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
auto& memory = m_program.Resources().memoryInfo[memoryIndex];
|
|
const auto access = BufferAccessOf(inst.Opcode());
|
|
if (ValidateSource(descriptor, badDword)) {
|
|
const auto source = InternSource(descriptor);
|
|
const auto resource = AddBuffer(source, memory, inst.Opcode(), inst.Flags<MemoryFlags>().pc);
|
|
if (resource == std::numeric_limits<std::uint32_t>::max()) fail("buffer resource limit exceeded");
|
|
AddMemoryPatch(memoryIndex, resource, 0u, false);
|
|
memory.gpuDescriptor = false;
|
|
m_info.usesFaultBuffer = m_info.usesFaultBuffer || access == BufferAccess::Atomic;
|
|
return true;
|
|
}
|
|
memory.gpuDescriptor = true;
|
|
m_info.usesDma = true;
|
|
m_info.bdaWrites = m_info.bdaWrites || access == BufferAccess::Write || access == BufferAccess::Atomic;
|
|
return true;
|
|
}
|
|
|
|
void ValidateAddressHandle(IrValue* value) const {
|
|
const IrValue* handle = value->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetAddressResource) {
|
|
fail("address operation requires GetAddressResource");
|
|
}
|
|
if (handle->ArgumentCount() != 2) {
|
|
fail("GetAddressResource must have two address dwords");
|
|
}
|
|
}
|
|
|
|
std::uint32_t AddBuffer(std::uint32_t source, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
for (std::uint32_t i = 0; i < m_info.buffers.size(); i++) {
|
|
if (m_info.buffers[i].source == source) {
|
|
Merge(m_info.buffers[i], memory, op, pc);
|
|
return i;
|
|
}
|
|
}
|
|
BufferResource resource;
|
|
resource.source = source;
|
|
resource.firstUsePc = pc;
|
|
Merge(resource, memory, op, pc);
|
|
m_info.buffers.push_back(resource);
|
|
return static_cast<std::uint32_t>(m_info.buffers.size() - 1);
|
|
}
|
|
|
|
static void Merge(BufferResource& resource, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto access = BufferAccessOf(op);
|
|
const bool atomic = access == BufferAccess::Atomic;
|
|
const bool write = access == BufferAccess::Write || atomic;
|
|
resource.firstUsePc = std::min(resource.firstUsePc, pc);
|
|
resource.maxByteExtent = std::max(resource.maxByteExtent, byteExtent(memory));
|
|
resource.read = resource.read || !write || atomic;
|
|
resource.written = resource.written || write;
|
|
resource.atomic = resource.atomic || atomic;
|
|
resource.formatted = resource.formatted || memory.formatted;
|
|
resource.descriptorFormatted = resource.descriptorFormatted || (memory.formatted && !memory.typed);
|
|
if (memory.formatted && memory.typed) {
|
|
const auto format = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat));
|
|
if (format.byteSize == 0u) fail("typed buffer instruction has an invalid format");
|
|
resource.typedAlignment = std::max(resource.typedAlignment, static_cast<std::uint8_t>(std::min(format.byteSize, 4u)));
|
|
}
|
|
if (memory.formatted && !memory.typed && !write) resource.formattedReadMask |= (1u << std::min(memory.dataDwords, 4u)) - 1u;
|
|
resource.scalar = resource.scalar || op == IrOpcode::ReadConstBuffer || memory.kind == ResourceKind::ScalarBuffer;
|
|
}
|
|
|
|
std::uint32_t AddImage(std::uint32_t source, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto resourceClass = ImageOpcodeInfoOf(op).resourceClass;
|
|
const auto mip = resourceClass == ImageResourceClass::Storage && memory.imageHasMip ? ImageMipMode::DynamicStorage : ImageMipMode::None;
|
|
const bool depth = (memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0;
|
|
for (std::uint32_t i = 0; i < m_info.images.size(); i++) {
|
|
auto& image = m_info.images[i];
|
|
if (image.source == source && image.resourceClass == resourceClass && image.dimension == memory.imageDimension && image.mipMode == mip && image.depthCompare == depth && image.r128 == memory.imageR128 && image.packed == memory.imagePacked && image.byElements == memory.imageByElements && image.byComponents == (memory.imageByElements != 0u ? memory.dataDwords / memory.imageByElements : 0u)) {
|
|
Merge(image, memory, op, pc);
|
|
return i;
|
|
}
|
|
}
|
|
if (m_info.images.size() >= ShaderInfo::MaxImages) {
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
ImageResource image;
|
|
image.source = source;
|
|
image.firstUsePc = pc;
|
|
image.resourceClass = resourceClass;
|
|
image.dimension = memory.imageDimension;
|
|
image.mipMode = mip;
|
|
image.depthCompare = depth;
|
|
image.r128 = memory.imageR128;
|
|
image.packed = memory.imagePacked;
|
|
image.byElements = memory.imageByElements;
|
|
image.byComponents = memory.imageByElements != 0u ? memory.dataDwords / memory.imageByElements : 0u;
|
|
Merge(image, memory, op, pc);
|
|
m_info.images.push_back(image);
|
|
return static_cast<std::uint32_t>(m_info.images.size() - 1);
|
|
}
|
|
|
|
static void Merge(ImageResource& image, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto access = ImageOpcodeInfoOf(op).access;
|
|
const bool atomic = access == ImageAccess::Atomic;
|
|
const bool write = access == ImageAccess::Write || atomic;
|
|
const bool atomic64 = IsImageAtomic64Opcode(op);
|
|
if ((image.read || image.written) && image.atomic64 != atomic64) {
|
|
throw std::runtime_error("an image accessed by 64-bit atomics is also accessed in another way");
|
|
}
|
|
image.atomic64 = atomic64;
|
|
image.firstUsePc = std::min(image.firstUsePc, pc);
|
|
image.read = image.read || !write || atomic;
|
|
image.written = image.written || write;
|
|
image.atomic = image.atomic || atomic;
|
|
image.srgbDecodeCompatible = image.srgbDecodeCompatible && !ImageOpcodeInfoOf(op).needsSampler;
|
|
image.fmaskCompatible = image.fmaskCompatible && op == IrOpcode::ImageRead && memory.dataBits == 32u;
|
|
image.depthBitsCompatible = image.depthBitsCompatible && memory.dataBits == 32u;
|
|
image.constantSwizzleCompatible = image.constantSwizzleCompatible && (op == IrOpcode::ImageSampleRaw || op == IrOpcode::ImageGatherRaw);
|
|
const bool flatVolumeLoad = op == IrOpcode::ImageRead && memory.imageSampleFlags == 0u;
|
|
const bool flatVolumeSample = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLevelZero;
|
|
const bool flatVolumeQuery = op == IrOpcode::ImageQueryDimensions;
|
|
image.flatVolumeCompatible = image.flatVolumeCompatible && (flatVolumeLoad || flatVolumeSample || flatVolumeQuery) && !memory.imagePacked && memory.imageByElements == 0u;
|
|
const bool flatLineLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u;
|
|
const bool flatLineSample = op == IrOpcode::ImageSampleRaw && (memory.imageSampleFlags == RdnaImageSampleFlagLevelZero || memory.imageSampleFlags == 0u);
|
|
image.flatLineCompatible = image.flatLineCompatible && (flatLineLoad || flatLineSample) && !memory.imagePacked && memory.imageByElements == 0u;
|
|
if ((memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0u) {
|
|
constexpr auto unsupported = RdnaImageSampleFlagLod | RdnaImageSampleFlagDerivative;
|
|
if (op == IrOpcode::ImageGatherRaw || (memory.imageSampleFlags & unsupported) != 0u) image.emulatedCompare |= EmulatedCompare::Unsupported;
|
|
if ((memory.imageSampleFlags & RdnaImageSampleFlagLevelZero) == 0u) image.emulatedCompare |= EmulatedCompare::RequiresSingleLevel;
|
|
}
|
|
}
|
|
|
|
std::uint32_t AddSampler(std::uint32_t source, std::uint32_t pc) {
|
|
for (std::uint32_t i = 0; i < m_info.samplers.size(); i++) {
|
|
if (m_info.samplers[i].source == source) {
|
|
m_info.samplers[i].firstUsePc = std::min(m_info.samplers[i].firstUsePc, pc);
|
|
return i;
|
|
}
|
|
}
|
|
if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) {
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
m_info.samplers.push_back({source, pc});
|
|
return static_cast<std::uint32_t>(m_info.samplers.size() - 1);
|
|
}
|
|
|
|
void AddSampledPair(std::uint32_t image, std::uint32_t sampler, std::uint32_t pc) {
|
|
for (auto& pair : m_info.sampledPairs) {
|
|
if (pair.image == image && pair.sampler == sampler) {
|
|
pair.firstUsePc = std::min(pair.firstUsePc, pc);
|
|
return;
|
|
}
|
|
}
|
|
if (m_info.sampledPairs.size() >= ShaderInfo::MaxSampledPairs) {
|
|
fail("sampled image/sampler pair limit exceeded");
|
|
}
|
|
m_info.sampledPairs.push_back({image, sampler, pc});
|
|
}
|
|
|
|
void AddHandlePatch(IrValue* handle, std::uint32_t resource) {
|
|
for (const auto& patch : m_handlePatches) {
|
|
if (patch.handle == handle) {
|
|
if (patch.resource != resource) {
|
|
fail(std::string(IrOpcodeName(handle->Opcode())) + " is reused with incompatible resource classes");
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
m_handlePatches.push_back({handle, resource});
|
|
}
|
|
|
|
void AddMemoryPatch(std::uint32_t index, std::uint32_t resource, std::uint32_t sampler, bool hasSampler) {
|
|
for (auto& patch : m_memoryPatches) {
|
|
if (patch.index != index) {
|
|
continue;
|
|
}
|
|
if (patch.resource != resource || (hasSampler && patch.hasSampler && patch.sampler != sampler)) {
|
|
fail("memory metadata is reused with incompatible resources");
|
|
}
|
|
if (hasSampler) {
|
|
patch.sampler = sampler;
|
|
patch.hasSampler = true;
|
|
}
|
|
return;
|
|
}
|
|
m_memoryPatches.push_back({index, resource, sampler, hasSampler});
|
|
}
|
|
|
|
void Collect(IrValue& inst) {
|
|
const auto op = inst.Opcode();
|
|
if (op == IrOpcode::ImageBvhIntersectRay) {
|
|
m_info.usesDma = true;
|
|
return;
|
|
}
|
|
const auto buffer = BufferAccessOf(op);
|
|
const auto addressInfo = AddressOpcodeInfoOf(op);
|
|
const auto imageInfo = ImageOpcodeInfoOf(op);
|
|
if (buffer == BufferAccess::None && addressInfo.access == AddressAccess::None && imageInfo.access == ImageAccess::None) {
|
|
return;
|
|
}
|
|
const auto flags = inst.Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
fail("memory metadata index " + std::to_string(flags.index) + " is out of range");
|
|
}
|
|
if (inst.ArgumentCount() == 0) {
|
|
fail("memory operation has no resource handle");
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[flags.index];
|
|
if (memory.planningOnly || IsIndirectPlanningMemory(flags.index)) {
|
|
return;
|
|
}
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t source = 0;
|
|
std::uint32_t resource = 0;
|
|
|
|
if (buffer != BufferAccess::None) {
|
|
if (!TakeGpuDescriptor(inst, flags.index)) fail("buffer operation requires a four-dword runtime V#");
|
|
return;
|
|
}
|
|
if (addressInfo.access != AddressAccess::None) {
|
|
if (!IsAddressResourceKind(memory.kind)) {
|
|
fail("address operation has invalid resource kind");
|
|
}
|
|
if (memory.kind == ResourceKind::Scratch) {
|
|
handle = inst.Argument(0)->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetScratchResource || handle->ArgumentCount() != 0) {
|
|
fail("scratch operation requires GetScratchResource");
|
|
}
|
|
if (m_info.scratchDwords == 0) {
|
|
fail("scratch operation requires a nonzero AGC per-thread size");
|
|
}
|
|
return;
|
|
}
|
|
ValidateAddressHandle(inst.Argument(0));
|
|
// Debug aid: APS5_TRACE_BDA=1 names every access that makes the program address-based
|
|
// (a raw scalar load the SRT walker left in place, or a flat/global access), so the
|
|
// reason a stage takes the BDA path can be read off without a shader dump.
|
|
// The first few accesses of a program are printed (a Bink kernel has hundreds); Run
|
|
// reports how many more there were.
|
|
if (bdaTraceEnabled() && ++m_bdaTraces <= bdaTraceLimit) {
|
|
const auto* kind = memory.kind == ResourceKind::ScalarAddress ? "scalar address" : memory.kind == ResourceKind::Global ? "global" : "flat";
|
|
const IrValue* offset = inst.ArgumentCount() > 1 ? inst.Argument(1)->Resolve() : nullptr;
|
|
const bool immediateOffset = offset != nullptr && offset->HasImmediate();
|
|
std::fprintf(stderr, "[bda] %s at pc 0x%08x: %s access, offset %s%s\n", std::string(IrOpcodeName(op)).c_str(), flags.pc, kind, immediateOffset ? "immediate" : "dynamic", memory.kind == ResourceKind::ScalarAddress && !immediateOffset ? " (a register offset is not planned by the SRT walker)" : "");
|
|
}
|
|
m_info.usesDma = true;
|
|
m_info.bdaWrites = m_info.bdaWrites || addressInfo.access == AddressAccess::Write || addressInfo.access == AddressAccess::Atomic;
|
|
return;
|
|
}
|
|
|
|
if (memory.kind != ResourceKind::Image || imageInfo.resourceClass == ImageResourceClass::None) {
|
|
fail("image operation has invalid resource kind");
|
|
}
|
|
handle = inst.Argument(0)->Resolve();
|
|
const IndirectImagePlan* indirect = FindIndirectImage(*handle);
|
|
if (indirect != nullptr) {
|
|
source = indirect->source;
|
|
} else {
|
|
GetHandle(inst.Argument(0), IrOpcode::GetImageResource, 8, handle, source);
|
|
}
|
|
resource = AddImage(source, memory, op, flags.pc);
|
|
NoteBlockImage(resource);
|
|
if (resource == std::numeric_limits<std::uint32_t>::max()) {
|
|
fail("image resource limit exceeded");
|
|
}
|
|
AddHandlePatch(handle, resource);
|
|
std::uint32_t sampler = 0;
|
|
if (imageInfo.needsSampler) {
|
|
if (inst.ArgumentCount() < 2) {
|
|
fail("sampled image operation has no sampler handle");
|
|
}
|
|
IrValue* samplerHandle = nullptr;
|
|
std::uint32_t samplerSource = 0;
|
|
const bool sampleAdjust = (memory.imageSampleFlags & RdnaImageSampleFlagAdjust) != 0;
|
|
GetHandle(inst.Argument(1), IrOpcode::GetSamplerResource, 4, samplerHandle, samplerSource, true, sampleAdjust);
|
|
sampler = AddSampler(samplerSource, flags.pc);
|
|
if (sampler == std::numeric_limits<std::uint32_t>::max()) {
|
|
fail("sampler resource limit exceeded");
|
|
}
|
|
AddHandlePatch(samplerHandle, sampler);
|
|
AddSampledPair(resource, sampler, flags.pc);
|
|
}
|
|
AddMemoryPatch(flags.index, resource, sampler, imageInfo.needsSampler);
|
|
}
|
|
|
|
const DescriptorSource* Source(std::uint32_t source) const {
|
|
return source < m_sources.size() ? &m_sources[source] : nullptr;
|
|
}
|
|
|
|
void LinkImageAliases() {
|
|
for (auto& buffer : m_info.buffers) {
|
|
const DescriptorSource* bufferSource = Source(buffer.source);
|
|
if (bufferSource == nullptr || bufferSource->dwordCount != 4) {
|
|
continue;
|
|
}
|
|
for (std::uint32_t image = 0; image < m_info.images.size(); image++) {
|
|
const DescriptorSource* imageSource = Source(m_info.images[image].source);
|
|
if (imageSource == nullptr || imageSource->dwordCount != 8 || imageSource->indirectImage.has_value()) {
|
|
continue;
|
|
}
|
|
bool alias = true;
|
|
for (std::uint32_t dword = 0; dword < 4; dword++) {
|
|
alias = alias && EquivalentValue(m_program.Resources(), bufferSource->dwords[dword], imageSource->dwords[dword]);
|
|
}
|
|
if (alias) {
|
|
buffer.imageAlias = image;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
IrProgram& m_program;
|
|
ShaderInfo m_info;
|
|
IrBuilder m_builder;
|
|
std::vector<DescriptorSource> m_sources;
|
|
std::vector<std::vector<std::uint32_t>> m_blockSources;
|
|
std::vector<std::vector<std::uint32_t>> m_blockImages;
|
|
std::uint32_t m_currentBlock = std::numeric_limits<std::uint32_t>::max();
|
|
std::vector<HandlePatch> m_handlePatches;
|
|
std::vector<MemoryPatch> m_memoryPatches;
|
|
// APS5_TRACE_BDA: address accesses seen by Collect (the first bdaTraceLimit are printed).
|
|
unsigned m_bdaTraces = 0;
|
|
std::vector<IndirectImagePlan> m_indirectImages;
|
|
std::unordered_map<std::uint32_t, std::vector<const IrValue*>> m_memoryUsers;
|
|
bool m_memoryUsersIndexed = false;
|
|
struct EdgeSelectorEntry {
|
|
IrBlock* block = nullptr;
|
|
std::vector<bool> edges;
|
|
IrValue* value = nullptr;
|
|
};
|
|
std::vector<EdgeSelectorEntry> m_edgeSelectors;
|
|
};
|
|
|
|
}
|
|
|
|
void ResourceTracker::Track(IrProgram& program) const {
|
|
Tracker(program).Run();
|
|
}
|
|
|
|
}
|