mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-18 23:07:07 -05:00
2042 lines
92 KiB
C++
2042 lines
92 KiB
C++
#include "Optimization/ResourceTracker.hpp"
|
|
#include "Optimization/SrtWalker.hpp"
|
|
#include "Optimization/ResourceMaterializer.hpp"
|
|
#include "SpirvBackend/SpirvBufferFormat.hpp"
|
|
#include "IntermediateRepresentation/IrBuilder.hpp"
|
|
#include "Optimization/SrtWalker/SrtInstructionPredicates.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <bit>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <limits>
|
|
#include <map>
|
|
#include <span>
|
|
#include <string>
|
|
#include <tuple>
|
|
#include <unordered_map>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
namespace ShaderRecompiler {
|
|
namespace {
|
|
|
|
constexpr std::uint32_t samplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
|
|
constexpr std::uint32_t samplerDword3ReservedMask = 0x3ffff000u;
|
|
|
|
[[noreturn]] void fail(const std::string& message) {
|
|
throw std::runtime_error(message);
|
|
}
|
|
|
|
// Debug aid: APS5_TRACE_BDA=1 names the accesses that make a program address-based (see Collect).
|
|
bool bdaTraceEnabled() {
|
|
static const bool enabled = std::getenv("APS5_TRACE_BDA") != nullptr;
|
|
return enabled;
|
|
}
|
|
constexpr unsigned bdaTraceLimit = 8;
|
|
|
|
std::string formatHex32(std::uint32_t value) {
|
|
static constexpr char digits[] = "0123456789abcdef";
|
|
std::string hex(8u, '0');
|
|
for (std::uint32_t index = 0; index < 8u; index++) {
|
|
hex[7u - index] = digits[(value >> (index * 4u)) & 0xfu];
|
|
}
|
|
return hex;
|
|
}
|
|
|
|
std::string describeValueChain(const IrValue* value, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return "Immediate";
|
|
}
|
|
std::string text = std::string(IrOpcodeName(value->Opcode()));
|
|
if (value->Opcode() == IrOpcode::LoadAddressU32 || value->Opcode() == IrOpcode::ReadConstBuffer) {
|
|
text += "@pc=0x" + formatHex32(value->Flags<MemoryFlags>().pc);
|
|
}
|
|
if (depth == 0u || value->ArgumentCount() == 0u) {
|
|
return text;
|
|
}
|
|
text += "(";
|
|
for (std::uint32_t index = 0; index < value->ArgumentCount(); index++) {
|
|
if (index != 0u) {
|
|
text += ", ";
|
|
}
|
|
text += describeValueChain(value->Argument(index), depth - 1u);
|
|
}
|
|
text += ")";
|
|
return text;
|
|
}
|
|
|
|
std::uint32_t possibleU32Bits(const IrValue* value) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return value->Type() == IrType::U32 ? value->ImmediateU32() : std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
switch (value->Opcode()) {
|
|
case IrOpcode::BitwiseAnd32:
|
|
return possibleU32Bits(value->Argument(0)) & possibleU32Bits(value->Argument(1));
|
|
case IrOpcode::BitwiseOr32:
|
|
return possibleU32Bits(value->Argument(0)) | possibleU32Bits(value->Argument(1));
|
|
case IrOpcode::ShiftLeftLogical32: {
|
|
const IrValue* shift = value->Argument(1)->Resolve();
|
|
return shift->HasImmediate() && shift->Type() == IrType::U32 ? possibleU32Bits(value->Argument(0)) << (shift->ImmediateU32() & 31u) : std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
default:
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
}
|
|
|
|
std::uint32_t byteExtent(const MemoryInfo& memory) {
|
|
const auto bytes = std::max((memory.dataBits + 7u) / 8u, 1u);
|
|
const auto count = std::max(memory.dataDwords, 1u);
|
|
const auto end = static_cast<std::uint64_t>(memory.offset) + static_cast<std::uint64_t>(bytes) * count;
|
|
return end > std::numeric_limits<std::uint32_t>::max() ? std::numeric_limits<std::uint32_t>::max() : static_cast<std::uint32_t>(end);
|
|
}
|
|
|
|
class Tracker {
|
|
public:
|
|
explicit Tracker(IrProgram& program) : m_program(program), m_info(program.Resources().info), m_builder(program) {
|
|
m_info.buffers.clear();
|
|
m_info.images.clear();
|
|
m_info.samplers.clear();
|
|
m_info.sampledPairs.clear();
|
|
m_info.usesDma = false;
|
|
m_info.bdaWrites = false;
|
|
}
|
|
|
|
void Run() {
|
|
if (m_program.Resources().resourceTrackingComplete) {
|
|
fail("resources already tracked");
|
|
}
|
|
if (!m_program.Resources().srtPlanComplete) {
|
|
fail("SRT plan is not ready");
|
|
}
|
|
SplitDescriptorPhis();
|
|
PlanTableColumns();
|
|
std::unordered_map<const IrBlock*, std::uint32_t> blockIndices;
|
|
for (const IrBlock* block : m_program.BlockOrder()) {
|
|
blockIndices.emplace(block, static_cast<std::uint32_t>(blockIndices.size()));
|
|
}
|
|
m_blockSources.assign(blockIndices.size(), {});
|
|
m_blockImages.assign(blockIndices.size(), {});
|
|
for (auto& block : m_program.Blocks()) {
|
|
const auto found = blockIndices.find(block.get());
|
|
m_currentBlock = found == blockIndices.end() ? std::numeric_limits<std::uint32_t>::max() : found->second;
|
|
for (IrValue* inst : block->Instructions()) {
|
|
Collect(*inst);
|
|
}
|
|
}
|
|
m_currentBlock = std::numeric_limits<std::uint32_t>::max();
|
|
PlanResourceControlFlow();
|
|
if (m_bdaTraces > bdaTraceLimit) std::fprintf(stderr, "[bda] %u more address accesses in this program not shown\n", m_bdaTraces - bdaTraceLimit);
|
|
CheckTableStores();
|
|
AssignTables();
|
|
LinkImageAliases();
|
|
for (const auto& patch : m_handlePatches) {
|
|
patch.handle->SetFlags<std::uint32_t>(patch.resource);
|
|
}
|
|
for (const auto& patch : m_memoryPatches) {
|
|
auto& memory = m_program.Resources().memoryInfo[patch.index];
|
|
memory.resource = patch.resource;
|
|
if (patch.hasSampler) {
|
|
memory.sampler = patch.sampler;
|
|
}
|
|
}
|
|
std::vector<const IrValue*> referenced;
|
|
for (const auto& plan : m_tableColumns) {
|
|
const auto& column = *m_sources[plan.source].tableColumn;
|
|
IrValue* offset = column.selected ? &m_builder.Constant(column.addend) : plan.offset;
|
|
for (std::uint32_t argument = 0; argument < plan.handle->ArgumentCount(); argument++) {
|
|
plan.handle->ReplaceArgument(argument, offset);
|
|
}
|
|
KeepSourceAlive(*plan.handle, m_sources[column.heapSource], referenced);
|
|
if (column.selected) {
|
|
KeepSourceAlive(*plan.handle, m_sources[column.selection], referenced);
|
|
}
|
|
if (plan.keyDomain.has_value()) {
|
|
KeepSourceAlive(*plan.handle, m_sources[plan.keyDomain->source], referenced);
|
|
}
|
|
}
|
|
for (const auto index : m_tablePlanningMemory) {
|
|
m_program.Resources().memoryInfo[index].planningOnly = true;
|
|
}
|
|
std::erase_if(m_program.Metadata().dynamicReads, [&](IrValue* value) {
|
|
const IrValue* inst = value->Resolve();
|
|
return std::ranges::find(m_tablePlanningReads, inst) != m_tablePlanningReads.end();
|
|
});
|
|
m_program.Resources().descriptorSources = std::move(m_sources);
|
|
m_program.Resources().info = std::move(m_info);
|
|
m_program.Resources().resourceTrackingComplete = true;
|
|
}
|
|
|
|
private:
|
|
struct HandlePatch {
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t resource = 0;
|
|
};
|
|
|
|
struct MemoryPatch {
|
|
std::uint32_t index = 0;
|
|
std::uint32_t resource = 0;
|
|
std::uint32_t sampler = 0;
|
|
bool hasSampler = false;
|
|
};
|
|
|
|
struct TableColumnPlan {
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t source = 0;
|
|
IrValue* offset = nullptr;
|
|
std::vector<std::uint32_t> memory;
|
|
std::vector<const IrValue*> reads;
|
|
std::vector<std::uint32_t> hopMemory;
|
|
std::vector<const IrValue*> hopReads;
|
|
std::optional<KeyDomain> keyDomain;
|
|
};
|
|
|
|
void MakeSource(const IrValue& handle, std::uint32_t width, bool sampler, bool sampleAdjust, DescriptorSource& descriptor) {
|
|
if (handle.ArgumentCount() != width) {
|
|
fail(std::string(IrOpcodeName(handle.Opcode())) + " has " + std::to_string(handle.ArgumentCount()) + " descriptor dwords, expected " + std::to_string(width));
|
|
}
|
|
descriptor.dwordCount = width;
|
|
for (std::uint32_t i = 0; i < width; i++) {
|
|
descriptor.dwords[i] = handle.Argument(i)->Resolve();
|
|
}
|
|
if (sampleAdjust) {
|
|
descriptor.dwords[3] = canonicalizeSampleAdjustDword3(descriptor.dwords[3]);
|
|
}
|
|
const IrValue* dword0 = descriptor.dwords[0]->Resolve();
|
|
if (sampler && dword0->HasImmediate() && dword0->Type() == IrType::U32 && (dword0->ImmediateU32() & samplerBorderClampMask) == 0u) {
|
|
descriptor.dwords[3] = &m_builder.Constant(0u);
|
|
}
|
|
}
|
|
|
|
IrValue* canonicalizeSampleAdjustDword3(IrValue* value) {
|
|
for (;;) {
|
|
value = value->Resolve();
|
|
if ((possibleU32Bits(value) & ~samplerDword3ReservedMask) == 0u) {
|
|
return &m_builder.Constant(0u);
|
|
}
|
|
if (value->Opcode() != IrOpcode::BitwiseOr32) {
|
|
return value;
|
|
}
|
|
IrValue* left = value->Argument(0)->Resolve();
|
|
IrValue* right = value->Argument(1)->Resolve();
|
|
const bool leftReserved = (possibleU32Bits(left) & ~samplerDword3ReservedMask) == 0u;
|
|
const bool rightReserved = (possibleU32Bits(right) & ~samplerDword3ReservedMask) == 0u;
|
|
if (leftReserved) {
|
|
value = right;
|
|
} else if (rightReserved) {
|
|
value = left;
|
|
} else {
|
|
return value;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool ValidateSource(const DescriptorSource& descriptor, std::uint32_t& badDword) const {
|
|
constexpr SrtWalker walker;
|
|
for (std::uint32_t i = 0; i < descriptor.dwordCount; i++) {
|
|
badDword = i;
|
|
if (descriptor.dwords[i]->Resolve()->Type() != IrType::U32) {
|
|
return false;
|
|
}
|
|
if (!walker.ValidateRuntimeValue(m_program.Resources(), descriptor.dwords[i])) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
std::uint32_t InternSource(const DescriptorSource& descriptor) {
|
|
for (std::uint32_t candidate = 0; candidate < m_sources.size(); candidate++) {
|
|
const auto& current = m_sources[candidate];
|
|
if (current.dwordCount != descriptor.dwordCount || current.tableColumn != descriptor.tableColumn) {
|
|
continue;
|
|
}
|
|
bool same = true;
|
|
for (std::uint32_t i = 0; i < descriptor.dwordCount; i++) {
|
|
same = same && EquivalentValue(m_program.Resources(), current.dwords[i], descriptor.dwords[i]);
|
|
}
|
|
if (same) {
|
|
NoteBlockSource(candidate);
|
|
return candidate;
|
|
}
|
|
}
|
|
m_sources.push_back(descriptor);
|
|
NoteBlockSource(static_cast<std::uint32_t>(m_sources.size() - 1));
|
|
return static_cast<std::uint32_t>(m_sources.size() - 1);
|
|
}
|
|
|
|
void NoteBlockImage(std::uint32_t image) {
|
|
if (m_currentBlock >= m_blockImages.size()) {
|
|
return;
|
|
}
|
|
auto& images = m_blockImages[m_currentBlock];
|
|
if (std::ranges::find(images, image) == images.end()) {
|
|
images.push_back(image);
|
|
}
|
|
}
|
|
|
|
void NoteBlockSource(std::uint32_t source) {
|
|
if (m_currentBlock >= m_blockSources.size()) {
|
|
return;
|
|
}
|
|
auto& sources = m_blockSources[m_currentBlock];
|
|
if (std::ranges::find(sources, source) == sources.end()) {
|
|
sources.push_back(source);
|
|
}
|
|
}
|
|
|
|
void PlanResourceControlFlow() {
|
|
const auto& order = m_program.BlockOrder();
|
|
const auto& info = m_program.Metadata().blockInfo;
|
|
if (order.empty() || order.size() != info.size()) {
|
|
return;
|
|
}
|
|
std::unordered_map<const IrBlock*, std::uint32_t> indexOf;
|
|
std::unordered_map<std::uint32_t, std::uint32_t> idToIndex;
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
indexOf.emplace(order[index], index);
|
|
idToIndex.emplace(info[index].id, index);
|
|
}
|
|
std::vector<ResourceBlock> controlFlow(order.size());
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
auto& block = controlFlow[index];
|
|
const auto& terminator = info[index].terminator;
|
|
const auto target = [&](std::uint32_t id) { return idToIndex.find(id); };
|
|
const auto trueEdge = target(terminator.trueBlock);
|
|
const auto falseEdge = target(terminator.falseBlock);
|
|
if (terminator.kind == TerminatorKind::ConditionalBranch && !terminator.loopHeader && info[index].condition != nullptr && trueEdge != idToIndex.end() && falseEdge != idToIndex.end()) {
|
|
block.condition = info[index].condition;
|
|
block.successors = {trueEdge->second, falseEdge->second};
|
|
} else if (terminator.kind != TerminatorKind::Return) {
|
|
for (const IrBlock* successor : order[index]->Successors()) {
|
|
const auto found = indexOf.find(successor);
|
|
if (found == indexOf.end()) {
|
|
return;
|
|
}
|
|
block.successors.push_back(found->second);
|
|
}
|
|
}
|
|
block.sources = m_blockSources[index];
|
|
block.images = m_blockImages[index];
|
|
}
|
|
std::vector<std::uint8_t> unconditional(order.size());
|
|
for (std::uint32_t index = 0; index < order.size() && unconditional[index] == 0u;) {
|
|
unconditional[index] = 1u;
|
|
if (controlFlow[index].successors.size() != 1u) {
|
|
break;
|
|
}
|
|
index = controlFlow[index].successors.front();
|
|
}
|
|
std::vector<std::uint8_t> reached(m_sources.size());
|
|
std::vector<std::uint8_t> reachedImages(m_info.images.size());
|
|
for (std::uint32_t index = 0; index < order.size(); index++) {
|
|
if (unconditional[index] == 0u) {
|
|
continue;
|
|
}
|
|
for (const auto source : controlFlow[index].sources) {
|
|
reached[source] = 1u;
|
|
}
|
|
for (const auto image : controlFlow[index].images) {
|
|
reachedImages[image] = 1u;
|
|
}
|
|
}
|
|
bool guarded = false;
|
|
for (const auto& block : controlFlow) {
|
|
for (const auto source : block.sources) {
|
|
guarded = guarded || reached[source] == 0u;
|
|
}
|
|
for (const auto image : block.images) {
|
|
guarded = guarded || reachedImages[image] == 0u;
|
|
}
|
|
}
|
|
if (guarded) {
|
|
m_program.Resources().controlFlow = std::move(controlFlow);
|
|
}
|
|
}
|
|
|
|
static bool immediateU32(IrValue* value, std::uint32_t& result) {
|
|
value = value->Resolve();
|
|
if (!value->HasImmediate() || value->Type() != IrType::U32) {
|
|
return false;
|
|
}
|
|
result = value->ImmediateU32();
|
|
return true;
|
|
}
|
|
|
|
const MemoryInfo* ScalarReadMemory(const IrValue& read, std::uint32_t& index) const {
|
|
if (read.Opcode() != IrOpcode::ReadConstBuffer || read.ArgumentCount() != 2u) {
|
|
return nullptr;
|
|
}
|
|
index = read.Flags<MemoryFlags>().index;
|
|
if (index >= m_program.Resources().memoryInfo.size()) {
|
|
return nullptr;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[index];
|
|
return memory.kind == ResourceKind::ScalarBuffer && memory.dataBits == 32u && memory.dataDwords == 1u ? &memory : nullptr;
|
|
}
|
|
|
|
const MemoryInfo* ScalarAddressMemory(const IrValue& read, std::uint32_t& index) const {
|
|
if (read.Opcode() != IrOpcode::LoadAddressU32 || read.ArgumentCount() != 4u) {
|
|
return nullptr;
|
|
}
|
|
index = read.Flags<MemoryFlags>().index;
|
|
if (index >= m_program.Resources().memoryInfo.size()) {
|
|
return nullptr;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[index];
|
|
std::uint32_t high = 0;
|
|
const IrValue* active = read.Argument(3)->Resolve();
|
|
const bool always = active->HasImmediate() && active->Type() == IrType::U1 && active->ImmediateBool();
|
|
return memory.kind == ResourceKind::ScalarAddress && !memory.addressIsFull && memory.dataBits == 32u && memory.dataDwords == 1u && immediateU32(read.Argument(2), high) && high == 0u && always ? &memory : nullptr;
|
|
}
|
|
|
|
static bool AccessesMemory(IrOpcode op) {
|
|
return BufferAccessOf(op) != BufferAccess::None || AddressOpcodeInfoOf(op).access != AddressAccess::None || ImageOpcodeInfoOf(op).access != ImageAccess::None;
|
|
}
|
|
|
|
void IndexMemoryUsers() {
|
|
for (const auto& block : m_program.Blocks()) {
|
|
for (const IrValue* inst : block->Instructions()) {
|
|
if (AccessesMemory(inst->Opcode())) {
|
|
m_memoryUsers[inst->Flags<MemoryFlags>().index].push_back(inst);
|
|
}
|
|
}
|
|
}
|
|
m_memoryUsersIndexed = true;
|
|
}
|
|
|
|
bool MemoryIndexBelongsTo(std::uint32_t index, const IrValue& owner) const {
|
|
if (m_memoryUsersIndexed) {
|
|
const auto users = m_memoryUsers.find(index);
|
|
return users == m_memoryUsers.end() || std::ranges::all_of(users->second, [&](const IrValue* user) { return user == &owner; });
|
|
}
|
|
for (const auto& block : m_program.Blocks()) {
|
|
for (const IrValue* inst : block->Instructions()) {
|
|
if (!AccessesMemory(inst->Opcode()) || inst == &owner) {
|
|
continue;
|
|
}
|
|
if (inst->Flags<MemoryFlags>().index == index) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool MakeRuntimeSource(const IrValue& handle, IrOpcode expected, std::uint32_t width, std::uint32_t& source, DescriptorSource& descriptor) {
|
|
if (handle.Opcode() != expected) {
|
|
return false;
|
|
}
|
|
MakeSource(handle, width, false, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
if (!ValidateSource(descriptor, badDword)) {
|
|
return false;
|
|
}
|
|
source = InternSource(descriptor);
|
|
return true;
|
|
}
|
|
|
|
static bool MatchScale(IrValue* value, IrValue*& key, std::uint32_t& stride) {
|
|
value = value->Resolve();
|
|
if (value->ArgumentCount() != 2u) {
|
|
return false;
|
|
}
|
|
std::uint32_t immediate = 0;
|
|
if (value->Opcode() == IrOpcode::ShiftLeftLogical32) {
|
|
if (!immediateU32(value->Argument(1), immediate) || immediate >= 32u) {
|
|
return false;
|
|
}
|
|
key = value->Argument(0)->Resolve();
|
|
stride = 1u << immediate;
|
|
return true;
|
|
}
|
|
if (value->Opcode() != IrOpcode::IMul32) {
|
|
return false;
|
|
}
|
|
if (immediateU32(value->Argument(1), immediate)) {
|
|
key = value->Argument(0)->Resolve();
|
|
} else if (immediateU32(value->Argument(0), immediate)) {
|
|
key = value->Argument(1)->Resolve();
|
|
} else {
|
|
return false;
|
|
}
|
|
stride = immediate;
|
|
return true;
|
|
}
|
|
|
|
static bool MatchAffineOffset(IrValue* value, IrValue*& key, std::uint32_t& stride, std::uint32_t& addend) {
|
|
value = value->Resolve();
|
|
addend = 0;
|
|
bool matched = MatchScale(value, key, stride);
|
|
if (!matched && value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) {
|
|
if (immediateU32(value->Argument(0), addend)) {
|
|
matched = MatchScale(value->Argument(1), key, stride);
|
|
} else if (immediateU32(value->Argument(1), addend)) {
|
|
matched = MatchScale(value->Argument(0), key, stride);
|
|
}
|
|
}
|
|
return matched && stride >= 4u && stride % 4u == 0u && key->Type() == IrType::U32;
|
|
}
|
|
|
|
static bool MatchMaskedOffset(IrValue* value, IrValue*& key, std::uint32_t& stride) {
|
|
value = value->Resolve();
|
|
if (value->Opcode() != IrOpcode::BitwiseAnd32 || value->ArgumentCount() != 2u) {
|
|
return false;
|
|
}
|
|
std::uint32_t mask = 0;
|
|
IrValue* scaled = nullptr;
|
|
if (immediateU32(value->Argument(1), mask)) {
|
|
scaled = value->Argument(0);
|
|
} else if (immediateU32(value->Argument(0), mask)) {
|
|
scaled = value->Argument(1);
|
|
} else {
|
|
return false;
|
|
}
|
|
return MatchScale(scaled, key, stride) && std::has_single_bit(stride) && stride >= 4u && (mask & ~(stride - 1u)) != 0u && key->Type() == IrType::U32;
|
|
}
|
|
|
|
static bool ReadsThroughDescriptor(const IrValue* value) {
|
|
std::vector<const IrValue*> pending{value};
|
|
std::vector<const IrValue*> seen;
|
|
while (!pending.empty()) {
|
|
const IrValue* current = pending.back()->Resolve();
|
|
pending.pop_back();
|
|
if (std::find(seen.begin(), seen.end(), current) != seen.end()) continue;
|
|
seen.push_back(current);
|
|
if (current->Opcode() == IrOpcode::ReadConstBuffer || current->Opcode() == IrOpcode::LoadAddressU32) return true;
|
|
for (std::size_t index = 0; index < current->ArgumentCount(); index++) pending.push_back(current->Argument(index));
|
|
}
|
|
return false;
|
|
}
|
|
|
|
std::optional<KeyDomain> MatchKeyDomain(IrValue* key) {
|
|
key = key->Resolve();
|
|
DescriptorSource direct;
|
|
direct.dwordCount = 1u;
|
|
direct.dwords[0] = key;
|
|
std::uint32_t badDword = 0;
|
|
if (ValidateSource(direct, badDword) && !ReadsThroughDescriptor(key)) {
|
|
KeyDomain domain{InternSource(direct), 0u, 4u};
|
|
domain.value = true;
|
|
return domain;
|
|
}
|
|
std::uint32_t memoryIndex = 0;
|
|
const MemoryInfo* memory = ScalarReadMemory(*key, memoryIndex);
|
|
if (memory == nullptr) {
|
|
return std::nullopt;
|
|
}
|
|
DescriptorSource source;
|
|
std::uint32_t sourceIndex = 0;
|
|
if (!MakeRuntimeSource(*key->Argument(0)->Resolve(), IrOpcode::GetBufferResource, 4u, sourceIndex, source)) {
|
|
return std::nullopt;
|
|
}
|
|
IrValue* selector = nullptr;
|
|
std::uint32_t stride = 0;
|
|
std::uint32_t addend = 0;
|
|
if (MatchAffineOffset(key->Argument(1), selector, stride, addend)) {
|
|
return KeyDomain{sourceIndex, memory->offset + addend, stride};
|
|
}
|
|
const auto bits = possibleU32Bits(key->Argument(1));
|
|
const auto zeros = bits == 0u ? 31u : std::min<std::uint32_t>(static_cast<std::uint32_t>(std::countr_zero(bits)), 31u);
|
|
return KeyDomain{sourceIndex, memory->offset, std::max(4u, 1u << zeros)};
|
|
}
|
|
|
|
static IrValue* SplitImmediateAddend(IrValue* value, std::uint32_t& addend) {
|
|
value = value->Resolve();
|
|
addend = 0;
|
|
while (value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) {
|
|
std::uint32_t immediate = 0;
|
|
IrValue* rest = nullptr;
|
|
if (immediateU32(value->Argument(0), immediate)) rest = value->Argument(1)->Resolve();
|
|
else if (immediateU32(value->Argument(1), immediate)) rest = value->Argument(0)->Resolve();
|
|
else break;
|
|
addend += immediate;
|
|
value = rest;
|
|
}
|
|
return value;
|
|
}
|
|
|
|
bool TryMakeTableColumn(IrValue& handle, bool sampler, bool r128, TableColumnPlan& plan) {
|
|
const auto expected = sampler ? IrOpcode::GetSamplerResource : IrOpcode::GetImageResource;
|
|
const std::uint32_t width = sampler ? 4u : 8u;
|
|
if (handle.Opcode() != expected || handle.ArgumentCount() != width) {
|
|
return false;
|
|
}
|
|
std::uint32_t words = width;
|
|
if (!sampler && r128) {
|
|
bool zeroTail = true;
|
|
for (std::uint32_t dword = 4u; dword < 8u; dword++) {
|
|
std::uint32_t immediate = 0;
|
|
zeroTail = zeroTail && immediateU32(handle.Argument(dword), immediate) && immediate == 0u;
|
|
}
|
|
if (zeroTail) {
|
|
words = 4u;
|
|
}
|
|
}
|
|
IrValue* heapHandle = nullptr;
|
|
IrValue* offset = nullptr;
|
|
IrValue* base = nullptr;
|
|
std::uint32_t baseAddend = 0;
|
|
std::uint32_t carry = 0;
|
|
bool folded = false;
|
|
std::uint32_t first = 0;
|
|
bool address = false;
|
|
for (std::uint32_t dword = 0; dword < words; dword++) {
|
|
IrValue* read = handle.Argument(dword)->Resolve();
|
|
std::uint32_t memoryIndex = 0;
|
|
const MemoryInfo* memory = ScalarReadMemory(*read, memoryIndex);
|
|
const bool addressRead = memory == nullptr && (memory = ScalarAddressMemory(*read, memoryIndex)) != nullptr;
|
|
if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) {
|
|
return false;
|
|
}
|
|
std::uint32_t readAddend = 0;
|
|
IrValue* readBase = SplitImmediateAddend(read->Argument(1), readAddend);
|
|
if (dword == 0u) {
|
|
first = memory->offset;
|
|
address = addressRead;
|
|
base = readBase;
|
|
baseAddend = readAddend;
|
|
} else if (addressRead != address || memory->offset + readAddend != first + baseAddend + dword * static_cast<std::uint32_t>(sizeof(std::uint32_t))) {
|
|
return false;
|
|
} else if (readAddend != baseAddend) {
|
|
if (readAddend < baseAddend || static_cast<std::uint64_t>(memory->offset) + (readAddend - baseAddend) > 0x7fffffffu) {
|
|
return false;
|
|
}
|
|
folded = true;
|
|
carry = std::max(carry, readAddend - baseAddend);
|
|
}
|
|
IrValue* currentHandle = read->Argument(0)->Resolve();
|
|
if (heapHandle != nullptr && currentHandle != heapHandle && !EquivalentValue(m_program.Resources(), heapHandle, currentHandle)) {
|
|
return false;
|
|
}
|
|
if (heapHandle == nullptr) heapHandle = currentHandle;
|
|
if (dword == 0u) {
|
|
offset = read->Argument(1)->Resolve();
|
|
} else if (!EquivalentValue(m_program.Resources(), base, readBase)) {
|
|
return false;
|
|
}
|
|
plan.memory.push_back(memoryIndex);
|
|
plan.reads.push_back(read);
|
|
}
|
|
if (address) {
|
|
DescriptorSource direct;
|
|
MakeSource(handle, width, false, false, direct);
|
|
std::uint32_t badDword = 0;
|
|
if (ValidateSource(direct, badDword)) {
|
|
return false;
|
|
}
|
|
}
|
|
IrValue* key = nullptr;
|
|
std::uint32_t stride = 0;
|
|
std::uint32_t addend = 0;
|
|
bool masked = false;
|
|
std::uint32_t descriptorOffset = 0;
|
|
if (address && immediateU32(offset, descriptorOffset)) {
|
|
return TryMakeIndirectColumn(handle, *heapHandle, sampler, words, descriptorOffset, first, plan);
|
|
}
|
|
if (!MatchAffineOffset(offset, key, stride, addend)) {
|
|
if (!MatchMaskedOffset(offset, key, stride)) {
|
|
return false;
|
|
}
|
|
masked = true;
|
|
}
|
|
DescriptorSource heapSource;
|
|
std::uint32_t heapIndex = 0;
|
|
if (!MakeRuntimeSource(*heapHandle, address ? IrOpcode::GetAddressResource : IrOpcode::GetBufferResource, address ? 2u : 4u, heapIndex, heapSource)) {
|
|
return false;
|
|
}
|
|
TableColumn column;
|
|
column.heapSource = heapIndex;
|
|
column.stride = stride;
|
|
column.addend = addend;
|
|
column.offset = first;
|
|
column.dwordCount = words;
|
|
column.maxKey = possibleU32Bits(key);
|
|
column.sampler = sampler;
|
|
column.address = address;
|
|
column.foldedAddend = folded ? baseAddend : 0u;
|
|
column.foldedCarry = folded ? carry : 0u;
|
|
if (!address && !sampler && std::ranges::find(m_unselectedHandles, &handle) == m_unselectedHandles.end()) {
|
|
DescriptorSource direct;
|
|
MakeSource(handle, width, false, false, direct);
|
|
DescriptorSource selection;
|
|
selection.dwordCount = 1u;
|
|
selection.dwords[0] = offset;
|
|
std::uint32_t badDword = 0;
|
|
if (ValidateSource(direct, badDword) && ValidateSource(selection, badDword)) {
|
|
column.selected = true;
|
|
column.selection = InternSource(selection);
|
|
}
|
|
}
|
|
const auto domain = masked ? std::optional<KeyDomain>{} : MatchKeyDomain(key);
|
|
DescriptorSource columnSource = heapSource;
|
|
columnSource.tableColumn = column;
|
|
plan.handle = &handle;
|
|
plan.source = InternSource(columnSource);
|
|
plan.keyDomain = domain;
|
|
auto& shared = *m_sources[plan.source].tableColumn;
|
|
shared.maxKey = std::max(shared.maxKey, column.maxKey);
|
|
if (!domain.has_value()) {
|
|
shared.unkeyed = true;
|
|
shared.keyDomains.clear();
|
|
} else if (!shared.unkeyed && std::find(shared.keyDomains.begin(), shared.keyDomains.end(), *domain) == shared.keyDomains.end()) {
|
|
shared.keyDomains.push_back(*domain);
|
|
}
|
|
plan.offset = offset;
|
|
return true;
|
|
}
|
|
|
|
bool TryMakeIndirectColumn(IrValue& handle, IrValue& descriptorPointer, bool sampler, std::uint32_t words, std::uint32_t descriptorOffset, std::uint32_t descriptorImmediate, TableColumnPlan& plan) {
|
|
std::vector<PointerHop> hops;
|
|
IrValue* pointer = &descriptorPointer;
|
|
while (pointer->Opcode() == IrOpcode::GetAddressResource && pointer->ArgumentCount() == 2u) {
|
|
std::uint32_t memoryIndex = 0;
|
|
if (ScalarAddressMemory(*pointer->Argument(0)->Resolve(), memoryIndex) == nullptr) {
|
|
break;
|
|
}
|
|
PointerHop hop;
|
|
IrValue* next = nullptr;
|
|
for (std::uint32_t dword = 0; dword < 2u; dword++) {
|
|
IrValue* read = pointer->Argument(dword)->Resolve();
|
|
const MemoryInfo* memory = ScalarAddressMemory(*read, memoryIndex);
|
|
std::uint32_t readOffset = 0;
|
|
if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read) || !immediateU32(read->Argument(1), readOffset)) {
|
|
return false;
|
|
}
|
|
IrValue* readPointer = read->Argument(0)->Resolve();
|
|
if (dword == 0u) {
|
|
hop = PointerHop{readOffset, memory->offset};
|
|
next = readPointer;
|
|
} else if (readOffset != hop.offset || memory->offset != hop.immediate + static_cast<std::uint32_t>(sizeof(std::uint32_t)) || (readPointer != next && !EquivalentValue(m_program.Resources(), next, readPointer))) {
|
|
return false;
|
|
}
|
|
plan.hopMemory.push_back(memoryIndex);
|
|
plan.hopReads.push_back(read);
|
|
}
|
|
hops.insert(hops.begin(), hop);
|
|
pointer = next;
|
|
}
|
|
if (pointer->Opcode() != IrOpcode::GetAddressResource || pointer->ArgumentCount() != 2u) {
|
|
return false;
|
|
}
|
|
IrValue* bufferHandle = nullptr;
|
|
IrValue* offset = nullptr;
|
|
std::uint32_t first = 0;
|
|
for (std::uint32_t dword = 0; dword < 2u; dword++) {
|
|
IrValue* read = pointer->Argument(dword)->Resolve();
|
|
std::uint32_t memoryIndex = 0;
|
|
const MemoryInfo* memory = ScalarReadMemory(*read, memoryIndex);
|
|
if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) {
|
|
return false;
|
|
}
|
|
if (dword == 0u) {
|
|
first = memory->offset;
|
|
bufferHandle = read->Argument(0)->Resolve();
|
|
offset = read->Argument(1)->Resolve();
|
|
} else if (memory->offset != first + static_cast<std::uint32_t>(sizeof(std::uint32_t)) || read->Argument(0)->Resolve() != bufferHandle || !EquivalentValue(m_program.Resources(), offset, read->Argument(1))) {
|
|
return false;
|
|
}
|
|
}
|
|
IrValue* key = nullptr;
|
|
std::uint32_t stride = 0;
|
|
std::uint32_t addend = 0;
|
|
if (!MatchAffineOffset(offset, key, stride, addend)) {
|
|
return false;
|
|
}
|
|
DescriptorSource heapSource;
|
|
std::uint32_t heapIndex = 0;
|
|
if (!MakeRuntimeSource(*bufferHandle, IrOpcode::GetBufferResource, 4u, heapIndex, heapSource)) {
|
|
return false;
|
|
}
|
|
TableColumn column;
|
|
column.heapSource = heapIndex;
|
|
column.stride = stride;
|
|
column.addend = addend;
|
|
column.offset = first;
|
|
column.dwordCount = words;
|
|
column.maxKey = possibleU32Bits(key);
|
|
column.sampler = sampler;
|
|
column.indirect = true;
|
|
column.indirectOffset = descriptorOffset;
|
|
column.indirectImmediate = descriptorImmediate;
|
|
column.pointerHops = std::move(hops);
|
|
const auto domain = MatchKeyDomain(key);
|
|
DescriptorSource columnSource = heapSource;
|
|
columnSource.tableColumn = column;
|
|
plan.handle = &handle;
|
|
plan.source = InternSource(columnSource);
|
|
plan.keyDomain = domain;
|
|
auto& shared = *m_sources[plan.source].tableColumn;
|
|
shared.maxKey = std::max(shared.maxKey, column.maxKey);
|
|
if (!domain.has_value()) {
|
|
shared.unkeyed = true;
|
|
shared.keyDomains.clear();
|
|
} else if (!shared.unkeyed && std::find(shared.keyDomains.begin(), shared.keyDomains.end(), *domain) == shared.keyDomains.end()) {
|
|
shared.keyDomains.push_back(*domain);
|
|
}
|
|
plan.offset = offset;
|
|
return true;
|
|
}
|
|
|
|
const TableColumnPlan* FindTableColumn(const IrValue& handle) const {
|
|
const auto found = std::ranges::find_if(m_tableColumns, [&](const TableColumnPlan& plan) {
|
|
return plan.handle == &handle;
|
|
});
|
|
return found == m_tableColumns.end() ? nullptr : &*found;
|
|
}
|
|
|
|
bool PlanTableColumn(IrValue& handle, bool sampler, bool r128) {
|
|
if (FindTableColumn(handle) != nullptr) {
|
|
return true;
|
|
}
|
|
TableColumnPlan plan;
|
|
if (!TryMakeTableColumn(handle, sampler, r128, plan)) {
|
|
return false;
|
|
}
|
|
m_tableColumns.push_back(std::move(plan));
|
|
return true;
|
|
}
|
|
|
|
static constexpr std::uint32_t phiSearchDepth = 8u;
|
|
|
|
static bool SplittableImageRead(const IrValue& inst) {
|
|
switch (inst.Opcode()) {
|
|
case IrOpcode::ImageSampleRaw:
|
|
case IrOpcode::ImageGatherRaw:
|
|
case IrOpcode::ImageQueryLod:
|
|
case IrOpcode::ImageQueryDimensions:
|
|
case IrOpcode::ImageRead: return inst.Type() == IrType::U32x4;
|
|
default: return false;
|
|
}
|
|
}
|
|
|
|
void CollectDescriptorPhis(IrValue* value, std::vector<IrValue*>& phis, std::uint32_t depth) const {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return;
|
|
}
|
|
if (value->IsPhi()) {
|
|
if (ResolveInvariantPhi(m_program.Resources(), value) == nullptr && std::ranges::find(phis, value) == phis.end()) {
|
|
phis.push_back(value);
|
|
}
|
|
return;
|
|
}
|
|
if (depth != 0u && value->Opcode() == IrOpcode::ReadFirstLane && value->ArgumentCount() == 2u) {
|
|
CollectDescriptorPhis(value->Argument(0), phis, depth - 1u);
|
|
return;
|
|
}
|
|
if (depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) {
|
|
return;
|
|
}
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
CollectDescriptorPhis(value->Argument(index), phis, depth - 1u);
|
|
}
|
|
}
|
|
|
|
static IrValue* PhiIncoming(const IrValue& phi, const IrBlock* predecessor) {
|
|
for (std::size_t index = 0; index < phi.ArgumentCount(); index++) {
|
|
if (phi.PhiBlock(index) == predecessor) {
|
|
return phi.Argument(index);
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
IrValue* EmitBefore(IrValue& position, IrOpcode op, IrType type, std::span<IrValue* const> arguments, std::uint64_t flags = 0) {
|
|
IrValue& value = m_program.CreateValue(op, type, flags);
|
|
for (IrValue* argument : arguments) {
|
|
value.AddArgument(argument);
|
|
}
|
|
position.Parent()->InsertInstructionBefore(&position, &value);
|
|
return &value;
|
|
}
|
|
|
|
IrValue* EmitCopy(const IrValue& original, std::span<IrValue* const> arguments, IrValue& position) {
|
|
return EmitBefore(position, original.Opcode(), original.Type(), arguments, original.Flags<std::uint64_t>());
|
|
}
|
|
|
|
IrValue* Rematerialize(IrValue* value, IrValue& position, bool emit, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->HasImmediate()) {
|
|
return value;
|
|
}
|
|
if (value->IsPhi()) {
|
|
IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value);
|
|
return invariant == nullptr || invariant->IsPhi() ? nullptr : Rematerialize(invariant, position, emit, depth);
|
|
}
|
|
if (depth == 0u) {
|
|
return nullptr;
|
|
}
|
|
const auto op = value->Opcode();
|
|
const bool register_ = op == IrOpcode::GetUserData;
|
|
const bool argumentless = (op == IrOpcode::GetShaderBase || op == IrOpcode::GetSrtResource) && value->ArgumentCount() == 0u;
|
|
const bool srtRead = op == IrOpcode::ReadConst && value->ArgumentCount() == 2u && value->Argument(0)->Resolve()->Opcode() == IrOpcode::GetSrtResource && value->Argument(1)->Resolve()->HasImmediate();
|
|
if (!register_ && !argumentless && !srtRead && !Detail::IsRuntimeUniformOp(op)) {
|
|
return nullptr;
|
|
}
|
|
std::vector<IrValue*> arguments;
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
IrValue* argument = register_ ? value->Argument(index) : Rematerialize(value->Argument(index), position, emit, depth - 1u);
|
|
if (argument == nullptr) {
|
|
return nullptr;
|
|
}
|
|
arguments.push_back(argument);
|
|
}
|
|
return emit ? EmitCopy(*value, arguments, position) : value;
|
|
}
|
|
|
|
IrValue* SubstituteEdge(IrValue* value, const IrBlock* predecessor, IrValue& position, bool emit, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
std::vector<IrValue*> phis;
|
|
CollectDescriptorPhis(value, phis, depth);
|
|
if (phis.empty()) {
|
|
return value;
|
|
}
|
|
if (value->IsPhi()) {
|
|
IrValue* incoming = PhiIncoming(*value, predecessor);
|
|
return incoming == nullptr ? nullptr : Rematerialize(incoming, position, emit, phiSearchDepth);
|
|
}
|
|
std::vector<IrValue*> arguments;
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
IrValue* argument = SubstituteEdge(value->Argument(index), predecessor, position, emit, depth - 1u);
|
|
if (argument == nullptr) {
|
|
return nullptr;
|
|
}
|
|
arguments.push_back(argument);
|
|
}
|
|
return emit ? EmitCopy(*value, arguments, position) : value;
|
|
}
|
|
|
|
IrValue& EdgeSelector(IrBlock& block, const std::vector<bool>& edges) {
|
|
for (const auto& selector : m_edgeSelectors) {
|
|
if (selector.block == &block && selector.edges == edges) {
|
|
return *selector.value;
|
|
}
|
|
}
|
|
IrValue& phi = m_program.CreateValue(IrOpcode::Phi, IrType::Bool);
|
|
const auto& predecessors = block.Predecessors();
|
|
for (std::size_t index = 0; index < predecessors.size(); index++) {
|
|
phi.AddPhiOperand(predecessors[index], &m_builder.ConstantBool(edges[index]));
|
|
}
|
|
block.InsertInstructionBefore(nullptr, &phi);
|
|
m_edgeSelectors.push_back({&block, edges, &phi});
|
|
return phi;
|
|
}
|
|
|
|
void SplitDescriptorPhi(IrValue& inst) {
|
|
const auto info = ImageOpcodeInfoOf(inst.Opcode());
|
|
const std::uint32_t handleCount = info.needsSampler ? 2u : 1u;
|
|
if (inst.ArgumentCount() < handleCount || inst.Parent() == nullptr) {
|
|
return;
|
|
}
|
|
std::array<IrValue*, 2> handles {};
|
|
std::array<bool, 2> split {};
|
|
std::vector<IrValue*> phis;
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
IrValue* handle = inst.Argument(slot)->Resolve();
|
|
const auto expected = slot == 0u ? IrOpcode::GetImageResource : IrOpcode::GetSamplerResource;
|
|
if (handle->Opcode() != expected || handle->ArgumentCount() != (slot == 0u ? 8u : 4u)) {
|
|
return;
|
|
}
|
|
handles[slot] = handle;
|
|
std::vector<IrValue*> handlePhis;
|
|
for (std::size_t dword = 0; dword < handle->ArgumentCount(); dword++) {
|
|
CollectDescriptorPhis(handle->Argument(dword), handlePhis, phiSearchDepth);
|
|
}
|
|
split[slot] = !handlePhis.empty();
|
|
for (IrValue* phi : handlePhis) {
|
|
if (std::ranges::find(phis, phi) == phis.end()) {
|
|
phis.push_back(phi);
|
|
}
|
|
}
|
|
}
|
|
if (phis.empty()) {
|
|
return;
|
|
}
|
|
if (!SplitDescriptorEdges(inst, handles, split, handleCount, phis)) {
|
|
SplitDescriptorWeb(inst, handles, split, handleCount);
|
|
}
|
|
}
|
|
|
|
bool SplitDescriptorEdges(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount, const std::vector<IrValue*>& phis) {
|
|
IrBlock* block = phis.front()->Parent();
|
|
if (block == nullptr) {
|
|
return false;
|
|
}
|
|
const auto& predecessors = block->Predecessors();
|
|
if (predecessors.size() < 2u) {
|
|
return false;
|
|
}
|
|
for (const IrValue* phi : phis) {
|
|
if (phi->Parent() != block || phi->PhiBlockCount() != predecessors.size()) {
|
|
return false;
|
|
}
|
|
for (const IrBlock* predecessor : predecessors) {
|
|
if (PhiIncoming(*phi, predecessor) == nullptr) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<std::uint32_t> edgeArm(predecessors.size());
|
|
std::vector<const IrBlock*> arms;
|
|
for (std::size_t edge = 0; edge < predecessors.size(); edge++) {
|
|
std::uint32_t arm = 0;
|
|
for (; arm < arms.size(); arm++) {
|
|
const bool same = std::ranges::all_of(phis, [&](const IrValue* phi) {
|
|
return EquivalentValue(m_program.Resources(), PhiIncoming(*phi, predecessors[edge]), PhiIncoming(*phi, arms[arm]));
|
|
});
|
|
if (same) {
|
|
break;
|
|
}
|
|
}
|
|
if (arm == arms.size()) {
|
|
arms.push_back(predecessors[edge]);
|
|
}
|
|
edgeArm[edge] = arm;
|
|
}
|
|
for (const IrBlock* arm : arms) {
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) {
|
|
if (SubstituteEdge(handles[slot]->Argument(dword), arm, inst, false, phiSearchDepth) == nullptr) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<IrValue*> conditions;
|
|
for (std::uint32_t arm = 0; arm + 1u < arms.size(); arm++) {
|
|
std::vector<bool> edges(edgeArm.size());
|
|
for (std::size_t edge = 0; edge < edgeArm.size(); edge++) {
|
|
edges[edge] = edgeArm[edge] == arm;
|
|
}
|
|
conditions.push_back(&EdgeSelector(*block, edges));
|
|
}
|
|
ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) {
|
|
return SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth);
|
|
});
|
|
return true;
|
|
}
|
|
|
|
template <typename Dword>
|
|
void ReplaceWithArmCopies(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount, std::span<IrValue* const> conditions, Dword&& dword) {
|
|
std::vector<IrValue*> copies;
|
|
const auto flags = inst.Flags<MemoryFlags>();
|
|
const auto memory = m_program.Resources().memoryInfo.at(flags.index);
|
|
for (std::uint32_t arm = 0; arm <= conditions.size(); arm++) {
|
|
std::vector<IrValue*> arguments(inst.Arguments().begin(), inst.Arguments().end());
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
if (!split[slot]) {
|
|
continue;
|
|
}
|
|
std::vector<IrValue*> dwords;
|
|
for (std::size_t index = 0; index < handles[slot]->ArgumentCount(); index++) {
|
|
dwords.push_back(dword(arm, slot, index));
|
|
}
|
|
arguments[slot] = EmitCopy(*handles[slot], dwords, inst);
|
|
}
|
|
auto copyFlags = flags;
|
|
if (arm != 0u) {
|
|
copyFlags.index = static_cast<std::uint32_t>(m_program.Resources().memoryInfo.size());
|
|
m_program.Resources().memoryInfo.push_back(memory);
|
|
}
|
|
std::uint64_t rawFlags = 0;
|
|
std::memcpy(&rawFlags, ©Flags, sizeof(copyFlags));
|
|
copies.push_back(EmitBefore(inst, inst.Opcode(), inst.Type(), arguments, rawFlags));
|
|
}
|
|
|
|
IrValue* result = copies.back();
|
|
if (copies.size() > 1u) {
|
|
std::array<IrValue*, 4> components {};
|
|
for (std::uint32_t component = 0; component < components.size(); component++) {
|
|
const auto extract = [&](IrValue* vector) {
|
|
const std::array<IrValue*, 2> arguments {vector, &m_builder.Constant(component)};
|
|
return EmitBefore(inst, IrOpcode::CompositeExtractU32x4, IrType::U32, arguments);
|
|
};
|
|
IrValue* selected = extract(copies.back());
|
|
for (std::uint32_t arm = static_cast<std::uint32_t>(copies.size()) - 1u; arm-- > 0u;) {
|
|
const std::array<IrValue*, 3> arguments {conditions[arm], extract(copies[arm]), selected};
|
|
selected = EmitBefore(inst, IrOpcode::SelectU32, IrType::U32, arguments);
|
|
}
|
|
components[component] = selected;
|
|
}
|
|
result = EmitBefore(inst, IrOpcode::CompositeConstructU32x4, IrType::U32x4, components);
|
|
}
|
|
inst.ReplaceAllUsesWith(result);
|
|
inst.Invalidate();
|
|
inst.Parent()->RemoveInstruction(&inst);
|
|
}
|
|
|
|
static bool HoldsProgramCounter(IrValue* value, std::uint32_t depth) {
|
|
value = value->Resolve();
|
|
if (value->Opcode() == IrOpcode::GetShaderBase) {
|
|
return true;
|
|
}
|
|
if (value->HasImmediate() || depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) {
|
|
return false;
|
|
}
|
|
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
|
if (HoldsProgramCounter(value->Argument(index), depth - 1u)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
struct DescriptorWeb {
|
|
static constexpr std::size_t maxStates = 512u;
|
|
static constexpr std::size_t maxArms = 64u;
|
|
std::vector<std::vector<IrValue*>> arms;
|
|
std::map<std::vector<IrValue*>, IrValue*> selectors;
|
|
std::vector<std::pair<IrBlock*, IrValue*>> phis;
|
|
std::vector<std::tuple<IrBlock*, IrValue*, IrValue*>> lanes;
|
|
std::unordered_map<IrValue*, IrValue*> resolved;
|
|
std::unordered_map<const IrBlock*, std::size_t> order;
|
|
};
|
|
|
|
IrValue* WebSelector(std::vector<IrValue*> tuple, DescriptorWeb& web, IrValue& position) {
|
|
for (IrValue*& value : tuple) {
|
|
value = value->Resolve();
|
|
if (!value->IsPhi()) {
|
|
continue;
|
|
}
|
|
const auto [entry, inserted] = web.resolved.try_emplace(value, value);
|
|
if (inserted) {
|
|
IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value);
|
|
entry->second = invariant == nullptr ? value : invariant->Resolve();
|
|
}
|
|
value = entry->second;
|
|
}
|
|
if (const auto known = web.selectors.find(tuple); known != web.selectors.end()) {
|
|
return known->second;
|
|
}
|
|
const auto phi = std::ranges::find_if(tuple, [](const IrValue* value) { return value->IsPhi(); });
|
|
const auto lane = std::ranges::find_if(tuple, [](const IrValue* value) { return value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::ReadFirstLane; });
|
|
if (phi == tuple.end() && lane != tuple.end()) {
|
|
return LaneSelector(tuple, **lane, web, position);
|
|
}
|
|
if (phi == tuple.end()) {
|
|
for (IrValue* value : tuple) {
|
|
if (HoldsProgramCounter(value, phiSearchDepth) || Rematerialize(value, position, false, phiSearchDepth) == nullptr) {
|
|
return nullptr;
|
|
}
|
|
}
|
|
const auto same = std::ranges::find_if(web.arms, [&](const std::vector<IrValue*>& arm) {
|
|
return std::ranges::equal(tuple, arm, [&](IrValue* left, IrValue* right) { return EquivalentValue(m_program.Resources(), left, right); });
|
|
});
|
|
if (same != web.arms.end()) {
|
|
return &m_builder.Constant(static_cast<std::uint32_t>(same - web.arms.begin()));
|
|
}
|
|
if (web.arms.size() == DescriptorWeb::maxArms) {
|
|
return nullptr;
|
|
}
|
|
web.arms.push_back(std::move(tuple));
|
|
return &m_builder.Constant(static_cast<std::uint32_t>(web.arms.size() - 1u));
|
|
}
|
|
if (web.order.empty()) {
|
|
const auto& order = m_program.BlockOrder();
|
|
for (std::size_t index = 0; index < order.size(); index++) {
|
|
web.order.emplace(order[index], index);
|
|
}
|
|
}
|
|
IrBlock* block = nullptr;
|
|
std::size_t latest = 0;
|
|
for (const IrValue* value : tuple) {
|
|
if (!value->IsPhi()) {
|
|
continue;
|
|
}
|
|
const auto found = web.order.find(value->Parent());
|
|
if (found == web.order.end()) {
|
|
return nullptr;
|
|
}
|
|
if (block == nullptr || found->second > latest) {
|
|
block = value->Parent();
|
|
latest = found->second;
|
|
}
|
|
}
|
|
if (block == nullptr || web.selectors.size() == DescriptorWeb::maxStates) {
|
|
return nullptr;
|
|
}
|
|
const auto& predecessors = block->Predecessors();
|
|
if (predecessors.size() < 2u) {
|
|
return nullptr;
|
|
}
|
|
for (const IrValue* value : tuple) {
|
|
if (value->IsPhi() && value->Parent() == block && value->PhiBlockCount() != predecessors.size()) {
|
|
return nullptr;
|
|
}
|
|
}
|
|
IrValue& selector = m_program.CreateValue(IrOpcode::Phi, IrType::U32);
|
|
web.selectors.emplace(tuple, &selector);
|
|
web.phis.emplace_back(block, &selector);
|
|
for (IrBlock* predecessor : predecessors) {
|
|
std::vector<IrValue*> incoming;
|
|
for (IrValue* value : tuple) {
|
|
IrValue* edge = value->IsPhi() && value->Parent() == block ? PhiIncoming(*value, predecessor) : value;
|
|
if (edge == nullptr) {
|
|
return nullptr;
|
|
}
|
|
incoming.push_back(edge);
|
|
}
|
|
IrValue* edgeSelector = WebSelector(std::move(incoming), web, position);
|
|
if (edgeSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
selector.AddPhiOperand(predecessor, edgeSelector);
|
|
}
|
|
return &selector;
|
|
}
|
|
|
|
IrValue* LaneSelector(const std::vector<IrValue*>& tuple, IrValue& first, DescriptorWeb& web, IrValue& position) {
|
|
IrBlock* block = first.Parent();
|
|
const auto op = first.Opcode();
|
|
const bool select = op == IrOpcode::SelectU32;
|
|
if (block == nullptr || first.ArgumentCount() != (select ? 3u : 2u) || web.selectors.size() == DescriptorWeb::maxStates) {
|
|
return nullptr;
|
|
}
|
|
IrValue* shared = first.Argument(select ? 0u : 1u);
|
|
std::vector<IrValue*> left;
|
|
std::vector<IrValue*> right;
|
|
for (IrValue* value : tuple) {
|
|
if (value->Opcode() == op && value->Parent() == block && value->ArgumentCount() == first.ArgumentCount() && EquivalentValue(m_program.Resources(), value->Argument(select ? 0u : 1u), shared)) {
|
|
left.push_back(value->Argument(select ? 1u : 0u));
|
|
if (select) {
|
|
right.push_back(value->Argument(2u));
|
|
}
|
|
} else if (value->HasImmediate() || (select && !value->IsPhi() && value->Opcode() != IrOpcode::ReadFirstLane)) {
|
|
left.push_back(value);
|
|
if (select) {
|
|
right.push_back(value);
|
|
}
|
|
} else {
|
|
return nullptr;
|
|
}
|
|
}
|
|
auto& instructions = block->Instructions();
|
|
auto latest = instructions.end();
|
|
for (auto it = instructions.begin(); it != instructions.end(); ++it) {
|
|
if (std::ranges::find(tuple, *it) != tuple.end()) {
|
|
latest = it;
|
|
}
|
|
}
|
|
if (latest == instructions.end()) {
|
|
return nullptr;
|
|
}
|
|
IrValue* before = std::next(latest) == instructions.end() ? nullptr : *std::next(latest);
|
|
IrValue* leftSelector = WebSelector(std::move(left), web, position);
|
|
if (leftSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
IrValue& node = m_program.CreateValue(op, IrType::U32, first.Flags<std::uint64_t>());
|
|
if (select) {
|
|
IrValue* rightSelector = WebSelector(std::move(right), web, position);
|
|
if (rightSelector == nullptr) {
|
|
return nullptr;
|
|
}
|
|
node.AddArgument(shared);
|
|
node.AddArgument(leftSelector);
|
|
node.AddArgument(rightSelector);
|
|
} else {
|
|
node.AddArgument(leftSelector);
|
|
node.AddArgument(shared);
|
|
}
|
|
web.selectors.emplace(tuple, &node);
|
|
web.lanes.emplace_back(block, before, &node);
|
|
return &node;
|
|
}
|
|
|
|
void SplitDescriptorWeb(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount) {
|
|
std::vector<IrValue*> tuple;
|
|
std::array<std::size_t, 2> offsets {};
|
|
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
|
offsets[slot] = tuple.size();
|
|
for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) {
|
|
tuple.push_back(handles[slot]->Argument(dword));
|
|
}
|
|
}
|
|
DescriptorWeb web;
|
|
IrValue* selector = WebSelector(tuple, web, inst);
|
|
const bool select = selector != nullptr && web.arms.size() > 1u;
|
|
if (select) {
|
|
for (const auto& [block, phi] : web.phis) {
|
|
block->InsertInstructionBefore(nullptr, phi);
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
if (before == nullptr) {
|
|
block->AppendInstruction(node);
|
|
} else {
|
|
block->InsertInstructionBefore(before, node);
|
|
}
|
|
}
|
|
} else {
|
|
for (const auto& [block, phi] : web.phis) {
|
|
for (std::size_t index = 0; index < phi->ArgumentCount(); index++) {
|
|
phi->ReplaceArgument(index, nullptr);
|
|
}
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
for (std::size_t index = 0; index < node->ArgumentCount(); index++) {
|
|
node->ReplaceArgument(index, nullptr);
|
|
}
|
|
}
|
|
for (const auto& [block, phi] : web.phis) {
|
|
phi->Invalidate();
|
|
}
|
|
for (const auto& [block, before, node] : web.lanes) {
|
|
node->Invalidate();
|
|
}
|
|
}
|
|
if (selector == nullptr || web.arms.empty()) {
|
|
return;
|
|
}
|
|
|
|
std::vector<IrValue*> conditions;
|
|
for (std::uint32_t arm = 0; arm + 1u < web.arms.size(); arm++) {
|
|
const std::array<IrValue*, 2> compare {selector, &m_builder.Constant(arm)};
|
|
conditions.push_back(EmitBefore(inst, IrOpcode::IEqual32, IrType::Bool, compare));
|
|
}
|
|
ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) {
|
|
return Rematerialize(web.arms[arm][offsets[slot] + dword], inst, true, phiSearchDepth);
|
|
});
|
|
}
|
|
|
|
void SplitDescriptorPhis() {
|
|
std::vector<IrValue*> candidates;
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
if (SplittableImageRead(*inst)) {
|
|
candidates.push_back(inst);
|
|
}
|
|
}
|
|
}
|
|
for (IrValue* inst : candidates) {
|
|
SplitDescriptorPhi(*inst);
|
|
}
|
|
}
|
|
|
|
void PlanTableColumns() {
|
|
static const bool enabled = std::getenv("APS5_NO_IMAGE_TABLES") == nullptr;
|
|
if (!enabled) {
|
|
return;
|
|
}
|
|
const auto sources = m_sources;
|
|
const auto columns = m_tableColumns;
|
|
const auto reads = m_tablePlanningReads;
|
|
const auto memory = m_tablePlanningMemory;
|
|
PlanTableColumnPass();
|
|
while (!PlannedWithinLimits()) {
|
|
const auto group = LargestSelectedGroup();
|
|
if (group.empty()) {
|
|
return;
|
|
}
|
|
m_unselectedHandles.insert(m_unselectedHandles.end(), group.begin(), group.end());
|
|
m_sources = sources;
|
|
m_tableColumns = columns;
|
|
m_tablePlanningReads = reads;
|
|
m_tablePlanningMemory = memory;
|
|
PlanTableColumnPass();
|
|
}
|
|
}
|
|
|
|
bool PlannedWithinLimits() {
|
|
std::vector<ImageResource> images;
|
|
std::vector<std::pair<std::uint32_t, std::uint32_t>> pairs;
|
|
std::vector<const IrValue*> directImages;
|
|
std::vector<std::pair<const IrValue*, bool>> directSamplers;
|
|
const auto direct = std::numeric_limits<std::uint32_t>::max() / 2u;
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode());
|
|
if (inst->Opcode() == IrOpcode::ImageBvhIntersectRay || imageInfo.access == ImageAccess::None || imageInfo.resourceClass == ImageResourceClass::None || inst->ArgumentCount() == 0u) {
|
|
continue;
|
|
}
|
|
const auto flags = inst->Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
continue;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[flags.index];
|
|
if (memory.kind != ResourceKind::Image || memory.planningOnly || IsTablePlanningMemory(flags.index)) {
|
|
continue;
|
|
}
|
|
const IrValue* handle = inst->Argument(0)->Resolve();
|
|
const TableColumnPlan* column = FindTableColumn(*handle);
|
|
std::uint32_t source = 0;
|
|
if (column != nullptr) {
|
|
source = column->source;
|
|
} else {
|
|
const auto found = std::ranges::find_if(directImages, [&](const IrValue* other) { return SameHandle(*other, *handle); });
|
|
source = direct + static_cast<std::uint32_t>(found - directImages.begin());
|
|
if (found == directImages.end()) directImages.push_back(handle);
|
|
}
|
|
const auto key = ImageKey(source, memory, inst->Opcode());
|
|
const auto image = std::ranges::find_if(images, [&](const ImageResource& other) { return SameImageKey(other, key); });
|
|
const auto imageIndex = static_cast<std::uint32_t>(image - images.begin());
|
|
if (image == images.end()) images.push_back(key);
|
|
if (!imageInfo.needsSampler || inst->ArgumentCount() < 2u) {
|
|
continue;
|
|
}
|
|
const IrValue* samplerHandle = inst->Argument(1)->Resolve();
|
|
const TableColumnPlan* samplerColumn = FindTableColumn(*samplerHandle);
|
|
std::uint32_t sampler = 0;
|
|
if (samplerColumn != nullptr) {
|
|
sampler = samplerColumn->source;
|
|
} else {
|
|
const bool sampleAdjust = (memory.imageSampleFlags & RdnaImageSampleFlagAdjust) != 0;
|
|
const auto found = std::ranges::find_if(directSamplers, [&](const auto& other) { return other.second == sampleAdjust && SameHandle(*other.first, *samplerHandle); });
|
|
sampler = direct + static_cast<std::uint32_t>(found - directSamplers.begin());
|
|
if (found == directSamplers.end()) directSamplers.emplace_back(samplerHandle, sampleAdjust);
|
|
}
|
|
if (std::ranges::find(pairs, std::pair{imageIndex, sampler}) == pairs.end()) pairs.emplace_back(imageIndex, sampler);
|
|
}
|
|
}
|
|
return images.size() <= ShaderInfo::MaxImages && pairs.size() <= ShaderInfo::MaxSampledPairs;
|
|
}
|
|
|
|
std::vector<const IrValue*> LargestSelectedGroup() const {
|
|
const auto unselected = [](TableColumn column) {
|
|
column.selected = false;
|
|
column.selection = 0u;
|
|
return column;
|
|
};
|
|
std::vector<std::vector<std::uint32_t>> groups;
|
|
for (const auto& plan : m_tableColumns) {
|
|
const auto& column = m_sources[plan.source].tableColumn;
|
|
if (!column.has_value()) {
|
|
continue;
|
|
}
|
|
const auto group = std::ranges::find_if(groups, [&](const std::vector<std::uint32_t>& members) { return unselected(*m_sources[members.front()].tableColumn) == unselected(*column); });
|
|
if (group == groups.end()) {
|
|
groups.push_back({plan.source});
|
|
} else if (std::ranges::find(*group, plan.source) == group->end()) {
|
|
group->push_back(plan.source);
|
|
}
|
|
}
|
|
const std::vector<std::uint32_t>* largest = nullptr;
|
|
for (const auto& group : groups) {
|
|
const bool selected = std::ranges::any_of(group, [&](std::uint32_t source) { return m_sources[source].tableColumn->selected; });
|
|
if (selected && group.size() > 1u && (largest == nullptr || group.size() > largest->size())) {
|
|
largest = &group;
|
|
}
|
|
}
|
|
std::vector<const IrValue*> handles;
|
|
if (largest == nullptr) {
|
|
return handles;
|
|
}
|
|
for (const auto& plan : m_tableColumns) {
|
|
if (std::ranges::find(*largest, plan.source) != largest->end()) {
|
|
handles.push_back(plan.handle);
|
|
}
|
|
}
|
|
return handles;
|
|
}
|
|
|
|
void PlanTableColumnPass() {
|
|
IndexMemoryUsers();
|
|
const auto directImages = DirectImageHandles();
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode());
|
|
if (imageInfo.access == ImageAccess::None || inst->ArgumentCount() == 0u) {
|
|
continue;
|
|
}
|
|
const auto flags = inst->Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
continue;
|
|
}
|
|
const auto memory = m_program.Resources().memoryInfo[flags.index];
|
|
IrValue& image = *inst->Argument(0)->Resolve();
|
|
if (std::ranges::none_of(directImages, [&](const IrValue* handle) { return SameHandle(*handle, image); })) {
|
|
static_cast<void>(PlanTableColumn(image, false, memory.imageR128));
|
|
}
|
|
if (imageInfo.needsSampler && inst->ArgumentCount() > 1u && !DirectSampler(*inst->Argument(1)->Resolve())) {
|
|
static_cast<void>(PlanTableColumn(*inst->Argument(1)->Resolve(), true, false));
|
|
}
|
|
}
|
|
}
|
|
for (const auto& plan : m_tableColumns) {
|
|
for (std::size_t index = 0; index < plan.reads.size(); index++) {
|
|
const IrValue* read = plan.reads[index];
|
|
const bool planned = std::ranges::all_of(read->Uses(), [&](const IrValue* user) {
|
|
return FindTableColumn(*user) != nullptr;
|
|
});
|
|
if (planned && std::ranges::find(m_tablePlanningReads, read) == m_tablePlanningReads.end()) {
|
|
m_tablePlanningReads.push_back(read);
|
|
m_tablePlanningMemory.push_back(plan.memory[index]);
|
|
}
|
|
}
|
|
}
|
|
for (bool marked = true; marked;) {
|
|
marked = false;
|
|
for (const auto& plan : m_tableColumns) {
|
|
for (std::size_t index = 0; index < plan.hopReads.size(); index++) {
|
|
const IrValue* read = plan.hopReads[index];
|
|
if (std::ranges::find(m_tablePlanningReads, read) != m_tablePlanningReads.end()) {
|
|
continue;
|
|
}
|
|
const bool planned = std::ranges::all_of(read->Uses(), [&](const IrValue* pointer) {
|
|
return pointer->Opcode() == IrOpcode::GetAddressResource && std::ranges::all_of(pointer->Uses(), [&](const IrValue* user) {
|
|
return std::ranges::find(m_tablePlanningReads, user) != m_tablePlanningReads.end();
|
|
});
|
|
});
|
|
if (planned) {
|
|
m_tablePlanningReads.push_back(read);
|
|
m_tablePlanningMemory.push_back(plan.hopMemory[index]);
|
|
marked = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
m_memoryUsersIndexed = false;
|
|
m_memoryUsers.clear();
|
|
}
|
|
|
|
void KeepSourceAlive(IrValue& position, const DescriptorSource& source, std::vector<const IrValue*>& referenced) {
|
|
for (std::uint32_t dword = 0; dword < source.dwordCount; dword++) {
|
|
IrValue* value = source.dwords[dword]->Resolve();
|
|
if (value->HasImmediate() || std::ranges::find(referenced, value) != referenced.end()) {
|
|
continue;
|
|
}
|
|
referenced.push_back(value);
|
|
const std::array<IrValue*, 1> arguments{value};
|
|
EmitBefore(position, IrOpcode::ReferenceU32, IrType::Void, arguments);
|
|
}
|
|
}
|
|
|
|
bool DirectSampler(const IrValue& handle) {
|
|
if (handle.Opcode() != IrOpcode::GetSamplerResource || handle.ArgumentCount() != 4u || FindTableColumn(handle) != nullptr) {
|
|
return false;
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(handle, 4u, true, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
return ValidateSource(descriptor, badDword);
|
|
}
|
|
|
|
bool DirectImage(const IrValue& handle) {
|
|
if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u) {
|
|
return false;
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(handle, 8u, false, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
return ValidateSource(descriptor, badDword);
|
|
}
|
|
|
|
std::vector<const IrValue*> DirectImageHandles() {
|
|
std::vector<const IrValue*> handles;
|
|
for (auto& block : m_program.Blocks()) {
|
|
for (IrValue* inst : block->Instructions()) {
|
|
const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode());
|
|
if (imageInfo.access == ImageAccess::None || inst->ArgumentCount() == 0u) {
|
|
continue;
|
|
}
|
|
const auto flags = inst->Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
continue;
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[flags.index];
|
|
const bool tableUnsupported = imageInfo.resourceClass == ImageResourceClass::Storage || memory.imagePacked || memory.imageByElements != 0u;
|
|
const IrValue* handle = inst->Argument(0)->Resolve();
|
|
if (tableUnsupported && DirectImage(*handle)) {
|
|
handles.push_back(handle);
|
|
}
|
|
}
|
|
}
|
|
return handles;
|
|
}
|
|
|
|
bool SameHandle(const IrValue& left, const IrValue& right) const {
|
|
if (&left == &right) {
|
|
return true;
|
|
}
|
|
if (left.Opcode() != right.Opcode() || left.ArgumentCount() != right.ArgumentCount()) {
|
|
return false;
|
|
}
|
|
for (std::size_t index = 0; index < left.ArgumentCount(); index++) {
|
|
if (!EquivalentValue(m_program.Resources(), left.Argument(index), right.Argument(index))) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool IsTablePlanningMemory(std::uint32_t index) const {
|
|
return std::ranges::find(m_tablePlanningMemory, index) != m_tablePlanningMemory.end();
|
|
}
|
|
|
|
bool TableSource(std::uint32_t source) const {
|
|
return source < m_sources.size() && m_sources[source].tableColumn.has_value();
|
|
}
|
|
|
|
bool SameBufferSource(const IrValue& handle, std::uint32_t source) const {
|
|
const auto& current = m_sources[source];
|
|
if (current.tableColumn.has_value() || current.dwordCount != 4u || handle.ArgumentCount() != 4u) {
|
|
return false;
|
|
}
|
|
for (std::uint32_t dword = 0; dword < 4u; dword++) {
|
|
if (!EquivalentValue(m_program.Resources(), current.dwords[dword], handle.Argument(dword))) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool SelectionWritten(const TableColumn& column) const {
|
|
const auto written = [&](const auto& matches) {
|
|
return std::any_of(m_info.buffers.begin(), m_info.buffers.end(), [&](const auto& buffer) { return (buffer.written || buffer.atomic) && matches(buffer.source); });
|
|
};
|
|
std::vector<const IrValue*> pending{m_sources[column.selection].dwords[0]};
|
|
std::vector<const IrValue*> seen;
|
|
while (!pending.empty()) {
|
|
const IrValue* current = pending.back()->Resolve();
|
|
pending.pop_back();
|
|
if (std::ranges::find(seen, current) != seen.end()) {
|
|
continue;
|
|
}
|
|
seen.push_back(current);
|
|
const bool pointer = current->Opcode() == IrOpcode::LoadAddressU32;
|
|
const bool buffer = current->Opcode() == IrOpcode::ReadConstBuffer && current->ArgumentCount() != 0u;
|
|
if ((pointer || buffer) && m_info.bdaWrites) {
|
|
return true;
|
|
}
|
|
if (pointer && written([](std::uint32_t) { return true; })) {
|
|
return true;
|
|
}
|
|
if (buffer) {
|
|
const IrValue* handle = current->Argument(0)->Resolve();
|
|
if (written([&](std::uint32_t source) { return SameBufferSource(*handle, source); })) {
|
|
return true;
|
|
}
|
|
}
|
|
for (std::size_t index = 0; index < current->ArgumentCount(); index++) {
|
|
pending.push_back(current->Argument(index));
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void CheckTableStores() {
|
|
for (auto& source : m_sources) {
|
|
if (source.tableColumn.has_value() && source.tableColumn->selected && SelectionWritten(*source.tableColumn)) {
|
|
source.tableColumn->selected = false;
|
|
source.tableColumn->keyDomains.clear();
|
|
}
|
|
if (!source.tableColumn.has_value() || source.tableColumn->keyDomains.empty()) {
|
|
continue;
|
|
}
|
|
auto& column = *source.tableColumn;
|
|
const bool written = std::any_of(column.keyDomains.begin(), column.keyDomains.end(), [&](const KeyDomain& domain) {
|
|
return std::any_of(m_info.buffers.begin(), m_info.buffers.end(), [&](const auto& buffer) { return (buffer.written || buffer.atomic) && buffer.source == domain.source; });
|
|
});
|
|
if (written) {
|
|
column.keyDomains.clear();
|
|
}
|
|
}
|
|
}
|
|
|
|
void AssignTables() {
|
|
std::uint32_t next = 0;
|
|
for (auto& image : m_info.images) {
|
|
if (!TableSource(image.source)) {
|
|
continue;
|
|
}
|
|
image.table = next++;
|
|
if (image.atomic || image.packed || image.byElements != 0u) {
|
|
image.tableOperation = TableOperation::Unsupported;
|
|
}
|
|
}
|
|
for (auto& sampler : m_info.samplers) {
|
|
if (TableSource(sampler.source)) {
|
|
sampler.table = next++;
|
|
}
|
|
}
|
|
}
|
|
|
|
void GetHandle(IrValue* value, IrOpcode expected, std::uint32_t width, IrValue*& handle, std::uint32_t& source, bool sampler = false, bool sampleAdjust = false) {
|
|
handle = value->Resolve();
|
|
if (handle->Opcode() != expected) {
|
|
fail("memory operation requires " + std::string(IrOpcodeName(expected)));
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(*handle, width, sampler, sampleAdjust, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
if (!ValidateSource(descriptor, badDword)) {
|
|
fail(std::string(IrOpcodeName(expected)) + " dword " + std::to_string(badDword) + " is not a valid runtime value; chain: " + describeValueChain(descriptor.dwords[badDword], 8u));
|
|
}
|
|
source = InternSource(descriptor);
|
|
}
|
|
|
|
bool TakeGpuDescriptor(IrValue& inst, std::uint32_t memoryIndex) {
|
|
const IrValue* handle = inst.Argument(0)->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) {
|
|
return false;
|
|
}
|
|
DescriptorSource descriptor;
|
|
MakeSource(*handle, 4u, false, false, descriptor);
|
|
std::uint32_t badDword = 0;
|
|
auto& memory = m_program.Resources().memoryInfo[memoryIndex];
|
|
const auto access = BufferAccessOf(inst.Opcode());
|
|
if (ValidateSource(descriptor, badDword)) {
|
|
const auto source = InternSource(descriptor);
|
|
const auto resource = AddBuffer(source, memory, inst.Opcode(), inst.Flags<MemoryFlags>().pc);
|
|
if (resource == std::numeric_limits<std::uint32_t>::max()) fail("buffer resource limit exceeded");
|
|
AddMemoryPatch(memoryIndex, resource, 0u, false);
|
|
memory.gpuDescriptor = false;
|
|
m_info.usesFaultBuffer = m_info.usesFaultBuffer || access == BufferAccess::Atomic;
|
|
return true;
|
|
}
|
|
memory.gpuDescriptor = true;
|
|
m_info.usesDma = true;
|
|
m_info.bdaWrites = m_info.bdaWrites || access == BufferAccess::Write || access == BufferAccess::Atomic;
|
|
return true;
|
|
}
|
|
|
|
void ValidateAddressHandle(IrValue* value) const {
|
|
const IrValue* handle = value->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetAddressResource) {
|
|
fail("address operation requires GetAddressResource");
|
|
}
|
|
if (handle->ArgumentCount() != 2) {
|
|
fail("GetAddressResource must have two address dwords");
|
|
}
|
|
}
|
|
|
|
std::uint32_t AddBuffer(std::uint32_t source, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
for (std::uint32_t i = 0; i < m_info.buffers.size(); i++) {
|
|
if (m_info.buffers[i].source == source) {
|
|
Merge(m_info.buffers[i], memory, op, pc);
|
|
return i;
|
|
}
|
|
}
|
|
BufferResource resource;
|
|
resource.source = source;
|
|
resource.firstUsePc = pc;
|
|
Merge(resource, memory, op, pc);
|
|
m_info.buffers.push_back(resource);
|
|
return static_cast<std::uint32_t>(m_info.buffers.size() - 1);
|
|
}
|
|
|
|
static void Merge(BufferResource& resource, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto access = BufferAccessOf(op);
|
|
const bool atomic = access == BufferAccess::Atomic;
|
|
const bool write = access == BufferAccess::Write || atomic;
|
|
resource.firstUsePc = std::min(resource.firstUsePc, pc);
|
|
resource.maxByteExtent = std::max(resource.maxByteExtent, byteExtent(memory));
|
|
resource.read = resource.read || !write || atomic;
|
|
resource.written = resource.written || write;
|
|
resource.atomic = resource.atomic || atomic;
|
|
resource.formatted = resource.formatted || memory.formatted;
|
|
resource.descriptorFormatted = resource.descriptorFormatted || (memory.formatted && !memory.typed);
|
|
if (memory.formatted && memory.typed) {
|
|
const auto format = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat));
|
|
if (format.byteSize == 0u) fail("typed buffer instruction has an invalid format");
|
|
resource.typedAlignment = std::max(resource.typedAlignment, static_cast<std::uint8_t>(std::min(format.byteSize, 4u)));
|
|
}
|
|
if (memory.formatted && !memory.typed && !write) resource.formattedReadMask |= (1u << std::min(memory.dataDwords, 4u)) - 1u;
|
|
resource.scalar = resource.scalar || op == IrOpcode::ReadConstBuffer || memory.kind == ResourceKind::ScalarBuffer;
|
|
}
|
|
|
|
static ImageResource ImageKey(std::uint32_t source, const MemoryInfo& memory, IrOpcode op) {
|
|
ImageResource image;
|
|
image.source = source;
|
|
image.resourceClass = ImageOpcodeInfoOf(op).resourceClass;
|
|
image.dimension = memory.imageDimension;
|
|
image.mipMode = image.resourceClass == ImageResourceClass::Storage && memory.imageHasMip ? ImageMipMode::DynamicStorage : ImageMipMode::None;
|
|
image.depthCompare = (memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0;
|
|
image.r128 = memory.imageR128;
|
|
image.packed = memory.imagePacked;
|
|
image.byElements = memory.imageByElements;
|
|
image.byComponents = memory.imageByElements != 0u ? memory.dataDwords / memory.imageByElements : 0u;
|
|
return image;
|
|
}
|
|
|
|
static bool SameImageKey(const ImageResource& image, const ImageResource& key) {
|
|
return image.source == key.source && image.resourceClass == key.resourceClass && image.dimension == key.dimension && image.mipMode == key.mipMode && image.depthCompare == key.depthCompare && image.r128 == key.r128 && image.packed == key.packed && image.byElements == key.byElements && image.byComponents == key.byComponents;
|
|
}
|
|
|
|
std::uint32_t AddImage(std::uint32_t source, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto key = ImageKey(source, memory, op);
|
|
for (std::uint32_t i = 0; i < m_info.images.size(); i++) {
|
|
auto& image = m_info.images[i];
|
|
if (SameImageKey(image, key)) {
|
|
Merge(image, memory, op, pc);
|
|
return i;
|
|
}
|
|
}
|
|
if (m_info.images.size() >= ShaderInfo::MaxImages) {
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
ImageResource image = key;
|
|
image.singleMipPlaneBiasCompatible = m_program.Resources().stage == IrShaderStage::Pixel;
|
|
image.firstUsePc = pc;
|
|
Merge(image, memory, op, pc);
|
|
m_info.images.push_back(image);
|
|
return static_cast<std::uint32_t>(m_info.images.size() - 1);
|
|
}
|
|
|
|
static void Merge(ImageResource& image, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) {
|
|
const auto access = ImageOpcodeInfoOf(op).access;
|
|
const bool atomic = access == ImageAccess::Atomic;
|
|
const bool write = access == ImageAccess::Write || atomic;
|
|
const bool atomic64 = IsImageAtomic64Opcode(op);
|
|
if ((image.read || image.written) && image.atomic64 != atomic64) {
|
|
throw std::runtime_error("an image accessed by 64-bit atomics is also accessed in another way");
|
|
}
|
|
image.atomic64 = atomic64;
|
|
image.firstUsePc = std::min(image.firstUsePc, pc);
|
|
image.read = image.read || !write || atomic;
|
|
image.written = image.written || write;
|
|
image.atomic = image.atomic || atomic;
|
|
image.srgbDecodeCompatible = image.srgbDecodeCompatible && !ImageOpcodeInfoOf(op).needsSampler;
|
|
image.fmaskCompatible = image.fmaskCompatible && op == IrOpcode::ImageRead && memory.dataBits == 32u;
|
|
image.singleMipPlaneBiasCompatible = image.singleMipPlaneBiasCompatible && op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagBias && memory.dataBits == 32u && !memory.imageR128 && !memory.imagePacked && memory.imageByElements == 0u;
|
|
image.depthBitsCompatible = image.depthBitsCompatible && memory.dataBits == 32u;
|
|
image.constantSwizzleCompatible = image.constantSwizzleCompatible && (op == IrOpcode::ImageSampleRaw || op == IrOpcode::ImageGatherRaw);
|
|
const bool cubeFaceLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u;
|
|
const bool cubeFaceSample = op == IrOpcode::ImageSampleRaw && (memory.imageSampleFlags == 0u || memory.imageSampleFlags == RdnaImageSampleFlagLevelZero || memory.imageSampleFlags == RdnaImageSampleFlagLod);
|
|
image.cubeFaceCompatible = image.cubeFaceCompatible && (cubeFaceLoad || cubeFaceSample) && !memory.imagePacked && memory.imageByElements == 0u;
|
|
const bool flatVolumeLoad = op == IrOpcode::ImageRead && memory.imageSampleFlags == 0u;
|
|
const bool flatVolumeSample = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLevelZero;
|
|
const bool flatVolumeQuery = op == IrOpcode::ImageQueryDimensions;
|
|
image.flatVolumeCompatible = image.flatVolumeCompatible && (flatVolumeLoad || flatVolumeSample || flatVolumeQuery) && !memory.imagePacked && memory.imageByElements == 0u;
|
|
const bool flatLineLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u;
|
|
const auto lineSampleFlags = memory.imageSampleFlags & ~RdnaImageSampleFlagCompare;
|
|
const bool flatLineSample = op == IrOpcode::ImageSampleRaw && (lineSampleFlags == RdnaImageSampleFlagLevelZero || lineSampleFlags == 0u);
|
|
const bool flatLineExplicitLod = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLod;
|
|
const bool flatLineLevelZero = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLevelZero;
|
|
const bool flatLineAccess = image.dimension == RdnaImageDimension::Dim2DArray ? flatLineLevelZero || flatLineExplicitLod : flatLineLoad || flatLineSample || flatLineExplicitLod;
|
|
image.flatLineCompatible = image.flatLineCompatible && flatLineAccess && !memory.imagePacked && memory.imageByElements == 0u;
|
|
image.flatVolumeLineCompatible = image.flatVolumeLineCompatible && (flatLineLoad || flatLineSample) && !memory.imagePacked && memory.imageByElements == 0u;
|
|
if ((memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0u) {
|
|
constexpr auto unsupported = RdnaImageSampleFlagLod | RdnaImageSampleFlagDerivative;
|
|
if (op == IrOpcode::ImageGatherRaw || (memory.imageSampleFlags & unsupported) != 0u) image.emulatedCompare |= EmulatedCompare::Unsupported;
|
|
if ((memory.imageSampleFlags & RdnaImageSampleFlagLevelZero) == 0u) image.emulatedCompare |= EmulatedCompare::RequiresSingleLevel;
|
|
} else if (ImageOpcodeInfoOf(op).needsSampler) {
|
|
if (op != IrOpcode::ImageSampleRaw) image.emulatedCompare |= EmulatedCompare::ConvertedSampleUnsupported;
|
|
if ((memory.imageSampleFlags & RdnaImageSampleFlagLevelZero) == 0u) image.emulatedCompare |= EmulatedCompare::ConvertedSampleRequiresSingleLevel;
|
|
}
|
|
}
|
|
|
|
std::uint32_t AddSampler(std::uint32_t source, std::uint32_t pc) {
|
|
for (std::uint32_t i = 0; i < m_info.samplers.size(); i++) {
|
|
if (m_info.samplers[i].source == source) {
|
|
m_info.samplers[i].firstUsePc = std::min(m_info.samplers[i].firstUsePc, pc);
|
|
return i;
|
|
}
|
|
}
|
|
if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) {
|
|
return std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
m_info.samplers.push_back({source, pc});
|
|
return static_cast<std::uint32_t>(m_info.samplers.size() - 1);
|
|
}
|
|
|
|
void AddSampledPair(std::uint32_t image, std::uint32_t sampler, std::uint32_t pc) {
|
|
for (auto& pair : m_info.sampledPairs) {
|
|
if (pair.image == image && pair.sampler == sampler) {
|
|
pair.firstUsePc = std::min(pair.firstUsePc, pc);
|
|
return;
|
|
}
|
|
}
|
|
if (m_info.sampledPairs.size() >= ShaderInfo::MaxSampledPairs) {
|
|
fail("sampled image/sampler pair limit exceeded");
|
|
}
|
|
m_info.sampledPairs.push_back({image, sampler, pc});
|
|
}
|
|
|
|
void AddHandlePatch(IrValue* handle, std::uint32_t resource) {
|
|
for (const auto& patch : m_handlePatches) {
|
|
if (patch.handle == handle) {
|
|
if (patch.resource != resource) {
|
|
fail(std::string(IrOpcodeName(handle->Opcode())) + " is reused with incompatible resource classes");
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
m_handlePatches.push_back({handle, resource});
|
|
}
|
|
|
|
void AddMemoryPatch(std::uint32_t index, std::uint32_t resource, std::uint32_t sampler, bool hasSampler) {
|
|
for (auto& patch : m_memoryPatches) {
|
|
if (patch.index != index) {
|
|
continue;
|
|
}
|
|
if (patch.resource != resource || (hasSampler && patch.hasSampler && patch.sampler != sampler)) {
|
|
fail("memory metadata is reused with incompatible resources");
|
|
}
|
|
if (hasSampler) {
|
|
patch.sampler = sampler;
|
|
patch.hasSampler = true;
|
|
}
|
|
return;
|
|
}
|
|
m_memoryPatches.push_back({index, resource, sampler, hasSampler});
|
|
}
|
|
|
|
void Collect(IrValue& inst) {
|
|
const auto op = inst.Opcode();
|
|
if (op == IrOpcode::ImageBvhIntersectRay) {
|
|
m_info.usesDma = true;
|
|
return;
|
|
}
|
|
const auto buffer = BufferAccessOf(op);
|
|
const auto addressInfo = AddressOpcodeInfoOf(op);
|
|
const auto imageInfo = ImageOpcodeInfoOf(op);
|
|
if (buffer == BufferAccess::None && addressInfo.access == AddressAccess::None && imageInfo.access == ImageAccess::None) {
|
|
return;
|
|
}
|
|
const auto flags = inst.Flags<MemoryFlags>();
|
|
if (flags.index >= m_program.Resources().memoryInfo.size()) {
|
|
fail("memory metadata index " + std::to_string(flags.index) + " is out of range");
|
|
}
|
|
if (inst.ArgumentCount() == 0) {
|
|
fail("memory operation has no resource handle");
|
|
}
|
|
const auto& memory = m_program.Resources().memoryInfo[flags.index];
|
|
if (memory.planningOnly || IsTablePlanningMemory(flags.index)) {
|
|
return;
|
|
}
|
|
IrValue* handle = nullptr;
|
|
std::uint32_t source = 0;
|
|
std::uint32_t resource = 0;
|
|
|
|
if (buffer != BufferAccess::None) {
|
|
if (!TakeGpuDescriptor(inst, flags.index)) fail("buffer operation requires a four-dword runtime V#");
|
|
return;
|
|
}
|
|
if (addressInfo.access != AddressAccess::None) {
|
|
if (!IsAddressResourceKind(memory.kind)) {
|
|
fail("address operation has invalid resource kind");
|
|
}
|
|
if (memory.kind == ResourceKind::Scratch) {
|
|
handle = inst.Argument(0)->Resolve();
|
|
if (handle->Opcode() != IrOpcode::GetScratchResource || handle->ArgumentCount() != 0) {
|
|
fail("scratch operation requires GetScratchResource");
|
|
}
|
|
if (m_info.scratchDwords == 0) {
|
|
fail("scratch operation requires a nonzero AGC per-thread size");
|
|
}
|
|
return;
|
|
}
|
|
ValidateAddressHandle(inst.Argument(0));
|
|
// Debug aid: APS5_TRACE_BDA=1 names every access that makes the program address-based
|
|
// (a raw scalar load the SRT walker left in place, or a flat/global access), so the
|
|
// reason a stage takes the BDA path can be read off without a shader dump.
|
|
// The first few accesses of a program are printed (a Bink kernel has hundreds); Run
|
|
// reports how many more there were.
|
|
if (bdaTraceEnabled() && ++m_bdaTraces <= bdaTraceLimit) {
|
|
const auto* kind = memory.kind == ResourceKind::ScalarAddress ? "scalar address" : memory.kind == ResourceKind::Global ? "global" : "flat";
|
|
const IrValue* offset = inst.ArgumentCount() > 1 ? inst.Argument(1)->Resolve() : nullptr;
|
|
const bool immediateOffset = offset != nullptr && offset->HasImmediate();
|
|
std::fprintf(stderr, "[bda] %s at pc 0x%08x: %s access, offset %s%s\n", std::string(IrOpcodeName(op)).c_str(), flags.pc, kind, immediateOffset ? "immediate" : "dynamic", memory.kind == ResourceKind::ScalarAddress && !immediateOffset ? " (a register offset is not planned by the SRT walker)" : "");
|
|
}
|
|
m_info.usesDma = true;
|
|
m_info.bdaWrites = m_info.bdaWrites || addressInfo.access == AddressAccess::Write || addressInfo.access == AddressAccess::Atomic;
|
|
return;
|
|
}
|
|
|
|
if (memory.kind != ResourceKind::Image || imageInfo.resourceClass == ImageResourceClass::None) {
|
|
fail("image operation has invalid resource kind");
|
|
}
|
|
handle = inst.Argument(0)->Resolve();
|
|
const TableColumnPlan* column = FindTableColumn(*handle);
|
|
if (column != nullptr) {
|
|
source = column->source;
|
|
} else {
|
|
GetHandle(inst.Argument(0), IrOpcode::GetImageResource, 8, handle, source);
|
|
}
|
|
resource = AddImage(source, memory, op, flags.pc);
|
|
NoteBlockImage(resource);
|
|
if (resource == std::numeric_limits<std::uint32_t>::max()) {
|
|
fail("image resource limit exceeded");
|
|
}
|
|
AddHandlePatch(handle, resource);
|
|
std::uint32_t sampler = 0;
|
|
if (imageInfo.needsSampler) {
|
|
if (inst.ArgumentCount() < 2) {
|
|
fail("sampled image operation has no sampler handle");
|
|
}
|
|
IrValue* samplerHandle = inst.Argument(1)->Resolve();
|
|
std::uint32_t samplerSource = 0;
|
|
const TableColumnPlan* samplerColumn = FindTableColumn(*samplerHandle);
|
|
if (samplerColumn != nullptr) {
|
|
samplerSource = samplerColumn->source;
|
|
} else {
|
|
const bool sampleAdjust = (memory.imageSampleFlags & RdnaImageSampleFlagAdjust) != 0;
|
|
GetHandle(inst.Argument(1), IrOpcode::GetSamplerResource, 4, samplerHandle, samplerSource, true, sampleAdjust);
|
|
}
|
|
sampler = AddSampler(samplerSource, flags.pc);
|
|
if (sampler == std::numeric_limits<std::uint32_t>::max()) {
|
|
fail("sampler resource limit exceeded");
|
|
}
|
|
AddHandlePatch(samplerHandle, sampler);
|
|
AddSampledPair(resource, sampler, flags.pc);
|
|
}
|
|
AddMemoryPatch(flags.index, resource, sampler, imageInfo.needsSampler);
|
|
}
|
|
|
|
const DescriptorSource* Source(std::uint32_t source) const {
|
|
return source < m_sources.size() ? &m_sources[source] : nullptr;
|
|
}
|
|
|
|
void LinkImageAliases() {
|
|
for (auto& buffer : m_info.buffers) {
|
|
const DescriptorSource* bufferSource = Source(buffer.source);
|
|
if (bufferSource == nullptr || bufferSource->dwordCount != 4) {
|
|
continue;
|
|
}
|
|
for (std::uint32_t image = 0; image < m_info.images.size(); image++) {
|
|
const DescriptorSource* imageSource = Source(m_info.images[image].source);
|
|
if (imageSource == nullptr || imageSource->dwordCount != 8 || imageSource->tableColumn.has_value()) {
|
|
continue;
|
|
}
|
|
bool alias = true;
|
|
for (std::uint32_t dword = 0; dword < 4; dword++) {
|
|
alias = alias && EquivalentValue(m_program.Resources(), bufferSource->dwords[dword], imageSource->dwords[dword]);
|
|
}
|
|
if (alias) {
|
|
buffer.imageAlias = image;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
IrProgram& m_program;
|
|
ShaderInfo m_info;
|
|
IrBuilder m_builder;
|
|
std::vector<DescriptorSource> m_sources;
|
|
std::vector<std::vector<std::uint32_t>> m_blockSources;
|
|
std::vector<std::vector<std::uint32_t>> m_blockImages;
|
|
std::uint32_t m_currentBlock = std::numeric_limits<std::uint32_t>::max();
|
|
std::vector<HandlePatch> m_handlePatches;
|
|
std::vector<MemoryPatch> m_memoryPatches;
|
|
// APS5_TRACE_BDA: address accesses seen by Collect (the first bdaTraceLimit are printed).
|
|
unsigned m_bdaTraces = 0;
|
|
std::unordered_map<std::uint32_t, std::vector<const IrValue*>> m_memoryUsers;
|
|
bool m_memoryUsersIndexed = false;
|
|
std::vector<TableColumnPlan> m_tableColumns;
|
|
std::vector<const IrValue*> m_tablePlanningReads;
|
|
std::vector<std::uint32_t> m_tablePlanningMemory;
|
|
std::vector<const IrValue*> m_unselectedHandles;
|
|
struct EdgeSelectorEntry {
|
|
IrBlock* block = nullptr;
|
|
std::vector<bool> edges;
|
|
IrValue* value = nullptr;
|
|
};
|
|
std::vector<EdgeSelectorEntry> m_edgeSelectors;
|
|
};
|
|
|
|
}
|
|
|
|
void ResourceTracker::Track(IrProgram& program) const {
|
|
Tracker(program).Run();
|
|
}
|
|
|
|
}
|