diff --git a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp index 839a465a6a..56dc21e87b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp @@ -77,6 +77,22 @@ struct Builder { return threadBit(low, high); } + std::pair ballotWords(IrValue& predicate) { + auto& ballot = emit(IrOpcode::Ballot, IrType::U32x4, {&predicate}); + return {&emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &constant(0u)}), &emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &constant(1u)})}; + } + + IrValue& wordAnd(IrValue& left, IrValue& right) { + return emit(IrOpcode::BitwiseAnd32, IrType::U32, {&left, &right}); + } + + IrValue& phi(IrType type, IrBlock& predecessor, IrValue& incoming) { + auto& created = program.CreateValue(IrOpcode::Phi, type); + block->AppendInstruction(&created); + created.AddPhiOperand(&predecessor, &incoming); + return created; + } + void keep(IrValue& value) { (void)emit(IrOpcode::ReferenceU32, IrType::Void, {&value}); } @@ -339,8 +355,13 @@ int main() { return count == 0u && !gone; }); - passed &= run("an exec merged by a phi inside a loop keeps the select", [&] { + passed &= run("an exec carried around a loop whose every incoming exec implies the write's exec drops the select", [&] { const auto [count, gone] = mergedExec(false, true); + return count == 1u && gone; + }); + + passed &= run("an exec carried around a loop with a wider incoming exec keeps the select", [&] { + const auto [count, gone] = mergedExec(true, true); return count == 0u && !gone; }); @@ -425,9 +446,10 @@ int main() { return b.eliminate() == 0u && !removed(written); }); - passed &= run("a write read inside a later loop must keep its select", [] { + const auto laterLoop = [](bool wideRead) { Builder b; - auto& exec = b.mask(16u); + auto& wide = b.mask(48u); + auto& exec = b.logicalAnd(wide, b.mask(16u)); auto& old = b.lane(); auto& written = b.select(exec, b.add(old, 1u), old); IrBlock& head = *b.block; @@ -435,7 +457,138 @@ int main() { head.AddBranch(&loop); loop.AddBranch(&loop); b.block = &loop; - b.keep(b.select(exec, b.add(written, 2u), old)); + b.keep(b.select(wideRead ? wide : exec, b.add(written, 2u), old)); + return std::pair{b.eliminate(), removed(written)}; + }; + + passed &= run("a write read inside a later loop under its own exec loses its select", [&] { + const auto [count, gone] = laterLoop(false); + return count == 1u && gone; + }); + + passed &= run("a write read inside a later loop under a wider exec keeps its select", [&] { + const auto [count, gone] = laterLoop(true); + return count == 0u && !gone; + }); + + enum class Latch { Narrowed, Revived }; + const auto carried = [](Latch latch, bool readInside) { + Builder b; + auto& wide = b.mask(48u); + auto& exec = b.logicalAnd(wide, b.mask(40u)); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + const auto [savedLow, savedHigh] = b.ballotWords(exec); + IrBlock& entry = *b.block; + IrBlock& header = b.newBlock(); + IrBlock& exit = b.newBlock(); + entry.AddBranch(&header); + header.AddBranch(&header); + header.AddBranch(&exit); + b.block = &header; + auto& low = b.phi(IrType::U32, entry, *savedLow); + auto& high = b.phi(IrType::U32, entry, *savedHigh); + auto& value = b.phi(IrType::U32, entry, written); + auto& loopExec = b.threadBit(low, high); + auto& next = b.select(loopExec, b.add(value, 2u), value); + if (readInside) b.keep(b.select(loopExec, b.add(value, 3u), old)); + if (latch == Latch::Narrowed) { + const auto [stayLow, stayHigh] = b.ballotWords(b.mask(24u)); + low.AddPhiOperand(&header, &b.wordAnd(low, *stayLow)); + high.AddPhiOperand(&header, &b.wordAnd(high, *stayHigh)); + } else { + const auto [revivedLow, revivedHigh] = b.ballotWords(wide); + low.AddPhiOperand(&header, revivedLow); + high.AddPhiOperand(&header, revivedHigh); + } + value.AddPhiOperand(&header, &next); + b.block = &exit; + b.keep(b.select(b.threadBit(*savedLow, *savedHigh), b.add(value, 4u), old)); + return std::pair{b.eliminate(), removed(written)}; + }; + + passed &= run("a write carried through a loop whose exec is saved, narrowed and restored loses its select", [&] { + const auto [count, gone] = carried(Latch::Narrowed, false); + return count == 1u && gone; + }); + + passed &= run("a write read inside a loop under the loop's narrowed exec loses its select", [&] { + const auto [count, gone] = carried(Latch::Narrowed, true); + return count == 1u && gone; + }); + + passed &= run("a lane that becomes active again inside a loop and reads the write keeps the select", [&] { + const auto [count, gone] = carried(Latch::Revived, true); + return count == 0u && !gone; + }); + + passed &= run("a lane that becomes active again inside a loop but never reads the write lets the select go", [&] { + const auto [count, gone] = carried(Latch::Revived, false); + return count == 1u && gone; + }); + + passed &= run("an exec proven only by assuming a loop phi is forgotten once that phi turns out wider", [] { + Builder b; + auto& wide = b.mask(48u); + auto& exec = b.logicalAnd(wide, b.mask(40u)); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + const auto [savedLow, savedHigh] = b.ballotWords(exec); + IrBlock& entry = *b.block; + IrBlock& header = b.newBlock(); + IrBlock& kill = b.newBlock(); + IrBlock& revive = b.newBlock(); + IrBlock& latch = b.newBlock(); + IrBlock& exit = b.newBlock(); + entry.AddBranch(&header); + header.AddBranch(&kill); + header.AddBranch(&revive); + kill.AddBranch(&latch); + revive.AddBranch(&latch); + latch.AddBranch(&header); + latch.AddBranch(&exit); + b.block = &header; + auto& low = b.phi(IrType::U32, entry, *savedLow); + auto& high = b.phi(IrType::U32, entry, *savedHigh); + (void)b.select(b.threadBit(low, high), written, old); + b.block = &kill; + const auto [stayLow, stayHigh] = b.ballotWords(b.mask(24u)); + auto& killedLow = b.wordAnd(low, *stayLow); + auto& killedHigh = b.wordAnd(high, *stayHigh); + b.keep(b.select(b.threadBit(killedLow, killedHigh), written, old)); + b.block = &revive; + const auto [revivedLow, revivedHigh] = b.ballotWords(wide); + b.block = &latch; + auto& mergedLow = b.phi(IrType::U32, kill, killedLow); + auto& mergedHigh = b.phi(IrType::U32, kill, killedHigh); + mergedLow.AddPhiOperand(&revive, revivedLow); + mergedHigh.AddPhiOperand(&revive, revivedHigh); + low.AddPhiOperand(&latch, &mergedLow); + high.AddPhiOperand(&latch, &mergedHigh); + return b.eliminate() == 0u && !removed(written); + }); + + passed &= run("a write mask computed inside an earlier loop does not let that loop's phis stand for it", [] { + Builder b; + auto& old = b.lane(); + IrBlock& entry = *b.block; + IrBlock& header = b.newBlock(); + IrBlock& exit = b.newBlock(); + entry.AddBranch(&header); + header.AddBranch(&header); + header.AddBranch(&exit); + b.block = &header; + auto& low = b.phi(IrType::U32, entry, b.constant(0u)); + auto& high = b.phi(IrType::U32, entry, b.constant(0u)); + auto& bound = b.phi(IrType::U32, entry, b.constant(64u)); + auto& exec = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.lane(), &bound}); + const auto [stayLow, stayHigh] = b.ballotWords(exec); + low.AddPhiOperand(&header, stayLow); + high.AddPhiOperand(&header, stayHigh); + bound.AddPhiOperand(&header, &b.emit(IrOpcode::ISub32, IrType::U32, {&bound, &b.constant(8u)})); + b.block = &exit; + auto& written = b.select(exec, b.add(old, 1u), old); + b.keep(b.select(b.threadBit(low, high), b.add(written, 2u), old)); return b.eliminate() == 0u && !removed(written); }); diff --git a/core/libs/tests/PixelAncillary.cpp b/core/libs/tests/PixelAncillary.cpp index 9832c0ebe5..15740c514a 100644 --- a/core/libs/tests/PixelAncillary.cpp +++ b/core/libs/tests/PixelAncillary.cpp @@ -340,6 +340,93 @@ static void HelperLanesRefused(HelperRead read, std::uint32_t chain = 0u) { } Require(false); } +enum class LoopRead { ExportAfter, ExportInside, UniformLoop, RevivedExportInside }; + +static IrValue& LaneBit(IrBuilder& builder, IrValue& low, IrValue& high) { + IrValue& lane = builder.Emit(IrOpcode::LaneId, IrType::U32, {}); + IrValue& word = builder.Select(builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {&lane, &builder.Constant(32u)}), low, high); + IrValue& shifted = builder.Emit(IrOpcode::ShiftRightLogical32, IrType::U32, {&word, &builder.BitwiseAnd(lane, builder.Constant(31u))}); + return builder.INotEqual(builder.BitwiseAnd(shifted, builder.Constant(1u)), builder.Constant(0u)); +} +static HelperProgram HelperLoop(IrProgram& program, LoopRead read) { + auto& entry = Begin(program); + auto& header = program.CreateBlock(); + auto& exit = program.CreateBlock(); + program.BlockOrder().push_back(&header); + program.BlockOrder().push_back(&exit); + entry.AddBranch(&header); + header.AddBranch(&header); + header.AddBranch(&exit); + IrBuilder builder(program); + builder.SetInsertionPoint(entry); + IrValue& helper = builder.Emit(IrOpcode::GetBuiltin, IrType::U32, {&builder.Constant(static_cast(StageInputKind::HelperInvocation)), &builder.Constant(0u)}); + IrValue& live = builder.IEqual(helper, builder.Constant(0u)); + IrValue& ancillary = AncillaryBuiltin(builder); + IrValue& layer = builder.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&ancillary, &builder.Constant(16u), &builder.Constant(11u)}); + IrValue& target = builder.Select(live, layer, builder.Constant(0u)); + IrValue& written = builder.Select(live, builder.Constant(SelectOther), ancillary); + IrValue& frontFacing = FrontFacingBuiltin(builder); + IrValue& saved = builder.Emit(IrOpcode::Ballot, IrType::U32x4, {&live}); + IrValue& savedLow = builder.CompositeExtract(saved, 0u); + IrValue& savedHigh = builder.CompositeExtract(saved, 1u); + IrValue& low = program.CreateValue(IrOpcode::Phi, IrType::U32); + IrValue& high = program.CreateValue(IrOpcode::Phi, IrType::U32); + IrValue& value = program.CreateValue(IrOpcode::Phi, IrType::U32); + header.AppendInstruction(&low); + header.AppendInstruction(&high); + header.AppendInstruction(&value); + low.AddPhiOperand(&entry, &savedLow); + high.AddPhiOperand(&entry, &savedHigh); + value.AddPhiOperand(&entry, &written); + builder.SetInsertionPoint(header); + IrValue& loopExec = read == LoopRead::UniformLoop ? live : LaneBit(builder, low, high); + IrValue& squared = builder.Emit(IrOpcode::FPMul32, IrType::F32, {&builder.BitCastF32(value), &builder.BitCastF32(value)}); + IrValue& next = builder.Select(loopExec, builder.BitCastU32(squared), value); + IrValue& stay = builder.Emit(IrOpcode::Ballot, IrType::U32x4, {&frontFacing}); + IrValue& stayLow = builder.CompositeExtract(stay, 0u); + IrValue& stayHigh = builder.CompositeExtract(stay, 1u); + const bool revived = read == LoopRead::RevivedExportInside; + low.AddPhiOperand(&header, revived ? &stayLow : &builder.BitwiseAnd(low, stayLow)); + high.AddPhiOperand(&header, revived ? &stayHigh : &builder.BitwiseAnd(high, stayHigh)); + value.AddPhiOperand(&header, &next); + program.Metadata().exportInfo.push_back(ExportInfo {.kind = ExportTargetKind::Mrt, .index = 0u, .en = 0xFu, .done = true, .vm = true}); + const auto exportValue = [&](IrValue& guard) { + IrValue& data = builder.Emit(IrOpcode::CompositeConstructU32x4, IrType::U32x4, {&value, &target, &builder.Constant(0u), &builder.Constant(0u)}); + static_cast(builder.Emit(IrOpcode::SetAttribute, IrType::Void, {&data, &guard}, ExportFlags {0u, 0u})); + }; + if (read == LoopRead::ExportInside || revived) exportValue(loopExec); + builder.SetInsertionPoint(exit); + if (read == LoopRead::ExportAfter) exportValue(LaneBit(builder, savedLow, savedHigh)); + if (read == LoopRead::UniformLoop) exportValue(live); + static_cast(builder.Emit(IrOpcode::Return, IrType::Void, {})); + return HelperProgram {&written, &layer, &value}; +} +static void HelperLoopOnly(LoopRead read) { + IrProgram program; + const HelperProgram built = HelperLoop(program, read); + LowerMasked(program); + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + Require(inst->Opcode() != IrOpcode::GetBuiltin || static_cast(inst->Argument(0)->Resolve()->ImmediateU32()) != StageInputKind::PackedAncillary); + } + } + Require(built.written->Parent() == nullptr); + const IrValue* field = built.layer->Argument(0)->Resolve(); + Require(field->Opcode() == IrOpcode::GetBuiltin && static_cast(field->Argument(0)->Resolve()->ImmediateU32()) == StageInputKind::Layer); + const IrValue* entryValue = built.exported->Argument(0)->Resolve(); + Require(entryValue->HasImmediate() && entryValue->ImmediateU32() == SelectOther); +} +static void HelperLoopRefused(LoopRead read) { + IrProgram program; + static_cast(HelperLoop(program, read)); + try { + LowerMasked(program); + } catch (const std::runtime_error& error) { + Require(std::string(error.what()).find("unsupported live use") != std::string::npos); + return; + } + Require(false); +} int main() { Extract(IrOpcode::BitFieldUExtract, 8u, 4u, StageInputKind::SampleId, 0u); Extract(IrOpcode::BitFieldUExtract, 9u, 2u, StageInputKind::SampleId, 1u); @@ -385,4 +472,8 @@ int main() { HelperLanesRefused(HelperRead::DataTestExport); HelperLanesRefused(HelperRead::Ballot); HelperLanesRefused(HelperRead::FullWaveExport, 6000u); + HelperLoopOnly(LoopRead::ExportAfter); + HelperLoopOnly(LoopRead::ExportInside); + HelperLoopOnly(LoopRead::UniformLoop); + HelperLoopRefused(LoopRead::RevivedExportInside); } diff --git a/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp b/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp index e9d218b806..be902084dd 100644 --- a/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp +++ b/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp @@ -274,6 +274,11 @@ Region regionOf(const std::vector& regions, const IrBlock* block) { return block != nullptr && block->Id() < regions.size() ? regions[block->Id()] : Region::Unreached; } +bool followable(const std::vector& regions, const IrBlock* block, bool acrossLoops) { + const Region region = regionOf(regions, block); + return region == Region::Once || (acrossLoops && region == Region::Loop); +} + struct ImplicationKey { const IrValue* first = nullptr; const IrValue* second = nullptr; @@ -291,10 +296,13 @@ class Implications { public: Implications(const std::vector& regions, std::uint32_t waveSize, std::size_t& work) : regions(regions), waveSize(waveSize), work(work) {} - void Reset(const IrValue* writeMask) { + void Reset(const IrValue* writeMask, bool acrossLoops) { mask = writeMask; + loops = acrossLoops; always = alwaysTrue(writeMask, waveSize); results.clear(); + proven.clear(); + assumptionReads = 0; } bool Implies(const IrValue* condition, std::uint32_t depth = 0) { @@ -302,33 +310,61 @@ public: if (condition == mask || always) return true; if (condition->HasImmediate()) return condition->Type() == IrType::Bool && !condition->ImmediateBool(); const ImplicationKey key{condition, nullptr}; - if (const auto known = results.find(key); known != results.end()) return known->second; + if (const auto known = lookup(key)) return *known; if (depth > ImplicationDepthLimit || work == 0) return false; --work; - results.emplace(key, false); + const Attempt attempt = assume(key); bool result = false; if (condition->Opcode() == IrOpcode::LogicalAnd) { result = Implies(condition->Argument(0), depth + 1u) || Implies(condition->Argument(1), depth + 1u); } else if (condition->IsPhi()) { - result = mergesOutsideLoops(*condition); + result = merges(*condition); for (std::size_t index = 0; result && index < condition->ArgumentCount(); ++index) { result = Implies(condition->Argument(index), depth + 1u); } } else if (const auto bit = threadBit(condition, waveSize)) { result = wordsImply(bit->low, bit->high, depth + 1u); } - results[key] = result; - return result; + return settle(key, attempt, result); } private: static constexpr std::uint32_t ImplicationDepthLimit = 64; - bool mergesOutsideLoops(const IrValue& phi) const { - if (regionOf(regions, phi.Parent()) != Region::Once || phi.ArgumentCount() != phi.PhiBlockCount()) return false; + enum class Proof : std::uint8_t { Assumed, Implied, NotImplied }; + + struct Attempt { + std::size_t proven = 0; + std::size_t assumptionReads = 0; + }; + + std::optional lookup(const ImplicationKey& key) { + const auto known = results.find(key); + if (known == results.end()) return std::nullopt; + if (known->second == Proof::Assumed) ++assumptionReads; + return known->second != Proof::NotImplied; + } + + Attempt assume(const ImplicationKey& key) { + results.emplace(key, Proof::Assumed); + return Attempt{proven.size(), assumptionReads}; + } + + bool settle(const ImplicationKey& key, const Attempt& attempt, bool implied) { + if (!implied && assumptionReads != attempt.assumptionReads) { + for (std::size_t index = attempt.proven; index < proven.size(); ++index) results.erase(proven[index]); + proven.resize(attempt.proven); + } + results[key] = implied ? Proof::Implied : Proof::NotImplied; + if (implied) proven.push_back(key); + return implied; + } + + bool merges(const IrValue& phi) const { + if (!followable(regions, phi.Parent(), loops) || phi.ArgumentCount() != phi.PhiBlockCount()) return false; for (std::size_t index = 0; index < phi.ArgumentCount(); ++index) { const IrValue* incoming = phi.Argument(index)->Resolve(); - if (!incoming->HasImmediate() && regionOf(regions, incoming->Parent()) != Region::Once) return false; + if (!incoming->HasImmediate() && !followable(regions, incoming->Parent(), loops)) return false; } return true; } @@ -343,10 +379,10 @@ private: bool wordsImply(const IrValue* low, const IrValue* high, std::uint32_t depth) { if (isImmediate(low, 0u) && (waveSize == 32u || isImmediate(high, 0u))) return true; const ImplicationKey key{low, high}; - if (const auto known = results.find(key); known != results.end()) return known->second; + if (const auto known = lookup(key)) return *known; if (depth > ImplicationDepthLimit || work == 0) return false; --work; - results.emplace(key, false); + const Attempt attempt = assume(key); bool result = false; if (low->Opcode() == IrOpcode::CompositeExtractU32x4 && isImmediate(low->Argument(1)->Resolve(), 0u)) { const IrValue* ballot = low->Argument(0)->Resolve(); @@ -364,23 +400,25 @@ private: } } } - if (!result && low->IsPhi() && mergesOutsideLoops(*low) && (waveSize == 32u || (high->IsPhi() && high->Parent() == low->Parent() && mergesOutsideLoops(*high)))) { + if (!result && low->IsPhi() && merges(*low) && (waveSize == 32u || (high->IsPhi() && high->Parent() == low->Parent() && merges(*high)))) { result = true; for (std::size_t index = 0; result && index < low->PhiBlockCount(); ++index) { const IrValue* highIncoming = waveSize == 32u ? high : incomingFrom(*high, low->PhiBlock(index)); result = highIncoming != nullptr && wordsImply(low->Argument(index)->Resolve(), highIncoming, depth + 1u); } } - results[key] = result; - return result; + return settle(key, attempt, result); } const std::vector& regions; std::uint32_t waveSize; std::size_t& work; const IrValue* mask = nullptr; + bool loops = false; bool always = false; - std::unordered_map results; + std::unordered_map results; + std::vector proven; + std::size_t assumptionReads = 0; }; std::vector blockRegions(const IrProgram& program) { @@ -436,8 +474,9 @@ public: MaskedLaneWalk(const IrProgram& program, const std::vector& regions) : program(program), regions(regions), limit(walkLimit(program)), implications(regions, program.WaveSize(), work) {} bool UnobservedWhereMasked(const IrValue& select, const IrValue* mask) { + const bool acrossLoops = mask->HasImmediate() || regionOf(regions, mask->Parent()) == Region::Once; work = limit; - implications.Reset(mask); + implications.Reset(mask, acrossLoops); startVisits(); pending.clear(); pending.push_back(&select); @@ -447,7 +486,7 @@ public: pending.pop_back(); for (const IrUse& use : value->OperandUses()) { const IrValue* user = use.user; - if (regionOf(regions, user->Parent()) != Region::Once) return false; + if (!followable(regions, user->Parent(), acrossLoops)) return false; if (isSelect(user->Opcode()) && use.operand == 1u && implications.Implies(user->Argument(0))) continue; if (user->Opcode() == IrOpcode::LogicalAnd && use.operand < 2u && implications.Implies(user->Argument(1u - use.operand))) continue; if (readsOnlyWhereActive(user->Opcode()) && use.operand + 1u < user->ArgumentCount() && implications.Implies(user->Argument(user->ArgumentCount() - 1u))) continue;