mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-23 00:53:02 -05:00
Jan ŠléglandClaude Opus 5.5
7f5cd4480d
A selection outside loops whose one arm only returns and whose other arm branches to a join it does not dominate gets an empty block on that edge as its merge (splitOneReturnJoin), but only when every other predecessor of the join dominates the selection or is dominated by it. When the parent selection's other arm is a region that ends in the same join, the selection has no structured merge and the shader fails to recompile. When every structurization attempt fails, Structurize now runs them again and accepts that split for any join the selection does not dominate. The split only inserts an empty block on the edge from the selection to the join. Graphs that structurize on main keep their result. A synthetic CFG and a synthetic program cover the shape; both fail on main with "conditional block N has no structured merge". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Fnrk5Dazjb2tLooVSpdRQq
1292 lines
55 KiB
C++
1292 lines
55 KiB
C++
#include "ControlFlow/GraphBuilder.hpp"
|
|
#include "ControlFlow/RequestSerializer.hpp"
|
|
#include "ControlFlow/Structurizer.hpp"
|
|
#include "RdnaDecoder/RdnaInstructionDecoder.hpp"
|
|
#include "Recompiler.hpp"
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <cstdio>
|
|
#include <exception>
|
|
#include <fstream>
|
|
#include <initializer_list>
|
|
#include <iterator>
|
|
#include <map>
|
|
#include <span>
|
|
#include <sstream>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <tuple>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
|
|
using namespace ShaderRecompiler;
|
|
|
|
enum class Split {
|
|
None,
|
|
Clone,
|
|
Route
|
|
};
|
|
|
|
struct Program {
|
|
std::string_view name;
|
|
std::string_view source;
|
|
std::vector<std::uint32_t> code;
|
|
Split split;
|
|
std::size_t clonedLimit = 0;
|
|
std::vector<std::uint32_t> reference = {};
|
|
};
|
|
|
|
using RouteState = std::map<std::uint32_t, bool>;
|
|
|
|
void require(bool condition, const std::string& message) {
|
|
if (!condition) throw std::runtime_error(message);
|
|
}
|
|
|
|
class StructuralDominators {
|
|
public:
|
|
explicit StructuralDominators(const ControlFlowGraph& graph) : dominators(graph.blocks.size()) {
|
|
std::vector<std::vector<std::uint32_t>> predecessors(graph.blocks.size());
|
|
for (const auto& block : graph.blocks) {
|
|
for (const auto successor : block.successors) predecessors[successor].push_back(block.id);
|
|
const auto& terminator = block.terminator;
|
|
if (terminator.mergeBlock != InvalidControlFlowId) predecessors[terminator.mergeBlock].push_back(block.id);
|
|
if (terminator.loopHeader && terminator.continueBlock != InvalidControlFlowId) predecessors[terminator.continueBlock].push_back(block.id);
|
|
}
|
|
|
|
std::vector<std::uint32_t> all;
|
|
all.reserve(graph.blocks.size());
|
|
for (const auto& block : graph.blocks) all.push_back(block.id);
|
|
for (const auto& block : graph.blocks) dominators[block.id] = block.id == graph.entryBlock ? std::vector<std::uint32_t>{block.id} : all;
|
|
|
|
bool changed = true;
|
|
while (changed) {
|
|
changed = false;
|
|
for (const auto& block : graph.blocks) {
|
|
if (block.id == graph.entryBlock) continue;
|
|
auto next = predecessors[block.id].empty() ? std::vector<std::uint32_t>{} : dominators[predecessors[block.id].front()];
|
|
for (std::size_t i = 1; i < predecessors[block.id].size(); ++i) {
|
|
std::vector<std::uint32_t> intersection;
|
|
const auto& other = dominators[predecessors[block.id][i]];
|
|
std::set_intersection(next.begin(), next.end(), other.begin(), other.end(), std::back_inserter(intersection));
|
|
next = std::move(intersection);
|
|
}
|
|
if (!std::binary_search(next.begin(), next.end(), block.id)) {
|
|
next.insert(std::lower_bound(next.begin(), next.end(), block.id), block.id);
|
|
}
|
|
if (next != dominators[block.id]) {
|
|
dominators[block.id] = std::move(next);
|
|
changed = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
bool Dominates(std::uint32_t dominator, std::uint32_t block) const {
|
|
const auto& values = dominators.at(block);
|
|
return std::binary_search(values.begin(), values.end(), dominator);
|
|
}
|
|
|
|
private:
|
|
std::vector<std::vector<std::uint32_t>> dominators;
|
|
};
|
|
|
|
std::vector<std::uint32_t> constructBlocks(const ControlFlowGraph& graph, const StructuralDominators& dominators, std::uint32_t header, std::uint32_t exclude) {
|
|
std::vector<std::uint32_t> blocks;
|
|
for (const auto& block : graph.blocks) {
|
|
if (dominators.Dominates(header, block.id) && (exclude == InvalidControlFlowId || !dominators.Dominates(exclude, block.id))) {
|
|
blocks.push_back(block.id);
|
|
}
|
|
}
|
|
return blocks;
|
|
}
|
|
|
|
bool inEnclosingContinueConstruct(const ControlFlowGraph& graph, const StructuralDominators& dominators, std::uint32_t header, std::uint32_t member) {
|
|
return std::any_of(graph.blocks.begin(), graph.blocks.end(), [&](const BasicBlock& loop) {
|
|
const auto continueBlock = loop.terminator.continueBlock;
|
|
return loop.terminator.loopHeader && dominators.Dominates(loop.id, header) && !dominators.Dominates(continueBlock, header) && dominators.Dominates(continueBlock, member);
|
|
});
|
|
}
|
|
|
|
void verifyStructured(const std::string& prefix, const ControlFlowGraph& graph) {
|
|
const StructuralDominators dominators(graph);
|
|
std::map<std::uint32_t, std::uint32_t> mergeOwners;
|
|
std::vector<std::pair<std::uint32_t, std::uint32_t>> loopExits;
|
|
for (const auto& block : graph.blocks) {
|
|
const auto& terminator = block.terminator;
|
|
if (terminator.mergeBlock == InvalidControlFlowId) continue;
|
|
require(mergeOwners.emplace(terminator.mergeBlock, block.id).second, prefix + "block " + std::to_string(terminator.mergeBlock) + " merges two constructs");
|
|
if (terminator.loopHeader) loopExits.emplace_back(terminator.mergeBlock, terminator.continueBlock);
|
|
}
|
|
for (const auto& block : graph.blocks) {
|
|
const auto& terminator = block.terminator;
|
|
if (terminator.loopHeader || terminator.mergeBlock == InvalidControlFlowId) continue;
|
|
for (const auto member : constructBlocks(graph, dominators, block.id, terminator.mergeBlock)) {
|
|
if (inEnclosingContinueConstruct(graph, dominators, block.id, member)) continue;
|
|
for (const auto successor : graph.FindBlock(member).successors) {
|
|
if (successor == terminator.mergeBlock || (dominators.Dominates(block.id, successor) && !dominators.Dominates(terminator.mergeBlock, successor))) continue;
|
|
const bool loopExit = std::any_of(loopExits.begin(), loopExits.end(), [&](const auto& exits) { return successor == exits.first || successor == exits.second; });
|
|
require(loopExit, prefix + "block " + std::to_string(member) + " of the selection at block " + std::to_string(block.id) + " branches to block " + std::to_string(successor) + " outside the construct");
|
|
}
|
|
}
|
|
}
|
|
for (const auto& loop : graph.blocks) {
|
|
if (!loop.terminator.loopHeader) continue;
|
|
const auto continueBlock = loop.terminator.continueBlock;
|
|
for (const auto member : constructBlocks(graph, dominators, loop.id, loop.terminator.mergeBlock)) {
|
|
const auto& terminator = graph.FindBlock(member).terminator;
|
|
if (terminator.loopHeader || dominators.Dominates(continueBlock, member)) continue;
|
|
require(terminator.mergeBlock != continueBlock, prefix + "the selection at block " + std::to_string(member) + " in the loop at block " + std::to_string(loop.id) + " merges at the loop's continue block " + std::to_string(continueBlock));
|
|
}
|
|
}
|
|
}
|
|
|
|
ControlFlowGraph makeGraph(const std::vector<std::vector<std::uint32_t>>& successors) {
|
|
ControlFlowGraph graph;
|
|
graph.entryBlock = 0;
|
|
for (std::uint32_t id = 0; id < successors.size(); ++id) {
|
|
BasicBlock block;
|
|
block.id = id;
|
|
block.startProgramCounter = id * 8;
|
|
block.endProgramCounter = id * 8 + 8;
|
|
block.instructionBegin = id * 2;
|
|
block.instructionEnd = id * 2 + 2;
|
|
block.successors = successors[id];
|
|
auto& terminator = block.terminator;
|
|
if (successors[id].empty()) {
|
|
terminator.kind = TerminatorKind::Return;
|
|
} else if (successors[id].size() == 1) {
|
|
terminator.kind = TerminatorKind::Branch;
|
|
terminator.trueBlock = successors[id][0];
|
|
} else {
|
|
terminator.kind = TerminatorKind::ConditionalBranch;
|
|
terminator.condition = BranchCondition::SccNonZero;
|
|
terminator.trueBlock = successors[id][0];
|
|
terminator.falseBlock = successors[id][1];
|
|
}
|
|
graph.blocks.push_back(std::move(block));
|
|
}
|
|
for (const auto& block : graph.blocks) {
|
|
for (const auto successor : block.successors) graph.blocks[successor].predecessors.push_back(block.id);
|
|
}
|
|
return graph;
|
|
}
|
|
|
|
void verifyLoopMergeDoesNotEnterNestedSelection() {
|
|
auto graph = makeGraph({{1, 5}, {2, 3}, {5}, {5, 4}, {1}, {}});
|
|
Structurizer{}.Structurize(graph);
|
|
verifyStructured("loop merge structural dominance: ", graph);
|
|
}
|
|
|
|
std::uint32_t followEmptyBlocks(const std::string& prefix, const ControlFlowGraph& graph, std::uint32_t blockId, RouteState& routes) {
|
|
for (std::size_t steps = 0; steps <= graph.blocks.size(); ++steps) {
|
|
const auto& block = graph.FindBlock(blockId);
|
|
const auto& terminator = block.terminator;
|
|
const bool empty = block.instructionBegin == block.instructionEnd;
|
|
require(terminator.gotoValue < 0 || empty, prefix + "block " + std::to_string(block.id) + " sets a route variable after guest instructions");
|
|
if (terminator.gotoValue >= 0) routes[terminator.gotoVariable] = terminator.gotoValue != 0;
|
|
if (!empty) return blockId;
|
|
if (terminator.kind == TerminatorKind::Branch) {
|
|
blockId = terminator.trueBlock;
|
|
} else if (terminator.kind == TerminatorKind::ConditionalBranch && terminator.condition == BranchCondition::GotoVariable) {
|
|
const auto route = routes.find(terminator.gotoVariable);
|
|
require(route != routes.end(), prefix + "block " + std::to_string(block.id) + " reads route variable " + std::to_string(terminator.gotoVariable) + " before any path sets it");
|
|
blockId = route->second ? terminator.trueBlock : terminator.falseBlock;
|
|
} else {
|
|
return blockId;
|
|
}
|
|
}
|
|
throw std::runtime_error(prefix + "a cycle of empty blocks");
|
|
}
|
|
|
|
void verifySameExecutions(const std::string& prefix, const ControlFlowGraph& original, const ControlFlowGraph& structured) {
|
|
std::vector<std::tuple<std::uint32_t, std::uint32_t, RouteState>> pending{{structured.blocks.front().id, original.entryBlock, {}}};
|
|
std::vector<std::tuple<std::uint32_t, std::uint32_t, RouteState>> visited;
|
|
RouteState none;
|
|
while (!pending.empty()) {
|
|
auto [structuredId, originalId, routes] = pending.back();
|
|
pending.pop_back();
|
|
const auto copyId = followEmptyBlocks(prefix, structured, structuredId, routes);
|
|
const auto sourceId = followEmptyBlocks(prefix, original, originalId, none);
|
|
auto state = std::make_tuple(copyId, sourceId, routes);
|
|
if (std::find(visited.begin(), visited.end(), state) != visited.end()) continue;
|
|
visited.push_back(state);
|
|
const auto& copy = structured.FindBlock(copyId);
|
|
const auto& source = original.FindBlock(sourceId);
|
|
require(copy.instructionBegin == source.instructionBegin && copy.instructionEnd == source.instructionEnd, prefix + "block " + std::to_string(copy.id) + " runs other instructions than block " + std::to_string(source.id));
|
|
require(copy.terminator.kind == source.terminator.kind && copy.terminator.condition == source.terminator.condition, prefix + "block " + std::to_string(copy.id) + " branches differently from block " + std::to_string(source.id));
|
|
if (source.terminator.kind == TerminatorKind::Branch || source.terminator.kind == TerminatorKind::ConditionalBranch) pending.emplace_back(copy.terminator.trueBlock, source.terminator.trueBlock, routes);
|
|
if (source.terminator.kind == TerminatorKind::ConditionalBranch) pending.emplace_back(copy.terminator.falseBlock, source.terminator.falseBlock, routes);
|
|
}
|
|
}
|
|
|
|
std::size_t clonedInstructions(const ControlFlowGraph& graph) {
|
|
std::map<std::pair<std::uint32_t, std::uint32_t>, std::size_t> copies;
|
|
std::size_t cloned = 0;
|
|
for (const auto& block : graph.blocks) {
|
|
if (block.instructionBegin != block.instructionEnd && copies[{block.instructionBegin, block.instructionEnd}]++ != 0) cloned += block.instructionEnd - block.instructionBegin;
|
|
}
|
|
return cloned;
|
|
}
|
|
|
|
std::size_t routeVariables(const ControlFlowGraph& graph) {
|
|
std::vector<std::uint32_t> variables;
|
|
for (const auto& block : graph.blocks) {
|
|
if (block.terminator.condition == BranchCondition::GotoVariable && std::find(variables.begin(), variables.end(), block.terminator.gotoVariable) == variables.end()) variables.push_back(block.terminator.gotoVariable);
|
|
}
|
|
return variables.size();
|
|
}
|
|
|
|
struct Structured {
|
|
std::size_t addedBlocks;
|
|
std::size_t clonedInstructions;
|
|
std::size_t routeVariables;
|
|
};
|
|
|
|
Structured verifyGraph(const std::string& name, std::span<const std::uint32_t> code) {
|
|
const std::string prefix = name + ": ";
|
|
const auto decoded = RdnaInstructionDecoder{}.Decode(code);
|
|
const auto original = GraphBuilder{}.Build(decoded);
|
|
auto graph = original;
|
|
Structurizer{}.Structurize(graph);
|
|
verifyStructured(prefix, graph);
|
|
verifySameExecutions(prefix, original, graph);
|
|
return {graph.blocks.size() - original.blocks.size(), clonedInstructions(graph), routeVariables(graph)};
|
|
}
|
|
|
|
std::vector<std::uint32_t> Store(std::vector<std::uint32_t> code) {
|
|
code.insert(code.end(), {0xe0700000u, 0x80000100u, 0xbf810000u});
|
|
return code;
|
|
}
|
|
|
|
void verifyRequest(const char* path) {
|
|
std::ifstream file(path, std::ios::binary);
|
|
std::ostringstream text;
|
|
text << file.rdbuf();
|
|
const auto request = RequestSerializer{}.Deserialize(text.str());
|
|
const auto result = verifyGraph(path, request.request.shader.code);
|
|
std::printf("%s: structured with %zu added blocks, %zu cloned instructions, %zu route variables\n", path, result.addedBlocks, result.clonedInstructions, result.routeVariables);
|
|
}
|
|
|
|
std::size_t recompile(const std::string& name, std::span<const std::uint32_t> code, std::uint32_t subgroupSize) {
|
|
const std::array<std::uint32_t, 4> userData{0x10000000u, 0x00100000u, 0x40u, 0x00027facu};
|
|
const std::array<std::uint32_t, 2> capabilities{1u, 61u};
|
|
const std::array<std::string_view, 1> extensions{"SPV_KHR_storage_buffer_storage_class"};
|
|
RecompileRequest request{};
|
|
request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}};
|
|
request.context.waveSize = 64;
|
|
request.context.userDataBaseRegister = 0;
|
|
request.context.userData = userData;
|
|
request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
|
|
request.target.vulkanVersion = 0x00403000u;
|
|
request.target.spirvVersion = 0x00010600u;
|
|
request.target.subgroupSize = subgroupSize;
|
|
request.target.bdaAbiVersion = 1;
|
|
request.target.supportedCapabilities = capabilities;
|
|
request.target.supportedExtensions = extensions;
|
|
request.target.maxWorkgroupSize = {1024u, 1024u, 64u};
|
|
request.target.maxWorkgroupInvocations = 1024;
|
|
request.target.maxWorkgroupSharedMemoryBytes = 49152;
|
|
request.layout = {0, 0, 0, 128};
|
|
request.useCache = false;
|
|
const auto result = Recompile(request);
|
|
require(!result.spirv.empty(), name + ": no SPIR-V for a " + std::to_string(subgroupSize) + "-lane subgroup");
|
|
return result.spirv.size();
|
|
}
|
|
|
|
void verifyLargeLoopExitFallback() {
|
|
const std::vector<std::uint32_t> code{
|
|
0x060202ffu, 0x3f800000u, 0x060202ffu, 0x3f000000u, 0x060202ffu, 0x3f001000u, 0x060202ffu, 0x3f002000u,
|
|
0x060202ffu, 0x3f003000u, 0x060202ffu, 0x3f004000u, 0x060202ffu, 0x3f005000u, 0x060202ffu, 0x3f006000u,
|
|
0x060202ffu, 0x3f007000u, 0x060202ffu, 0x3f008000u, 0x060202ffu, 0x3f009000u, 0x060202ffu, 0x3f00a000u,
|
|
0x060202ffu, 0x3f00b000u, 0x060202ffu, 0x3f00c000u, 0x060202ffu, 0x3f00d000u, 0x060202ffu, 0x3f00e000u,
|
|
0x060202ffu, 0x3f00f000u, 0x060202ffu, 0x3f010000u, 0x060202ffu, 0x3f810000u, 0x060202ffu, 0x3f000010u,
|
|
0x060202ffu, 0x3f001010u, 0x060202ffu, 0x3f002010u, 0x060202ffu, 0x3f003010u, 0x060202ffu, 0x3f004010u,
|
|
0x060202ffu, 0x3f005010u, 0x060202ffu, 0x3f006010u, 0x060202ffu, 0x3f007010u, 0x060202ffu, 0x3f008010u,
|
|
0x060202ffu, 0x3f009010u, 0x060202ffu, 0x3f00a010u, 0x060202ffu, 0x3f00b010u, 0x060202ffu, 0x3f00c010u,
|
|
0x060202ffu, 0x3f00d010u, 0x060202ffu, 0x3f00e010u, 0x060202ffu, 0x3f00f010u, 0x060202ffu, 0x3f010010u,
|
|
0x7d880084u, 0xbf870025u, 0x060202ffu, 0x3f830000u, 0x060202ffu, 0x3f000030u, 0x060202ffu, 0x3f001030u,
|
|
0x060202ffu, 0x3f002030u, 0x060202ffu, 0x3f003030u, 0x060202ffu, 0x3f004030u, 0x060202ffu, 0x3f005030u,
|
|
0x060202ffu, 0x3f006030u, 0x060202ffu, 0x3f007030u, 0x060202ffu, 0x3f008030u, 0x060202ffu, 0x3f009030u,
|
|
0x060202ffu, 0x3f00a030u, 0x060202ffu, 0x3f00b030u, 0x060202ffu, 0x3f00c030u, 0x060202ffu, 0x3f00d030u,
|
|
0x060202ffu, 0x3f00e030u, 0x060202ffu, 0x3f00f030u, 0x060202ffu, 0x3f010030u, 0xbf820099u, 0x060202ffu,
|
|
0x3f820000u, 0x060202ffu, 0x3f000020u, 0x060202ffu, 0x3f001020u, 0x060202ffu, 0x3f002020u, 0x060202ffu,
|
|
0x3f003020u, 0x060202ffu, 0x3f004020u, 0x060202ffu, 0x3f005020u, 0x060202ffu, 0x3f006020u, 0x060202ffu,
|
|
0x3f007020u, 0x060202ffu, 0x3f008020u, 0x060202ffu, 0x3f009020u, 0x060202ffu, 0x3f00a020u, 0x060202ffu,
|
|
0x3f00b020u, 0x060202ffu, 0x3f00c020u, 0x060202ffu, 0x3f00d020u, 0x060202ffu, 0x3f00e020u, 0x060202ffu,
|
|
0x3f00f020u, 0x060202ffu, 0x3f010020u, 0x7d880085u, 0xbf87004cu, 0x060202ffu, 0x3f850000u, 0x060202ffu,
|
|
0x3f000050u, 0x060202ffu, 0x3f001050u, 0x060202ffu, 0x3f002050u, 0x060202ffu, 0x3f003050u, 0x060202ffu,
|
|
0x3f004050u, 0x060202ffu, 0x3f005050u, 0x060202ffu, 0x3f006050u, 0x060202ffu, 0x3f007050u, 0x060202ffu,
|
|
0x3f008050u, 0x060202ffu, 0x3f009050u, 0x060202ffu, 0x3f00a050u, 0x060202ffu, 0x3f00b050u, 0x060202ffu,
|
|
0x3f00c050u, 0x060202ffu, 0x3f00d050u, 0x060202ffu, 0x3f00e050u, 0x060202ffu, 0x3f00f050u, 0x060202ffu,
|
|
0x3f010050u, 0x7d880088u, 0xbf870001u, 0xbf82ff68u, 0x060202ffu, 0x3f870000u, 0x060202ffu, 0x3f000070u,
|
|
0x060202ffu, 0x3f001070u, 0x060202ffu, 0x3f002070u, 0x060202ffu, 0x3f003070u, 0x060202ffu, 0x3f004070u,
|
|
0x060202ffu, 0x3f005070u, 0x060202ffu, 0x3f006070u, 0x060202ffu, 0x3f007070u, 0x060202ffu, 0x3f008070u,
|
|
0x060202ffu, 0x3f009070u, 0x060202ffu, 0x3f00a070u, 0x060202ffu, 0x3f00b070u, 0x060202ffu, 0x3f00c070u,
|
|
0x060202ffu, 0x3f00d070u, 0x060202ffu, 0x3f00e070u, 0x060202ffu, 0x3f00f070u, 0x060202ffu, 0x3f010070u,
|
|
0xbf820027u, 0x060202ffu, 0x3f840000u, 0x060202ffu, 0x3f000040u, 0x060202ffu, 0x3f001040u, 0x060202ffu,
|
|
0x3f002040u, 0x060202ffu, 0x3f003040u, 0x060202ffu, 0x3f004040u, 0x060202ffu, 0x3f005040u, 0x060202ffu,
|
|
0x3f006040u, 0x060202ffu, 0x3f007040u, 0x060202ffu, 0x3f008040u, 0x060202ffu, 0x3f009040u, 0x060202ffu,
|
|
0x3f00a040u, 0x060202ffu, 0x3f00b040u, 0x060202ffu, 0x3f00c040u, 0x060202ffu, 0x3f00d040u, 0x060202ffu,
|
|
0x3f00e040u, 0x060202ffu, 0x3f00f040u, 0x060202ffu, 0x3f010040u, 0x7d880087u, 0xbf87ff43u, 0xbf82ff8du,
|
|
0x060202ffu, 0x3f860000u, 0x060202ffu, 0x3f000060u, 0x060202ffu, 0x3f001060u, 0x060202ffu, 0x3f002060u,
|
|
0x060202ffu, 0x3f003060u, 0x060202ffu, 0x3f004060u, 0x060202ffu, 0x3f005060u, 0x060202ffu, 0x3f006060u,
|
|
0x060202ffu, 0x3f007060u, 0x060202ffu, 0x3f008060u, 0x060202ffu, 0x3f009060u, 0x060202ffu, 0x3f00a060u,
|
|
0x060202ffu, 0x3f00b060u, 0x060202ffu, 0x3f00c060u, 0x060202ffu, 0x3f00d060u, 0x060202ffu, 0x3f00e060u,
|
|
0x060202ffu, 0x3f00f060u, 0x060202ffu, 0x3f010060u, 0xe0700000u, 0x80010100u, 0xbf810000u,
|
|
};
|
|
const std::string name("large loop-exit shader preserves the original guarded fallback");
|
|
static_cast<void>(verifyGraph(name, code));
|
|
static_cast<void>(recompile(name, code, 32u));
|
|
static_cast<void>(recompile(name, code, 64u));
|
|
}
|
|
|
|
void verifyNestedExitForwarders() {
|
|
constexpr auto none = InvalidControlFlowId;
|
|
constexpr std::array<std::array<std::uint32_t, 2>, 21> targets{{
|
|
{1, none}, {2, none}, {2, 3}, {2, 4}, {5, 6}, {1, 7}, {8, 9}, {10, none},
|
|
{11, 12}, {13, 14}, {1, 15}, {16, none}, {9, none}, {17, 18}, {19, none},
|
|
{16, none}, {none, none}, {16, none}, {15, 20}, {10, none}, {19, none},
|
|
}};
|
|
constexpr std::array<std::uint32_t, 21> order{0, 1, 2, 3, 4, 6, 9, 14, 19, 10, 15, 13, 18, 20, 17, 8, 12, 11, 5, 7, 16};
|
|
std::vector<std::uint32_t> code;
|
|
std::array<std::size_t, targets.size()> starts{};
|
|
std::vector<std::pair<std::size_t, std::uint32_t>> patches;
|
|
for (std::size_t at = 0; at < order.size(); ++at) {
|
|
const auto block = order[at];
|
|
starts[block] = code.size();
|
|
code.insert(code.end(), {0x060202ffu, 0x3f800000u + block * 0x10000u});
|
|
for (std::uint32_t padding = 0; padding < 17; ++padding) code.insert(code.end(), {0x060202ffu, 0x3f000000u + padding * 0x1000u + block * 0x10u});
|
|
const auto next = at + 1 < order.size() ? order[at + 1] : none;
|
|
const auto [first, second] = targets[block];
|
|
if (first == none) {
|
|
code.insert(code.end(), {0xe0700000u, 0x80010100u, 0xbf810000u});
|
|
continue;
|
|
}
|
|
if (second != none) {
|
|
code.push_back(0x7d880080u + block + 3u);
|
|
patches.emplace_back(code.size(), first);
|
|
code.push_back(0xbf870000u);
|
|
}
|
|
const auto branch = second != none ? second : first;
|
|
if (branch != next) {
|
|
patches.emplace_back(code.size(), branch);
|
|
code.push_back(0xbf820000u);
|
|
}
|
|
}
|
|
for (const auto& [instruction, target] : patches) {
|
|
const auto displacement = static_cast<std::int64_t>(starts[target]) - static_cast<std::int64_t>(instruction) - 1;
|
|
require(displacement >= -32768 && displacement <= 32767, "nested exit forwarder test branch is out of range");
|
|
code[instruction] |= static_cast<std::uint32_t>(displacement) & 0xffffu;
|
|
}
|
|
const std::string name("nested exit forwarders keep conditional selections structured");
|
|
static_cast<void>(verifyGraph(name, code));
|
|
static_cast<void>(recompile(name, code, 32u));
|
|
static_cast<void>(recompile(name, code, 64u));
|
|
}
|
|
|
|
void verifyProgram(const Program& program) {
|
|
const std::string name(program.name);
|
|
const auto result = verifyGraph(name, program.code);
|
|
if (program.split == Split::None) {
|
|
require(result.clonedInstructions == 0 && result.routeVariables == 0, name + ": expected no clone or routing, got " + std::to_string(result.clonedInstructions) + " cloned instructions and " + std::to_string(result.routeVariables) + " route variables");
|
|
} else if (program.split == Split::Clone) {
|
|
require(result.clonedInstructions != 0 && result.routeVariables == 0, name + ": expected a clone, got " + std::to_string(result.clonedInstructions) + " cloned instructions and " + std::to_string(result.routeVariables) + " route variables");
|
|
} else {
|
|
require(result.clonedInstructions <= program.clonedLimit && result.routeVariables != 0, name + ": expected routing with at most " + std::to_string(program.clonedLimit) + " cloned instructions, got " + std::to_string(result.clonedInstructions) + " cloned instructions and " + std::to_string(result.routeVariables) + " route variables");
|
|
}
|
|
const auto words = recompile(name, program.code, 64u);
|
|
static_cast<void>(recompile(name, program.code, 32u));
|
|
if (!program.reference.empty()) {
|
|
const auto reference = recompile(name, program.reference, 64u);
|
|
require(words * 10u <= reference * 11u, name + ": " + std::to_string(words) + " SPIR-V words, more than 10% over the " + std::to_string(reference) + " of the program without the entering branch");
|
|
}
|
|
}
|
|
|
|
const std::vector<std::uint32_t> DwordTableGetpcFirst{
|
|
0xbe8c1f00u, 0x800cff0cu, 0x00000058u, 0x820d800du, 0x83928202u, 0xd4c40016u, 0x02000500u, 0x8f128212u, 0xf4000386u, 0x24000000u,
|
|
0xbf8cc07fu, 0x7d820003u, 0x808c0e0cu, 0x828d800du, 0xbefd210cu, 0x7e020281u, 0xbf820003u, 0x7e020282u, 0xbf820001u, 0x7e020283u,
|
|
0xe0700000u, 0x80000100u, 0xbf810000u, 0x00000020u, 0x00000018u, 0x00000010u};
|
|
const std::vector<std::uint32_t> DwordTableIndexFirst{
|
|
0x816ac102u, 0x83ea826au, 0x7e020280u, 0x8f6a826au, 0xbe8e1f00u, 0x800ebc0eu, 0x820f800fu, 0xf4000487u, 0xd4000008u, 0x8aea167eu,
|
|
0xbf8cc07fu, 0x808e120eu, 0x828f800fu, 0xbefd210eu, 0x7e020281u, 0xbf820003u, 0x7e020282u, 0xbf820001u, 0x7e020283u, 0xe0700000u,
|
|
0x80000100u, 0xbf810000u, 0x00000018u, 0x00000010u, 0x00000008u};
|
|
const std::vector<std::uint32_t> LongBranchVcc{
|
|
0x7e020281u, 0xbeea1f00u, 0x806a906au, 0x826b806bu, 0xbefd216au, 0x7e020282u, 0xe0700000u, 0x80000100u, 0xbf810000u};
|
|
const std::vector<std::uint32_t> LongBranchBack{
|
|
0xbe880380u, 0x7e020280u, 0x4a020281u, 0x80089008u, 0xbf0ac008u, 0xbf840004u, 0xbe901f00u, 0x80909410u, 0x82918011u, 0xbefd2110u,
|
|
0xe0700000u, 0x80000100u, 0xbf810000u};
|
|
|
|
std::vector<std::uint32_t> patched(std::vector<std::uint32_t> code, std::size_t word, std::initializer_list<std::uint32_t> replacement) {
|
|
std::copy(replacement.begin(), replacement.end(), code.begin() + static_cast<std::ptrdiff_t>(word));
|
|
return code;
|
|
}
|
|
|
|
std::string refusal(std::span<const std::uint32_t> code) {
|
|
try {
|
|
static_cast<void>(GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(code)));
|
|
} catch (const std::exception& error) {
|
|
return error.what();
|
|
}
|
|
return {};
|
|
}
|
|
|
|
void verifyJumpTable(const std::string& name, std::span<const std::uint32_t> code, std::uint32_t loadPc, const std::vector<std::uint64_t>& values, const std::vector<std::uint32_t>& targets) {
|
|
const auto graph = GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(code));
|
|
require(graph.codeTableLoads.size() == 1u && graph.codeTableLoads.front().programCounter == loadPc && graph.codeTableLoads.front().values == values, name + ": wrong code table");
|
|
std::vector<std::uint32_t> lowered;
|
|
for (const auto& block : graph.blocks) {
|
|
if (block.terminator.indirectPcSgpr != InvalidControlFlowId) lowered.insert(lowered.end(), block.terminator.indirectTargetProgramCounters.begin(), block.terminator.indirectTargetProgramCounters.end());
|
|
}
|
|
std::sort(lowered.begin(), lowered.end());
|
|
require(lowered == targets, name + ": the jump does not reach exactly the table targets");
|
|
}
|
|
|
|
void verifyLongBranch(const std::string& name, std::span<const std::uint32_t> code, std::uint32_t branchPc, std::uint32_t target) {
|
|
const auto decoded = RdnaInstructionDecoder{}.Decode(code);
|
|
const auto graph = GraphBuilder{}.Build(decoded);
|
|
const auto source = std::find_if(graph.blocks.begin(), graph.blocks.end(), [&](const BasicBlock& block) { return block.instructionEnd != block.instructionBegin && decoded.instructions[block.instructionEnd - 1u].programCounter == branchPc; });
|
|
require(source != graph.blocks.end() && source->terminator.kind == TerminatorKind::Branch && graph.FindBlock(source->terminator.trueBlock).startProgramCounter == target, name + ": the jump does not branch to " + std::to_string(target));
|
|
}
|
|
|
|
void verifyNullSwappc() {
|
|
const std::array<std::uint32_t, 2> jump{0xbefd210cu, 0xbf810000u};
|
|
const auto swappc = RdnaInstructionDecoder{}.Decode(jump).instructions.front();
|
|
require(swappc.op == RdnaOpcode::SSetpcB64 && swappc.opcodeId == 0x21u && swappc.destination.kind == RdnaOperandKind::Null &&
|
|
swappc.source0.kind == RdnaOperandKind::ScalarRegister && swappc.source0.reg == 12u, "s_swappc_b64 null, s[12:13] does not decode as s_setpc_b64 s[12:13]");
|
|
const std::array<std::uint32_t, 3> front{0x7e020281u, 0xbefd2106u, 0xbf810000u};
|
|
require(DecodeRdnaFrontProgram(front).code.size() == 2u, "a front program ending in s_swappc_b64 null, s[6:7] does not end there");
|
|
verifyJumpTable("dword jump table after its base", DwordTableGetpcFirst, 0x20u, {0x20u, 0x18u, 0x10u}, {0x3cu, 0x44u, 0x4cu});
|
|
verifyJumpTable("dword jump table after its index", DwordTableIndexFirst, 0x1cu, {0x18u, 0x10u, 0x08u}, {0x38u, 0x40u, 0x48u});
|
|
verifyLongBranch("long branch through vcc", LongBranchVcc, 0x10u, 0x18u);
|
|
verifyLongBranch("backward long branch", LongBranchBack, 0x24u, 0x08u);
|
|
const std::array<std::uint32_t, 3> call{0xbe8c1f00u, 0xbe8e210cu, 0xbf810000u};
|
|
const std::vector<std::tuple<std::string_view, std::vector<std::uint32_t>, std::string_view>> refused{
|
|
{"s_swappc_b64 with a return address", {call.begin(), call.end()}, "unclosable scalar call/return pairing"},
|
|
{"index rewritten after the bound", patched(DwordTableGetpcFirst, 5, {0xbe9203ffu, 0x00000040u}), "unsupported dynamic s_setpc_b64"},
|
|
{"index scaled for two-dword entries", patched(DwordTableGetpcFirst, 7, {0x8f128312u}), "unsupported dynamic s_setpc_b64"},
|
|
{"entry rewritten after the load", patched(DwordTableGetpcFirst, 11, {0xbe8e0380u}), "unsupported dynamic s_setpc_b64"},
|
|
{"base rewritten after the load", patched(DwordTableGetpcFirst, 11, {0xbe8d0380u}), "unsupported dynamic s_setpc_b64"},
|
|
{"borrow other than zero", patched(DwordTableGetpcFirst, 13, {0x828d810du}), "unsupported dynamic s_setpc_b64"},
|
|
{"long branch with a register high half", patched(LongBranchVcc, 3, {0x826b046bu}), "unsupported dynamic s_setpc_b64"},
|
|
{"branch into a long branch", patched(LongBranchVcc, 0, {0xbf850001u}), "does not start in its block"},
|
|
};
|
|
for (const auto& [name, code, message] : refused) {
|
|
const auto error = refusal(code);
|
|
require(error.find(message) != std::string::npos, std::string(name) + ": expected a refusal with '" + std::string(message) + "', got '" + error + "'");
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
int main(int argc, char** argv) {
|
|
try {
|
|
verifyLoopMergeDoesNotEnterNestedSelection();
|
|
} catch (const std::exception& error) {
|
|
std::fprintf(stderr, "%s\n", error.what());
|
|
return 1;
|
|
}
|
|
if (argc > 1) {
|
|
int failures = 0;
|
|
for (int i = 1; i < argc; ++i) {
|
|
try {
|
|
verifyRequest(argv[i]);
|
|
} catch (const std::exception& error) {
|
|
std::fprintf(stderr, "%s: %s\n", argv[i], error.what());
|
|
++failures;
|
|
}
|
|
}
|
|
return failures == 0 ? 0 : 1;
|
|
}
|
|
const std::vector<Program> programs{
|
|
{"nested selections share a loop continuation", "Synthetic loop with two nested selections and a shared continue; the inner selection does not dominate the continuation.",
|
|
{0x060202ffu, 0x3f800000u, 0x060202ffu, 0x3f810000u, 0x7d880084u, 0xbf870001u, 0xbf820018u, 0x060202ffu, 0x3f820000u, 0x7d880085u, 0xbf87000cu, 0x060202ffu, 0x3f850000u, 0x7d880088u, 0xbf870003u, 0x060202ffu, 0x3f870000u, 0xbf82fff0u, 0x060202ffu, 0x3f860000u, 0x7d880089u, 0xbf87fff9u, 0xbf820008u, 0x060202ffu, 0x3f830000u, 0x7d880086u, 0xbf870001u, 0xbf820003u, 0x060202ffu, 0x3f840000u, 0xbf82fff0u, 0x060202ffu, 0x3f880000u, 0xe0700000u, 0x80000100u, 0xbf810000u}, Split::None},
|
|
{"reduced shared exit junction from the public CFG fuzz report", "Public reduced word stream from #2762; synthetic regression, not title data.",
|
|
{0x060202f2u, 0x060202f4u, 0x7d880084u, 0xbf870001u, 0xbf820011u, 0x060202ffu, 0x3f820000u, 0x7d880085u, 0xbf870005u,
|
|
0x060202ffu, 0x3f850000u, 0x7d880088u, 0xbf870009u, 0xbf82fff3u, 0x060202ffu, 0x3f840000u, 0x7d880087u, 0xbf870001u,
|
|
0xbf820003u, 0x060202ffu, 0x3f860000u, 0xbf82ffebu, 0x060202f6u, 0xe0700000u, 0x80010100u, 0xbf810000u}, Split::None},
|
|
{"both selection arms leave through a shared loop exit tail", R"(
|
|
v_mov_b32 v1, 0
|
|
s_mov_b32 s8, 0
|
|
loop:
|
|
s_cmp_lt_u32 s8, 4
|
|
s_cbranch_scc0 done
|
|
s_cmp_eq_u32 s8, s2
|
|
s_cbranch_scc1 right
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccz tail
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_branch latch
|
|
right:
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz tail
|
|
v_add_nc_u32 v1, 2, v1
|
|
latch:
|
|
s_add_u32 s8, s8, 1
|
|
s_branch loop
|
|
tail:
|
|
v_add_nc_u32 v1, 3, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7e020280u, 0xbe880380u, 0xbf0a8408u, 0xbf84000cu, 0xbf060208u, 0xbf850004u, 0x7d880088u, 0xbf860007u, 0x4a020281u,
|
|
0xbf820003u, 0x7d880090u, 0xbf860003u, 0x4a020282u, 0x80088108u, 0xbf82fff3u, 0x4a020283u}), Split::None},
|
|
{"shared tail", R"(
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz outer_else
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz tail
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_branch done
|
|
outer_else:
|
|
v_add_nc_u32 v1, 2, v0
|
|
tail:
|
|
v_add_nc_u32 v1, 3, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7d880090u, 0xbf860004u, 0x7d880088u, 0xbf870003u, 0x4a020081u, 0xbf820002u, 0x4a020082u, 0x4a020283u}), Split::Clone},
|
|
{"shared tail with a selection", R"(
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz outer_else
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz tail
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_branch done
|
|
outer_else:
|
|
v_add_nc_u32 v1, 2, v0
|
|
tail:
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz tail_join
|
|
v_add_nc_u32 v1, 5, v1
|
|
tail_join:
|
|
v_add_nc_u32 v1, 3, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7d880090u, 0xbf860004u, 0x7d880088u, 0xbf870003u, 0x4a020081u, 0xbf820005u, 0x4a020082u, 0x7d880084u, 0xbf860001u, 0x4a020285u, 0x4a020283u}), Split::Clone},
|
|
{"memory access in a single-block loop latch", R"(
|
|
s_mov_b32 s8, 0
|
|
v_mov_b32 v1, 0
|
|
loop:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_add_u32 s8, s8, 1
|
|
s_cmp_lt_u32 s8, 4
|
|
s_cbranch_scc1 loop
|
|
s_endpgm)",
|
|
{0xbe880380u, 0x7e020280u, 0xe0700000u, 0x80000100u, 0x80088108u, 0xbf0a8408u, 0xbf85fffbu, 0xbf810000u}, Split::None},
|
|
{"shared tail in a loop", R"(
|
|
s_mov_b32 s8, 0
|
|
v_mov_b32 v1, 0
|
|
loop:
|
|
v_cmp_gt_u32 vcc, s8, v0
|
|
s_cbranch_vccz outer_else
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz tail
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_branch latch
|
|
outer_else:
|
|
v_add_nc_u32 v1, 2, v1
|
|
tail:
|
|
v_add_nc_u32 v1, 3, v1
|
|
latch:
|
|
s_add_u32 s8, s8, 16
|
|
s_cmp_lt_u32 s8, 64
|
|
s_cbranch_scc1 loop
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0xbe880380u, 0x7e020280u, 0x7d880008u, 0xbf860004u, 0x7d880088u, 0xbf870003u, 0x4a020281u, 0xbf820002u, 0x4a020282u, 0x4a020283u, 0x80089008u, 0xbf0ac008u, 0xbf85fff5u}), Split::Clone},
|
|
{"loop selection arms that leave the iteration rejoin before a join the enclosing selection shares", R"(
|
|
v_mov_b32 v1, 0
|
|
s_mov_b32 s8, 0
|
|
loop:
|
|
s_cmp_eq_u32 s8, s2
|
|
s_cbranch_scc1 join
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccz arm
|
|
v_add_nc_u32 v1, 1, v1
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz away
|
|
arm:
|
|
v_add_nc_u32 v1, 2, v1
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz aside
|
|
join:
|
|
v_add_nc_u32 v1, 3, v1
|
|
s_cmp_lt_u32 s8, 4
|
|
s_cbranch_scc0 done
|
|
s_branch latch
|
|
away:
|
|
v_add_nc_u32 v1, 4, v1
|
|
s_branch latch
|
|
aside:
|
|
v_add_nc_u32 v1, 5, v1
|
|
latch:
|
|
s_add_u32 s8, s8, 1
|
|
s_branch loop
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7e020280u, 0xbe880380u, 0xbf060208u, 0xbf850008u, 0x7d880088u, 0xbf860003u, 0x4a020281u, 0x7d880090u, 0xbf860007u, 0x4a020282u, 0x7d880084u,
|
|
0xbf860006u, 0x4a020283u, 0xbf0a8408u, 0xbf840006u, 0xbf820003u, 0x4a020284u, 0xbf820001u, 0x4a020285u, 0x80088108u, 0xbf82ffedu}), Split::None},
|
|
{"continue beside the inner merge of a nested selection in a loop", R"(
|
|
s_mov_b32 s8, 0
|
|
v_mov_b32 v1, 0
|
|
loop:
|
|
v_cmp_gt_u32 vcc, s8, v0
|
|
s_cbranch_vccz join
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz inner
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccnz latch
|
|
inner:
|
|
v_add_nc_u32 v1, 1, v1
|
|
join:
|
|
v_add_nc_u32 v1, 2, v1
|
|
s_cmp_ge_u32 s8, 64
|
|
s_cbranch_scc1 done
|
|
latch:
|
|
s_add_u32 s8, s8, 16
|
|
s_branch loop
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0xbe880380u, 0x7e020280u, 0x7d880008u, 0xbf860005u, 0x7d880088u, 0xbf870002u, 0x7d880084u, 0xbf870004u, 0x4a020281u, 0x4a020282u, 0xbf09c008u, 0xbf850002u, 0x80089008u, 0xbf82fff4u}), Split::None},
|
|
{"nested selections in a loop whose second latch continues from inside them", R"(
|
|
s_mov_b32 s8, 0
|
|
v_mov_b32 v1, 0
|
|
s_branch loop
|
|
again:
|
|
v_add_nc_u32 v1, 2, v1
|
|
s_add_u32 s8, s8, 16
|
|
loop:
|
|
s_cmp_eq_u32 s8, s2
|
|
s_cbranch_scc1 latch
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccz latch
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccnz again
|
|
latch:
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_add_u32 s8, s8, 16
|
|
s_cmp_lt_u32 s8, 64
|
|
s_cbranch_scc1 loop
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0xbe880380u, 0x7e020280u, 0xbf820002u, 0x4a020282u, 0x80089008u, 0xbf060208u, 0xbf850004u, 0x7d880088u, 0xbf860002u, 0x7d880084u, 0xbf87fff8u, 0x4a020281u, 0x80089008u, 0xbf0ac008u, 0xbf85fff6u}), Split::None},
|
|
{"an inner loop that ends the program or continues its outer loop", R"(
|
|
v_mov_b32 v1, 0
|
|
s_mov_b32 s8, 0
|
|
outer:
|
|
v_add_nc_u32 v1, 2, v1
|
|
inner:
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccz done
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_add_u32 s8, s8, 16
|
|
s_cmp_lt_u32 s8, 64
|
|
s_cbranch_scc1 inner
|
|
s_mov_b32 s8, 0
|
|
s_branch outer
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7e020280u, 0xbe880380u, 0x4a020282u, 0x7d880088u, 0xbf860006u, 0x4a020281u, 0x80089008u, 0xbf0ac008u, 0xbf85fffau, 0xbe880380u, 0xbf82fff7u}), Split::Route, 0},
|
|
{"an inner loop whose body leaves both loops", R"(
|
|
v_mov_b32 v1, 0
|
|
s_mov_b32 s8, 0
|
|
outer:
|
|
s_cmp_lt_u32 s8, 64
|
|
s_cbranch_scc0 done
|
|
inner:
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccz latch
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_add_u32 s8, s8, 16
|
|
s_cmp_eq_u32 s8, s2
|
|
s_cbranch_scc1 done
|
|
s_branch inner
|
|
latch:
|
|
v_add_nc_u32 v1, 2, v1
|
|
s_add_u32 s8, s8, 16
|
|
s_branch outer
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7e020280u, 0xbe880380u, 0xbf0ac008u, 0xbf84000au, 0x7d880088u, 0xbf860005u, 0x4a020281u, 0x80089008u, 0xbf060208u, 0xbf850004u,
|
|
0xbf82fff9u, 0x4a020282u, 0x80089008u, 0xbf82fff4u}), Split::Route, 0},
|
|
{"loop exit to the end of the program beside a kill exit", R"(
|
|
v_mov_b32 v1, 0
|
|
s_mov_b64 s[20:21], exec
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_andn2_b64 s[20:21], s[20:21], vcc
|
|
s_cbranch_scc0 kill
|
|
s_mov_b32 s8, 0
|
|
loop:
|
|
v_add_nc_u32 v1, 1, v1
|
|
v_cmp_eq_u32 vcc, s8, v0
|
|
s_andn2_b64 s[20:21], s[20:21], vcc
|
|
s_cbranch_scc0 kill
|
|
s_add_u32 s8, s8, 1
|
|
s_cmp_lt_u32 s8, 4
|
|
s_cbranch_scc1 loop
|
|
s_mov_b64 exec, s[20:21]
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
kill:
|
|
s_mov_b64 exec, 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0xbe94047eu, 0x7d880084u, 0x8a946a14u, 0xbf84000cu, 0xbe880380u, 0x4a020281u, 0x7d840008u, 0x8a946a14u, 0xbf840007u,
|
|
0x80088108u, 0xbf0a8408u, 0xbf85fff9u, 0xbefe0414u, 0xe0700000u, 0x80000100u, 0xbf810000u, 0xbefe0480u, 0xbf810000u},
|
|
Split::Route, 0},
|
|
{"inner loop exit to a return after the outer loop", R"(
|
|
v_mov_b32 v1, 0
|
|
s_mov_b32 s8, 0
|
|
outer:
|
|
s_mov_b32 s9, 0
|
|
inner:
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_cmp_eq_u32 s9, s2
|
|
s_cbranch_scc1 early_exit
|
|
s_add_u32 s9, s9, 1
|
|
s_cmp_lt_u32 s9, 4
|
|
s_cbranch_scc1 inner
|
|
s_add_u32 s8, s8, 1
|
|
s_cmp_lt_u32 s8, 4
|
|
s_cbranch_scc1 outer
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
early_exit:
|
|
v_add_nc_u32 v1, 2, v1
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0xbe880380u, 0xbe890380u, 0x4a020281u, 0xbf060209u, 0xbf850009u, 0x80098109u, 0xbf0a8409u, 0xbf85fffau, 0x80088108u,
|
|
0xbf0a8408u, 0xbf85fff6u, 0xe0700000u, 0x80000100u, 0xbf810000u, 0x4a020282u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route, 0},
|
|
{"loop exit tail beside the continue block", R"(
|
|
v_mov_b32 v1, 0
|
|
s_mov_b32 s8, 0
|
|
loop:
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_cmp_eq_u32 s8, s2
|
|
s_cbranch_scc0 latch
|
|
v_add_nc_u32 v1, 2, v1
|
|
s_branch done
|
|
latch:
|
|
s_add_u32 s8, s8, 1
|
|
s_cmp_lt_u32 s8, 4
|
|
s_cbranch_scc1 loop
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7e020280u, 0xbe880380u, 0x4a020281u, 0xbf060208u, 0xbf840002u, 0x4a020282u, 0xbf820003u, 0x80088108u, 0xbf0a8408u, 0xbf85fff8u}), Split::None},
|
|
{"shared early exit", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz join
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz early_exit
|
|
v_add_nc_u32 v1, 1, v0
|
|
join:
|
|
s_mov_b64 exec, s[8:9]
|
|
v_cmp_gt_u32 vcc, 32, v0
|
|
s_cbranch_vccz early_exit
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
early_exit:
|
|
s_mov_b64 exec, 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf880003u, 0x7d880084u, 0xbf860007u, 0x4a020081u, 0xbefe0408u, 0x7d8800a0u, 0xbf860003u, 0xe0700000u, 0x80000100u, 0xbf810000u, 0xbefe0480u, 0xbf810000u}, Split::Clone},
|
|
{"short-circuit condition", R"(
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz body
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz done
|
|
body:
|
|
v_add_nc_u32 v1, 1, v0
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz done
|
|
v_add_nc_u32 v1, 2, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7d880090u, 0xbf860002u, 0x7d880088u, 0xbf870004u, 0x4a020081u, 0x7d880084u, 0xbf860001u, 0x4a020282u}), Split::Clone},
|
|
{"entered region with an expensive body", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz body
|
|
v_add_nc_u32 v1, 1, v0
|
|
v_cmp_gt_u32 vcc, 8, v1
|
|
s_cbranch_vccnz done
|
|
body:
|
|
s_mov_b32 s10, 0
|
|
loop:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
s_add_u32 s10, s10, 1
|
|
s_cmp_lt_u32 s10, 4
|
|
s_cbranch_scc1 loop
|
|
s_mov_b64 exec, s[8:9]
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
done:
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf880003u, 0x4a020081u, 0x7d880288u, 0xbf870015u, 0xbe8a0380u, 0xe0300000u, 0x80000200u,
|
|
0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u,
|
|
0x7e160501u, 0x4a02020bu, 0x800a810au, 0xbf0a840au, 0xbf85ffefu, 0xbefe0408u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route, 0,
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf800000u, 0x4a020081u, 0x7d880288u, 0xbf870015u, 0xbe8a0380u, 0xe0300000u, 0x80000200u,
|
|
0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u,
|
|
0x7e160501u, 0x4a02020bu, 0x800a810au, 0xbf0a840au, 0xbf85ffefu, 0xbefe0408u, 0xe0700000u, 0x80000100u, 0xbf810000u}},
|
|
{"return tail past an entered region", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz body
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz done
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_mov_b64 exec, s[8:9]
|
|
body:
|
|
s_mov_b32 s10, 0
|
|
loop:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
s_add_u32 s10, s10, 1
|
|
s_cmp_lt_u32 s10, 4
|
|
s_cbranch_scc1 loop
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
done:
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf880004u, 0x7d880088u, 0xbf870016u, 0x4a020081u, 0xbefe0408u, 0xbe8a0380u, 0xe0300000u,
|
|
0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u,
|
|
0x3a020303u, 0x7e160501u, 0x4a02020bu, 0x800a810au, 0xbf0a840au, 0xbf85ffefu, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Clone},
|
|
{"entered region in a loop", R"(
|
|
s_mov_b32 s12, 0
|
|
v_mov_b32 v1, 0
|
|
outer:
|
|
v_cmp_gt_u32 vcc, s12, v0
|
|
s_cbranch_vccz body
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz join
|
|
v_add_nc_u32 v1, 1, v1
|
|
body:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
join:
|
|
v_add_nc_u32 v1, 2, v1
|
|
s_add_u32 s12, s12, 16
|
|
s_cmp_lt_u32 s12, 64
|
|
s_cbranch_scc0 exit_loop
|
|
s_branch outer
|
|
exit_loop:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0xbe8c0380u, 0x7e020280u, 0x7d88000cu, 0xbf860003u, 0x7d880088u, 0xbf87000fu, 0x4a020281u, 0xe0300000u, 0x80000200u, 0x10060302u,
|
|
0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u,
|
|
0x4a02020bu, 0x4a020282u, 0x800c900cu, 0xbf0ac00cu, 0xbf840001u, 0xbf82ffe8u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route},
|
|
{"nested entered regions", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz outer_body
|
|
v_add_nc_u32 v1, 1, v0
|
|
v_cmp_gt_u32 vcc, 8, v1
|
|
s_cbranch_vccnz done
|
|
outer_body:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
s_cmp_eq_u32 s11, 64
|
|
s_cbranch_scc0 inner_body
|
|
v_add_nc_u32 v1, 2, v1
|
|
v_cmp_gt_u32 vcc, 4, v1
|
|
s_cbranch_vccz done
|
|
inner_body:
|
|
s_mov_b32 s10, 0
|
|
inner_loop:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
s_add_u32 s10, s10, 1
|
|
s_cmp_lt_u32 s10, 4
|
|
s_cbranch_scc1 inner_loop
|
|
s_mov_b64 exec, s[8:9]
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
done:
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf880003u, 0x4a020081u, 0x7d880288u, 0xbf87001au, 0xe0300000u, 0x80000200u, 0x10060302u,
|
|
0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xbf06c00bu, 0xbf840003u, 0x4a020282u, 0x7d880284u, 0xbf86000eu, 0xbe8a0380u,
|
|
0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0x800a810au, 0xbf0a840au, 0xbf85fff6u,
|
|
0xbefe0408u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route},
|
|
{"shared body after an overwritten condition", R"(
|
|
v_mov_b32 v1, 0
|
|
s_cmp_eq_u32 s2, 64
|
|
s_cbranch_scc0 shared
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_cmp_eq_u32 s3, 0
|
|
s_cbranch_scc1 done
|
|
shared:
|
|
s_cbranch_scc1 store
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
store:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
done:
|
|
s_endpgm)",
|
|
{0x7e020280u, 0xbf06c002u, 0xbf840003u, 0x4a020081u, 0xbf068003u, 0xbf850011u, 0xbf85000eu, 0xe0300000u, 0x80000200u, 0x10060302u,
|
|
0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u,
|
|
0x4a02020bu, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route},
|
|
{"entered region beside early returns", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz body
|
|
v_add_nc_u32 v1, 1, v0
|
|
v_cmp_gt_u32 vcc, 8, v1
|
|
s_cbranch_vccnz tail
|
|
v_readfirstlane_b32 s11, v1
|
|
s_cmp_eq_u32 s11, 12
|
|
s_cbranch_scc0 done
|
|
s_cmp_eq_u32 s11, 38
|
|
s_cbranch_scc1 done
|
|
body:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
tail:
|
|
v_add_nc_u32 v1, 1, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 2, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 3, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 4, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 5, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 6, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 7, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 8, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 9, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 10, v1
|
|
v_xor_b32 v1, v3, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbf860008u, 0x4a020081u, 0x7d880288u, 0xbf870013u, 0x7e160501u, 0xbf068c0bu, 0xbf840024u, 0xbf06a60bu,
|
|
0xbf850022u, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u,
|
|
0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0x4a020281u, 0x3a020303u, 0x4a020282u, 0x3a020303u, 0x4a020283u,
|
|
0x3a020303u, 0x4a020284u, 0x3a020303u, 0x4a020285u, 0x3a020303u, 0x4a020286u, 0x3a020303u, 0x4a020287u, 0x3a020303u, 0x4a020288u,
|
|
0x3a020303u, 0x4a020289u, 0x3a020303u, 0x4a02028au, 0x3a020303u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route, 4},
|
|
{"selection arm that only returns", R"(
|
|
s_cbranch_execz inner
|
|
s_cbranch_scc1 done
|
|
inner:
|
|
v_cmp_gt_u32 vcc, 3, v1
|
|
s_cbranch_vccnz second
|
|
v_mul_f32 v3, v2, v1
|
|
s_cbranch_vccz join
|
|
second:
|
|
s_cbranch_vccnz done
|
|
join:
|
|
v_add_nc_u32 v1, 2, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0xbf880001u, 0xbf850006u, 0x7d880283u, 0xbf870002u, 0x10060302u, 0xbf860001u, 0xbf870001u, 0x4a020282u, 0xe0700000u, 0x80000100u,
|
|
0xbf810000u},
|
|
Split::Route, 9},
|
|
{"shared early exit beside an expensive region", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz join
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz early_exit
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
join:
|
|
s_mov_b64 exec, s[8:9]
|
|
v_cmp_gt_u32 vcc, 32, v0
|
|
s_cbranch_vccz early_exit
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
early_exit:
|
|
s_mov_b64 exec, 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf88000eu, 0x7d880084u, 0xbf860019u, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u,
|
|
0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0xbefe0408u, 0x7d8800a0u,
|
|
0xbf86000au, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0700000u, 0x80000100u,
|
|
0xbf810000u, 0xbefe0480u, 0xbf810000u},
|
|
Split::Clone},
|
|
{"early exit arm beside a branch to the parent's merge", R"(
|
|
v_mov_b32 v1, 0
|
|
s_cmp_eq_u32 s2, 64
|
|
s_cbranch_scc1 join
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_cmp_eq_u32 s3, 0
|
|
s_cbranch_scc1 early_exit
|
|
join:
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz store
|
|
v_add_nc_u32 v1, 2, v1
|
|
store:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
early_exit:
|
|
s_mov_b64 exec, 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0xbf06c002u, 0xbf850003u, 0x4a020081u, 0xbf068003u, 0xbf850006u, 0x7d880088u, 0xbf870001u, 0x4a020282u, 0xe0700000u,
|
|
0x80000100u, 0xbf810000u, 0xbefe0480u, 0xbf810000u},
|
|
Split::None},
|
|
{"early exit arm beside a parent arm that reaches the same merge", R"(
|
|
v_mov_b32 v1, 0
|
|
s_cmp_eq_u32 s2, 64
|
|
s_cbranch_scc1 other
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_cmp_eq_u32 s3, 0
|
|
s_cbranch_scc1 early_exit
|
|
join:
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz store
|
|
v_add_nc_u32 v1, 2, v1
|
|
store:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
other:
|
|
v_add_nc_u32 v1, 3, v0
|
|
s_branch join
|
|
early_exit:
|
|
s_mov_b64 exec, 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0xbf06c002u, 0xbf850009u, 0x4a020081u, 0xbf068003u, 0xbf850008u, 0x7d880088u, 0xbf870001u, 0x4a020282u, 0xe0700000u,
|
|
0x80000100u, 0xbf810000u, 0x4a020083u, 0xbf82fff8u, 0xbefe0480u, 0xbf810000u},
|
|
Split::None},
|
|
{"return block entered from the return block after it", R"(
|
|
v_mov_b32 v1, 0
|
|
s_cmp_eq_u32 s2, 64
|
|
s_cbranch_scc0 later
|
|
v_add_nc_u32 v1, 1, v0
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
middle:
|
|
v_add_nc_u32 v1, 2, v0
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
later:
|
|
s_cmp_eq_u32 s3, 0
|
|
s_cbranch_scc1 middle
|
|
v_add_nc_u32 v1, 3, v0
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0xbf06c002u, 0xbf840008u, 0x4a020081u, 0xe0700000u, 0x80000100u, 0xbf810000u, 0x4a020082u, 0xe0700000u, 0x80000100u,
|
|
0xbf810000u, 0xbf068003u, 0xbf85fffau, 0x4a020083u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::None},
|
|
{"loop body between an early return and the loop test", R"(
|
|
s_mov_b32 s8, 0
|
|
v_mov_b32 v1, 0
|
|
s_cmp_eq_u32 s2, 64
|
|
s_cbranch_scc0 test
|
|
v_add_nc_u32 v1, 1, v0
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
body:
|
|
v_add_nc_u32 v1, 2, v1
|
|
s_add_u32 s8, s8, 16
|
|
test:
|
|
s_cmp_lt_u32 s8, 64
|
|
s_cbranch_scc1 body
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0xbe880380u, 0x7e020280u, 0xbf06c002u, 0xbf840006u, 0x4a020081u, 0xe0700000u, 0x80000100u, 0xbf810000u, 0x4a020282u, 0x80089008u,
|
|
0xbf0ac008u, 0xbf85fffcu, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::None},
|
|
{"dword jump table after its base", R"(
|
|
s_getpc_b64 s[12:13]
|
|
base:
|
|
s_add_u32 s12, s12, table - base
|
|
s_addc_u32 s13, s13, 0
|
|
s_min_u32 s18, s2, 2
|
|
v_cmp_gt_u32 s[22:23], v0, s2
|
|
s_lshl_b32 s18, s18, 2
|
|
s_load_dword s14, s[12:13], s18
|
|
s_waitcnt lgkmcnt(0)
|
|
v_cmp_lt_u32 vcc, s3, v0
|
|
s_sub_u32 s12, s12, s14
|
|
s_subb_u32 s13, s13, 0
|
|
s_swappc_b64 null, s[12:13]
|
|
case0:
|
|
v_mov_b32 v1, 1
|
|
s_branch done
|
|
case1:
|
|
v_mov_b32 v1, 2
|
|
s_branch done
|
|
case2:
|
|
v_mov_b32 v1, 3
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
table:
|
|
.long table - case0, table - case1, table - case2)",
|
|
DwordTableGetpcFirst, Split::None},
|
|
{"dword jump table after its index", R"(
|
|
s_add_i32 vcc_lo, s2, -1
|
|
s_min_u32 vcc_lo, vcc_lo, 2
|
|
v_mov_b32 v1, 0
|
|
s_lshl_b32 vcc_lo, vcc_lo, 2
|
|
s_getpc_b64 s[14:15]
|
|
base:
|
|
s_add_u32 s14, s14, table - base - 8
|
|
s_addc_u32 s15, s15, 0
|
|
s_load_dword s18, s[14:15], vcc_lo offset:8
|
|
s_andn2_b64 vcc, exec, s[22:23]
|
|
s_waitcnt lgkmcnt(0)
|
|
s_sub_u32 s14, s14, s18
|
|
s_subb_u32 s15, s15, 0
|
|
s_swappc_b64 null, s[14:15]
|
|
case0:
|
|
v_mov_b32 v1, 1
|
|
s_branch done
|
|
case1:
|
|
v_mov_b32 v1, 2
|
|
s_branch done
|
|
case2:
|
|
v_mov_b32 v1, 3
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
table:
|
|
.long table - 8 - case0, table - 8 - case1, table - 8 - case2)",
|
|
DwordTableIndexFirst, Split::None},
|
|
{"long branch through vcc", R"(
|
|
v_mov_b32 v1, 1
|
|
s_getpc_b64 vcc
|
|
base:
|
|
s_add_u32 vcc_lo, vcc_lo, target - base
|
|
s_addc_u32 vcc_hi, vcc_hi, 0
|
|
s_swappc_b64 null, vcc
|
|
v_mov_b32 v1, 2
|
|
target:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
LongBranchVcc, Split::None},
|
|
{"backward long branch", R"(
|
|
s_mov_b32 s8, 0
|
|
v_mov_b32 v1, 0
|
|
loop:
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_add_u32 s8, s8, 16
|
|
s_cmp_lt_u32 s8, 64
|
|
s_cbranch_scc0 done
|
|
s_getpc_b64 s[16:17]
|
|
base:
|
|
s_sub_u32 s16, s16, base - loop
|
|
s_subb_u32 s17, s17, 0
|
|
s_swappc_b64 null, s[16:17]
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
LongBranchBack, Split::None},
|
|
};
|
|
int failures = 0;
|
|
for (const auto& program : programs) {
|
|
try {
|
|
verifyProgram(program);
|
|
} catch (const std::exception& error) {
|
|
std::fprintf(stderr, "%s\n%.*s\n", error.what(), static_cast<int>(program.source.size()), program.source.data());
|
|
++failures;
|
|
}
|
|
}
|
|
try {
|
|
verifyNullSwappc();
|
|
verifyLargeLoopExitFallback();
|
|
verifyNestedExitForwarders();
|
|
} catch (const std::exception& error) {
|
|
std::fprintf(stderr, "%s\n", error.what());
|
|
++failures;
|
|
}
|
|
if (failures != 0) return 1;
|
|
std::puts("Control flow structurization tests passed");
|
|
return 0;
|
|
}
|