mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-22 00:22:57 -05:00
Merge pull request #5089 from fdfawef324/feat/recompiler-waterfall-descriptors
feat(recompiler): plan waterfall loops over per-lane T#s as non-uniform tables
This commit is contained in:
9 files changed
+767
-16
No files matched your search
@@ -24,6 +24,7 @@
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <set>
|
||||
#include <span>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
@@ -358,6 +359,9 @@ void verifyEntryRoundTrip() {
|
||||
auto previousVersion = file;
|
||||
previousVersion[4] = static_cast<std::byte>(ShaderDiskCache::FormatVersion - 1u);
|
||||
require(ShaderDiskCache::DecodeEntry(previousVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the previous format version loads");
|
||||
auto upstreamVersion = file;
|
||||
upstreamVersion[4] = std::byte{30};
|
||||
require(ShaderDiskCache::DecodeEntry(upstreamVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the upstream resource plan format loads");
|
||||
auto incompatible = variant;
|
||||
incompatible.artifact.runtimeAbiVersion = RuntimeAbi::Version + 1u;
|
||||
const auto incompatibleFile = ShaderDiskCache::EncodeEntry(key, incompatible);
|
||||
@@ -1554,6 +1558,9 @@ void verifyPlanEntryDamage(const std::vector<std::byte>& key, const IrResourcePl
|
||||
auto previousFormat = file;
|
||||
previousFormat[4] = static_cast<std::byte>(ShaderDiskCache::FormatVersion - 1u);
|
||||
require(ShaderDiskCache::DecodePlanEntry(previousFormat, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry of the previous format loads");
|
||||
auto upstreamFormat = file;
|
||||
upstreamFormat[4] = std::byte{30};
|
||||
require(ShaderDiskCache::DecodePlanEntry(upstreamFormat, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry with the upstream resource plan format loads");
|
||||
auto otherSource = file;
|
||||
otherSource[8] ^= std::byte{1};
|
||||
require(ShaderDiskCache::DecodePlanEntry(otherSource, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry of another source version loads");
|
||||
@@ -1621,7 +1628,23 @@ void verifyMalformedPlans() {
|
||||
require(!ShaderDiskCache::EncodePlan(emptied, bytes), "a plan with an empty value slot was encoded");
|
||||
}
|
||||
|
||||
void verifyIndirectImagePlanModes() {
|
||||
std::set<std::vector<std::byte>> tableEncodings;
|
||||
for (std::uint32_t flags = 0u; flags < 8u; ++flags) {
|
||||
auto plan = handBuiltPlan();
|
||||
auto& table = *plan.descriptorSources[1].indirectImage;
|
||||
table.laneKey = (flags & 1u) != 0u;
|
||||
table.indexed = (flags & 2u) != 0u;
|
||||
table.implicitDerivatives = (flags & 4u) != 0u;
|
||||
const auto bytes = encodedPlan(plan);
|
||||
require(tableEncodings.insert(bytes).second, "indirect image table modes share a disk plan encoding");
|
||||
IrResourcePlan decoded;
|
||||
require(ShaderDiskCache::DecodePlan(bytes, decoded) && samePlan(plan, decoded), "indirect image table modes changed in a disk plan round trip");
|
||||
}
|
||||
}
|
||||
|
||||
void verifyPlanRoundTrip() {
|
||||
verifyIndirectImagePlanModes();
|
||||
const auto handBuilt = handBuiltPlan();
|
||||
const auto handBytes = encodedPlan(handBuilt);
|
||||
IrResourcePlan handDecoded;
|
||||
@@ -1881,6 +1904,13 @@ void verifyDefaultDirectory(const char* self) {
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
try {
|
||||
if (argc == 2 && std::string_view(argv[1]) == "--resource-plan-codec") {
|
||||
verifyResultRoundTrip();
|
||||
verifyEntryRoundTrip();
|
||||
verifyIndirectImagePlanModes();
|
||||
std::cout << "resource plan codec tests passed\n";
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::string_view(argv[1]) == "--load") return runLoadingProcess();
|
||||
if (argc == 2 && std::string_view(argv[1]) == "--no-failure-memo") return runWithoutFailureMemo();
|
||||
if (argc == 2 && std::string_view(argv[1]).starts_with("--load-plan")) return runPlanLoadingProcess(argv[1]);
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
|
||||
#include "Recompiler.hpp"
|
||||
#include "VulkanTestDevice.hpp"
|
||||
#include <spirv/unified1/spirv.hpp>
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <iostream>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
using AgcDriver::Graphics::Require;
|
||||
using namespace ShaderRecompiler;
|
||||
|
||||
constexpr std::uint32_t Records = 4;
|
||||
constexpr std::uint32_t Textures = 3;
|
||||
|
||||
struct alignas(4096) GuestData {
|
||||
std::array<std::array<std::uint32_t, 8>, Records> table{};
|
||||
std::array<std::uint32_t, 64> output{};
|
||||
};
|
||||
|
||||
alignas(256) std::array<std::array<std::uint32_t, 64>, Textures> textures{};
|
||||
std::array<GuestData, 3> guest;
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 46> WholeWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u,
|
||||
0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au,
|
||||
0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu,
|
||||
0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu,
|
||||
0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u,
|
||||
0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 50> PartialWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x36200087u, 0xbe9c037eu, 0x7daa2087u, 0xe0382000u,
|
||||
0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u,
|
||||
0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u,
|
||||
0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u,
|
||||
0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u,
|
||||
0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu, 0xe0702000u,
|
||||
0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 50> PartialWave64{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x36200087u, 0xbe9c047eu, 0x7daa2087u, 0xe0382000u,
|
||||
0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94047eu, 0xbe961414u, 0xd7600008u, 0x00002d02u,
|
||||
0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u,
|
||||
0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98047eu, 0x7da40408u,
|
||||
0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u,
|
||||
0x00020a0eu, 0xbe9a047eu, 0x8a941a14u, 0xbefe0418u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe041cu, 0xe0702000u,
|
||||
0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 50> GuardedWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e1402c1u, 0xbe9c037eu, 0x7da80202u, 0xbf880026u, 0xe0382000u,
|
||||
0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u,
|
||||
0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u,
|
||||
0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u,
|
||||
0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u,
|
||||
0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu, 0xe0702000u,
|
||||
0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 59> StaleWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x7e040208u, 0x7e060209u, 0x7e08020au, 0x7e0a020bu,
|
||||
0x7e0c020cu, 0x7e0e020du, 0x7e10020eu, 0x7e12020fu, 0x36200087u, 0xbe9c037eu, 0x7daa2087u, 0xe0382000u,
|
||||
0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbefe031cu, 0xbe94037eu, 0xbe961314u, 0xd7600008u,
|
||||
0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu,
|
||||
0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu,
|
||||
0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu,
|
||||
0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu,
|
||||
0xe0702000u, 0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 37> FirstLaneWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u,
|
||||
0xbf8c3f70u, 0xbe94037eu, 0x7e100502u, 0x7e120503u, 0x7e140504u, 0x7e160505u, 0x7e180506u, 0x7e1a0507u,
|
||||
0x7e1c0508u, 0x7e1e0509u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu,
|
||||
0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a7e1a18u, 0xbf85ffeau,
|
||||
0xbf8c3f70u, 0xbefe0314u, 0xe0702000u, 0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 42> Compare64Wave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u,
|
||||
0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au,
|
||||
0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu,
|
||||
0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7de40408u, 0x7de4080au, 0x7de40c0cu, 0x7de4100eu,
|
||||
0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe4u, 0xbf8c3f70u, 0xe0702000u,
|
||||
0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 68> MaskedStaleWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x7e040280u, 0x7e060280u, 0x7e080280u, 0x7e0a0280u,
|
||||
0x7e0c0280u, 0x7e0e0280u, 0x7e100280u, 0x7e120280u, 0x36200087u, 0xbe9c037eu, 0x7da42087u, 0x7e040208u,
|
||||
0x7e060209u, 0x7e08020au, 0x7e0a020bu, 0x7e0c020cu, 0x7e0e020du, 0x7e10020eu, 0x7e12020fu, 0xbefe031cu,
|
||||
0x7daa2087u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbefe031cu, 0xbe94037eu,
|
||||
0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu,
|
||||
0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu,
|
||||
0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du,
|
||||
0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u,
|
||||
0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 45> SevenDwordsWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u,
|
||||
0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au,
|
||||
0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu,
|
||||
0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu,
|
||||
0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u,
|
||||
0xbf85ffe1u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 46> OutsideWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u,
|
||||
0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au,
|
||||
0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu,
|
||||
0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu,
|
||||
0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xf0000f08u,
|
||||
0x00020a0eu, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 46> StoreWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u,
|
||||
0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au,
|
||||
0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu,
|
||||
0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu,
|
||||
0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0200f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u,
|
||||
0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 46> LodWave32{
|
||||
0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u,
|
||||
0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au,
|
||||
0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu,
|
||||
0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu,
|
||||
0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf1800308u, 0x00820a0eu, 0xbe9a037eu, 0x8a141a14u,
|
||||
0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u,
|
||||
};
|
||||
|
||||
std::array<std::uint32_t, 4> Buffer(const void* pointer, std::uint32_t stride, std::uint32_t records) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(pointer);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>(address >> 32u) | (stride << 16u), records, 0x01016facu};
|
||||
}
|
||||
|
||||
std::array<std::uint32_t, 8> Image(std::uint32_t texture) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(textures[texture].data());
|
||||
const std::uint32_t format = texture == 1u ? 22u : 20u;
|
||||
return {static_cast<std::uint32_t>(address >> 8u), static_cast<std::uint32_t>(address >> 40u) | (format << 20u) | (3u << 30u), 7u, 0x90000facu, 0u, 0u, 0u, 0u};
|
||||
}
|
||||
|
||||
std::uint32_t TextureOf(std::uint32_t iteration, std::uint32_t key) {
|
||||
return key == Records - 1u - iteration ? Textures : (key + iteration) % Textures;
|
||||
}
|
||||
|
||||
bool DecoratesNonUniform(std::span<const std::uint32_t> words) {
|
||||
for (std::size_t index = 5u; index < words.size();) {
|
||||
const auto count = words[index] >> 16u;
|
||||
if (count == 0u) return false;
|
||||
if ((words[index] & 0xffffu) == spv::OpDecorate && count >= 3u && index + 2u < words.size() && words[index + 2u] == spv::DecorationNonUniform) return true;
|
||||
index += count;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
RecompileRequest Request(const AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t wave, std::uint32_t lanes, std::span<const std::uint32_t> userData, const std::array<MemoryRegion, 1>& regions) {
|
||||
const ShaderComputeStageInfo compute{{lanes, 1u, 1u}, 0u, {false, false, false}, false, 1u};
|
||||
return {{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0u, {}}, {wave, 0u, userData, compute, std::nullopt, std::nullopt, regions}, device.Target(), {0u, 0u, 0u, 128u}};
|
||||
}
|
||||
|
||||
std::array<std::uint32_t, 20> UserData(GuestData& data, std::uint32_t lanes) {
|
||||
const auto table = Buffer(data.table.data(), 32u, Records);
|
||||
const auto output = Buffer(data.output.data(), 4u, lanes);
|
||||
const auto fixed = Image(0u);
|
||||
constexpr std::array<std::uint32_t, 4> sampler{0x00000092u, 0x00fff000u, 0u, 0u};
|
||||
std::array<std::uint32_t, 20> userData{};
|
||||
std::copy(table.begin(), table.end(), userData.begin());
|
||||
std::copy(output.begin(), output.end(), userData.begin() + 4u);
|
||||
std::copy(fixed.begin(), fixed.end(), userData.begin() + 8u);
|
||||
std::copy(sampler.begin(), sampler.end(), userData.begin() + 16u);
|
||||
return userData;
|
||||
}
|
||||
|
||||
void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t wave, bool partial, const std::string& what) {
|
||||
for (std::uint32_t texture = 0u; texture < Textures; ++texture) {
|
||||
for (std::uint32_t x = 0u; x < textures[texture].size(); ++x) textures[texture][x] = 0x1000u * (texture + 1u) + x;
|
||||
}
|
||||
const auto lanes = wave == 64u ? 64u : std::min(32u, device.Target().subgroupSize);
|
||||
CompiledShaderArtifact artifact;
|
||||
for (std::uint32_t iteration = 0u; iteration < guest.size(); ++iteration) {
|
||||
auto& data = guest[iteration];
|
||||
for (std::uint32_t key = 0u; key < Records; ++key) {
|
||||
const auto texture = TextureOf(iteration, key);
|
||||
data.table[key] = texture == Textures ? std::array<std::uint32_t, 8>{} : Image(texture);
|
||||
}
|
||||
data.output.fill(0xdeadbeefu);
|
||||
const auto userData = UserData(data, lanes);
|
||||
const std::array regions{MemoryRegion{reinterpret_cast<std::uintptr_t>(&data), std::as_bytes(std::span(&data, 1u))}};
|
||||
const auto shader = Recompile(Request(device, code, wave, lanes, userData, regions));
|
||||
Require(DecoratesNonUniform(shader.spirv.Words()), what + ": the per-lane table slot is not decorated NonUniform");
|
||||
if (artifact.variantId == 0u) artifact = shader;
|
||||
else Require(shader.cacheHit && shader.variantId == artifact.variantId, what + ": a new table changed the compiled artifact");
|
||||
device.Dispatch(shader, 1u, 1u, 1u);
|
||||
device.SubmitRecorded(false);
|
||||
}
|
||||
device.WaitIdle();
|
||||
for (std::uint32_t iteration = 0u; iteration < guest.size(); ++iteration) {
|
||||
for (std::uint32_t lane = 0u; lane < lanes; ++lane) {
|
||||
const auto texture = TextureOf(iteration, lane % Records);
|
||||
const bool skipped = partial && lane % 8u == 7u;
|
||||
const auto expected = skipped || texture == Textures ? 0u : textures[texture][lane / 4u];
|
||||
const auto actual = guest[iteration].output[lane];
|
||||
Require(actual == expected, what + ": iteration " + std::to_string(iteration) + " lane " + std::to_string(lane) + " read " + std::to_string(actual) + ", expected " + std::to_string(expected));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void RequireRefused(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::string& what) {
|
||||
auto& data = guest[0];
|
||||
const auto userData = UserData(data, 32u);
|
||||
const std::array regions{MemoryRegion{reinterpret_cast<std::uintptr_t>(&data), std::as_bytes(std::span(&data, 1u))}};
|
||||
auto request = Request(device, code, 32u, std::min(32u, device.Target().subgroupSize), userData, regions);
|
||||
request.useCache = false;
|
||||
bool refused = false;
|
||||
try {
|
||||
static_cast<void>(Recompile(request));
|
||||
} catch (const std::exception& error) {
|
||||
refused = std::string_view(error.what()).find("GetImageResource dword 0 is not a valid runtime value") != std::string_view::npos;
|
||||
}
|
||||
Require(refused, what + " was accepted");
|
||||
}
|
||||
|
||||
void RunWithoutRecords(AgcDriver::VulkanDevice& device, const std::array<std::uint32_t, 4>& table, const std::string& what) {
|
||||
auto& data = guest[0];
|
||||
data.output.fill(0xdeadbeefu);
|
||||
const auto lanes = std::min(32u, device.Target().subgroupSize);
|
||||
auto userData = UserData(data, lanes);
|
||||
std::copy(table.begin(), table.end(), userData.begin());
|
||||
const std::array regions{MemoryRegion{reinterpret_cast<std::uintptr_t>(&data), std::as_bytes(std::span(&data, 1u))}};
|
||||
std::string failure;
|
||||
try {
|
||||
const auto shader = Recompile(Request(device, GuardedWave32, 32u, lanes, userData, regions));
|
||||
device.Dispatch(shader, 1u, 1u, 1u);
|
||||
device.SubmitRecorded(false);
|
||||
device.WaitIdle();
|
||||
} catch (const std::exception& error) {
|
||||
failure = error.what();
|
||||
}
|
||||
Require(failure.empty(), what + " failed: " + failure.substr(0u, 300u));
|
||||
for (std::uint32_t lane = 0u; lane < lanes; ++lane) {
|
||||
const auto actual = data.output[lane];
|
||||
Require(actual == 0xffffffffu, what + ": lane " + std::to_string(lane) + " read " + std::to_string(actual) + ", expected the value it holds outside the loop");
|
||||
}
|
||||
}
|
||||
|
||||
void RunLod(AgcDriver::VulkanDevice& device, const std::array<std::uint32_t, Records>& records, bool refused, const std::string& what) {
|
||||
auto& data = guest[0];
|
||||
for (std::uint32_t key = 0u; key < Records; ++key) data.table[key] = records[key] == Textures ? std::array<std::uint32_t, 8>{} : Image(records[key]);
|
||||
const auto userData = UserData(data, 32u);
|
||||
const std::array regions{MemoryRegion{reinterpret_cast<std::uintptr_t>(&data), std::as_bytes(std::span(&data, 1u))}};
|
||||
auto request = Request(device, LodWave32, 32u, std::min(32u, device.Target().subgroupSize), userData, regions);
|
||||
request.useCache = false;
|
||||
std::string failure;
|
||||
try {
|
||||
const auto shader = Recompile(request);
|
||||
Require(DecoratesNonUniform(shader.spirv.Words()), what + ": the per-lane table slot is not decorated NonUniform");
|
||||
} catch (const std::exception& error) {
|
||||
failure = error.what();
|
||||
}
|
||||
if (refused) Require(failure.find("different runtime modes under an implicit-derivative access") != std::string::npos, what + " was not refused: " + failure.substr(0u, 300u));
|
||||
else Require(failure.empty(), what + " failed: " + failure.substr(0u, 300u));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main() {
|
||||
try {
|
||||
const auto device = OpenVulkanTestDevice();
|
||||
if (!device) return VulkanTestSkipped;
|
||||
Run(*device, WholeWave32, 32u, false, "wave32 waterfall over the whole wave");
|
||||
Run(*device, PartialWave32, 32u, true, "wave32 waterfall under a partial EXEC");
|
||||
if (device->Target().subgroupSize >= 32u) Run(*device, PartialWave64, 64u, true, "wave64 waterfall under a partial EXEC");
|
||||
Run(*device, FirstLaneWave32, 32u, false, "wave32 waterfall picking lanes with v_readfirstlane");
|
||||
Run(*device, Compare64Wave32, 32u, false, "wave32 waterfall comparing the T# with v_cmpx_eq_u64");
|
||||
Run(*device, GuardedWave32, 32u, false, "wave32 waterfall behind a branch over the lanes inside the table");
|
||||
RunWithoutRecords(*device, {0u, 0u, 0u, 0u}, "a waterfall no lane enters over a null table V#");
|
||||
RunWithoutRecords(*device, Buffer(guest[0].table.data(), 0u, 0u), "a waterfall no lane enters over a table V# with stride 0 and no records");
|
||||
auto unchecked = Buffer(guest[0].table.data(), 32u, 0u);
|
||||
unchecked[1] |= 1u << 31u;
|
||||
unchecked[3] |= (1u << 23u) | (2u << 28u);
|
||||
RunWithoutRecords(*device, unchecked, "a waterfall no lane enters over a swizzled ADD_TID table V# with OOB_SELECT 2 and no records");
|
||||
RequireRefused(*device, StaleWave32, "a waterfall over a table T# and a T# left in the VGPRs of lanes the load skipped");
|
||||
RequireRefused(*device, MaskedStaleWave32, "a waterfall over a table T# and a T# written under EXEC to the VGPRs of lanes the load skipped");
|
||||
RequireRefused(*device, SevenDwordsWave32, "a waterfall that compares seven of the eight T# dwords");
|
||||
RequireRefused(*device, OutsideWave32, "an image load after the waterfall restores EXEC");
|
||||
RequireRefused(*device, StoreWave32, "an image store in a waterfall");
|
||||
RunLod(*device, {1u, 1u, 1u, 1u}, false, "an LOD query in a waterfall over a table of one image mode");
|
||||
RunLod(*device, {0u, 1u, 0u, 1u}, true, "an LOD query in a waterfall over a table of float and integer images");
|
||||
RunLod(*device, {Textures, 0u, 0u, 0u}, false, "an LOD query in a waterfall over a null record 0 and integer images");
|
||||
RunLod(*device, {Textures, 0u, 1u, 0u}, true, "an LOD query in a waterfall over a null record 0 and float and integer images");
|
||||
std::cout << "waterfall image tests passed\n";
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
std::cerr << error.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
+6
-2
@@ -20,8 +20,9 @@ struct BlockInfo {
|
||||
};
|
||||
|
||||
struct DescriptorSource {
|
||||
// A T# loaded from a table buffer (`heapSource`) at `entryOffset + key * 32`, the key a
|
||||
// wave-uniform runtime value (handle argument `keyArg`). With `hasMaterial` the key is itself
|
||||
// A T# loaded from a table buffer (`heapSource`) at `entryOffset + key * 32` (the table V#'s
|
||||
// stride when `indexed`), the key a wave-uniform runtime value (handle argument `keyArg`), or
|
||||
// each lane's own value when `laneKey`. With `hasMaterial` the key is itself
|
||||
// `M[readfirstlane(i) * selectorStride + selectorOffset]` over `materialSource`, so the key
|
||||
// set can be enumerated from the material records (see ResourceMaterializer).
|
||||
struct IndirectImage {
|
||||
@@ -32,6 +33,9 @@ struct DescriptorSource {
|
||||
std::uint32_t keyArg = 0;
|
||||
std::uint32_t entryOffset = 0;
|
||||
bool hasMaterial = false;
|
||||
bool laneKey = false;
|
||||
bool indexed = false;
|
||||
bool implicitDerivatives = false;
|
||||
|
||||
bool operator==(const IndirectImage& other) const = default;
|
||||
};
|
||||
|
||||
@@ -333,8 +333,19 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con
|
||||
const ShaderBufferResource heap = decodeBufferDescriptor(heapValue);
|
||||
const std::uint64_t heapSize = heap.GetSize();
|
||||
if (heap.Type() != 0u || heap.Base48() > std::numeric_limits<std::uint64_t>::max() - heapSize) throw std::runtime_error("bindless heap buffer is invalid");
|
||||
if (heapSize != 0u && (heapSize < table.entryOffset || (heapSize - table.entryOffset) % TableEntryBytes != 0u)) throw std::runtime_error("bindless heap contains a partial descriptor");
|
||||
const auto entries = heapSize > table.entryOffset ? static_cast<std::uint32_t>(std::min<std::uint64_t>((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits<std::uint32_t>::max())) : 0u;
|
||||
std::uint64_t entryBytes = TableEntryBytes;
|
||||
std::uint32_t entries = 0u;
|
||||
if (table.indexed) {
|
||||
if (heap.NumRecords() != 0u) {
|
||||
if (heap.SwizzleEnabled() || heap.AddTid() || heap.OutOfBounds() > 1u) throw std::runtime_error("bindless image table indexed through a V# with swizzling, ADD_TID or an unbounded range check is not implemented");
|
||||
entryBytes = heap.Stride();
|
||||
if (entryBytes < static_cast<std::uint64_t>(table.entryOffset) + TableEntryBytes) throw std::runtime_error("bindless heap record is smaller than its descriptor");
|
||||
entries = heap.NumRecords();
|
||||
}
|
||||
} else {
|
||||
if (heapSize != 0u && (heapSize < table.entryOffset || (heapSize - table.entryOffset) % TableEntryBytes != 0u)) throw std::runtime_error("bindless heap contains a partial descriptor");
|
||||
entries = heapSize > table.entryOffset ? static_cast<std::uint32_t>(std::min<std::uint64_t>((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits<std::uint32_t>::max())) : 0u;
|
||||
}
|
||||
const auto readWord = [&](std::uint64_t address, std::uint32_t& word) {
|
||||
if (!runtime.readMemory(runtime.userContext, address, &word)) {
|
||||
throw std::runtime_error("failed to read a bindless image table from memory");
|
||||
@@ -388,11 +399,12 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con
|
||||
const std::uint64_t heapBase = heap.Base48();
|
||||
std::vector<DescriptorValue> candidates(keys.size());
|
||||
std::vector<std::uint8_t> valid(keys.size(), 0u);
|
||||
std::vector<std::uint32_t> modes(keys.size(), 0u);
|
||||
std::uint32_t paddedNull = 0u;
|
||||
for (std::size_t i = 0u; i < keys.size(); ++i) {
|
||||
auto& candidate = candidates[i];
|
||||
candidate.dwordCount = 8u;
|
||||
const std::uint64_t address = heapBase + table.entryOffset + static_cast<std::uint64_t>(keys[i]) * TableEntryBytes;
|
||||
const std::uint64_t address = heapBase + table.entryOffset + static_cast<std::uint64_t>(keys[i]) * entryBytes;
|
||||
for (std::uint32_t dword = 0u; dword < 8u; ++dword) readWord(address + dword * sizeof(std::uint32_t), candidate.dwords[dword]);
|
||||
if (nullImageDescriptor(candidate)) {
|
||||
++paddedNull;
|
||||
@@ -401,9 +413,17 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con
|
||||
if (!validImageDescriptor(candidate, image.r128)) throw std::runtime_error("bindless image table contains an invalid descriptor");
|
||||
const auto decoded = decodeImageDescriptor(candidate, image, plan.srgbDecodeFormats);
|
||||
if (decoded.fmask) throw std::runtime_error("bindless FMASK images are unsupported");
|
||||
static_cast<void>(ResourceMaterializer::RuntimeImageMode(image, candidate, plan.info.runtimeImageModes.at(imageIndex)));
|
||||
modes[i] = ResourceMaterializer::RuntimeImageMode(image, candidate, plan.info.runtimeImageModes.at(imageIndex));
|
||||
valid[i] = 1u;
|
||||
}
|
||||
if (table.laneKey && table.implicitDerivatives) {
|
||||
std::optional<std::uint32_t> boundMode;
|
||||
for (std::size_t i = 0u; i < keys.size(); ++i) {
|
||||
if (valid[i] == 0u) continue;
|
||||
if (boundMode.has_value() && modes[i] != *boundMode) throw std::runtime_error("bindless image table keyed per lane holds images of different runtime modes under an implicit-derivative access, which is not implemented");
|
||||
boundMode = modes[i];
|
||||
}
|
||||
}
|
||||
resolution.mapping.clear();
|
||||
for (std::size_t i = 0u; i < keys.size(); ++i) {
|
||||
if (valid[i] != 0u) resolution.mapping.emplace_back(keys[i], static_cast<std::uint32_t>(i));
|
||||
|
||||
@@ -112,6 +112,7 @@ public:
|
||||
if (!m_program.Resources().srtPlanComplete) {
|
||||
fail("SRT plan is not ready");
|
||||
}
|
||||
FoldWaterfallDescriptors();
|
||||
SplitDescriptorPhis();
|
||||
PlanIndirectImages();
|
||||
std::unordered_map<const IrBlock*, std::uint32_t> blockIndices;
|
||||
@@ -150,7 +151,9 @@ public:
|
||||
plan.handle->ReplaceArgument(dword, plan.key);
|
||||
}
|
||||
for (const auto index : plan.memory) {
|
||||
m_program.Resources().memoryInfo[index].planningOnly = true;
|
||||
if (plan.planningReads) {
|
||||
m_program.Resources().memoryInfo[index].planningOnly = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::erase_if(m_program.Metadata().dynamicReads, [&](IrValue* value) {
|
||||
@@ -184,6 +187,14 @@ private:
|
||||
std::array<IrValue*, 8> roots {};
|
||||
std::array<std::uint32_t, 8> memory {};
|
||||
std::array<const IrValue*, 8> reads {};
|
||||
bool planningReads = true;
|
||||
};
|
||||
|
||||
struct LaneTableRead {
|
||||
IrValue* written = nullptr;
|
||||
IrValue* other = nullptr;
|
||||
IrValue* load = nullptr;
|
||||
std::uint32_t component = 0;
|
||||
};
|
||||
|
||||
void MakeSource(const IrValue& handle, std::uint32_t width, bool sampler, bool sampleAdjust, DescriptorSource& descriptor) {
|
||||
@@ -590,6 +601,359 @@ private:
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool MatchLaneTableRead(IrValue* value, LaneTableRead& read) {
|
||||
value = value->Resolve();
|
||||
read = LaneTableRead {};
|
||||
if (value->Opcode() == IrOpcode::SelectU32 && value->ArgumentCount() == 3u) {
|
||||
read.written = value->Argument(0)->Resolve();
|
||||
read.other = value->Argument(2)->Resolve();
|
||||
value = value->Argument(1)->Resolve();
|
||||
}
|
||||
const auto extract = value->Opcode();
|
||||
const bool extracted = extract == IrOpcode::CompositeExtractU32x2 || extract == IrOpcode::CompositeExtractU32x3 || extract == IrOpcode::CompositeExtractU32x4;
|
||||
if (extracted) {
|
||||
if (value->ArgumentCount() != 2u || !immediateU32(value->Argument(1), read.component)) {
|
||||
return false;
|
||||
}
|
||||
value = value->Argument(0)->Resolve();
|
||||
}
|
||||
const auto load = value->Opcode();
|
||||
const bool wide = load == IrOpcode::LoadBufferU32x2 || load == IrOpcode::LoadBufferU32x3 || load == IrOpcode::LoadBufferU32x4;
|
||||
if ((wide != extracted) || (!wide && load != IrOpcode::LoadBufferU32) || value->ArgumentCount() != 5u) {
|
||||
return false;
|
||||
}
|
||||
read.load = value;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SameBufferAddress(const IrValue& left, const IrValue& right) const {
|
||||
if (&left == &right) {
|
||||
return true;
|
||||
}
|
||||
const IrValue* leftHandle = left.Argument(0)->Resolve();
|
||||
const IrValue* rightHandle = right.Argument(0)->Resolve();
|
||||
if (leftHandle != rightHandle) {
|
||||
if (leftHandle->Opcode() != IrOpcode::GetBufferResource || rightHandle->Opcode() != IrOpcode::GetBufferResource || leftHandle->ArgumentCount() != 4u || rightHandle->ArgumentCount() != 4u) {
|
||||
return false;
|
||||
}
|
||||
for (std::size_t dword = 0; dword < 4u; dword++) {
|
||||
if (!EquivalentValue(m_program.Resources(), leftHandle->Argument(dword), rightHandle->Argument(dword))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (std::size_t argument = 1; argument < 4u; argument++) {
|
||||
if (!EquivalentValue(m_program.Resources(), left.Argument(argument), right.Argument(argument))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool MayHoldDescriptor(std::array<IrValue*, 8> tuple, std::uint32_t depth, std::vector<std::array<IrValue*, 8>>& visited) const {
|
||||
for (IrValue*& value : tuple) {
|
||||
value = value->Resolve();
|
||||
}
|
||||
if (std::ranges::find(visited, tuple) != visited.end()) {
|
||||
return false;
|
||||
}
|
||||
visited.push_back(tuple);
|
||||
if (std::ranges::all_of(tuple, [](const IrValue* value) { return value->HasImmediate() && value->Type() == IrType::U32 && value->ImmediateU32() == 0u; })) {
|
||||
return false;
|
||||
}
|
||||
constexpr SrtWalker walker;
|
||||
LaneTableRead read;
|
||||
const bool runtime = std::ranges::all_of(tuple, [&](const IrValue* value) { return value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value); });
|
||||
if (runtime || std::ranges::all_of(tuple, [&](IrValue* value) { return MatchLaneTableRead(value, read); })) {
|
||||
return true;
|
||||
}
|
||||
if (depth == phiSearchDepth) {
|
||||
return true;
|
||||
}
|
||||
const IrValue& first = *tuple[0];
|
||||
if (first.Opcode() == IrOpcode::SelectU32 && first.ArgumentCount() == 3u) {
|
||||
const IrValue* condition = first.Argument(0)->Resolve();
|
||||
std::array<IrValue*, 8> selected {};
|
||||
std::array<IrValue*, 8> kept {};
|
||||
for (std::size_t dword = 0; dword < tuple.size(); dword++) {
|
||||
const IrValue& value = *tuple[dword];
|
||||
if (value.Opcode() != IrOpcode::SelectU32 || value.ArgumentCount() != 3u || value.Argument(0)->Resolve() != condition) {
|
||||
return false;
|
||||
}
|
||||
selected[dword] = value.Argument(1);
|
||||
kept[dword] = value.Argument(2);
|
||||
}
|
||||
return MayHoldDescriptor(selected, depth + 1u, visited) || MayHoldDescriptor(kept, depth + 1u, visited);
|
||||
}
|
||||
if (!first.IsPhi()) {
|
||||
return false;
|
||||
}
|
||||
for (std::size_t edge = 0; edge < first.ArgumentCount(); edge++) {
|
||||
std::array<IrValue*, 8> incoming {};
|
||||
for (std::size_t dword = 0; dword < tuple.size(); dword++) {
|
||||
if (!tuple[dword]->IsPhi() || tuple[dword]->Parent() != first.Parent()) {
|
||||
return false;
|
||||
}
|
||||
incoming[dword] = PhiIncoming(*tuple[dword], first.PhiBlock(edge));
|
||||
if (incoming[dword] == nullptr) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (MayHoldDescriptor(incoming, depth + 1u, visited)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool TryMakeLaneTableImage(IrValue& handle, bool implicitDerivatives, IndirectImagePlan& plan) {
|
||||
if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u || handle.Parent() == nullptr) {
|
||||
return false;
|
||||
}
|
||||
const auto& memoryInfo = m_program.Resources().memoryInfo;
|
||||
std::array<LaneTableRead, 8> reads {};
|
||||
const MemoryInfo* first = nullptr;
|
||||
std::uint64_t firstPosition = 0;
|
||||
for (std::uint32_t dword = 0; dword < reads.size(); dword++) {
|
||||
auto& read = reads[dword];
|
||||
if (!MatchLaneTableRead(handle.Argument(dword), read) || read.written != reads[0].written) {
|
||||
return false;
|
||||
}
|
||||
const auto index = read.load->Flags<MemoryFlags>().index;
|
||||
if (index >= memoryInfo.size()) {
|
||||
return false;
|
||||
}
|
||||
const auto& memory = memoryInfo[index];
|
||||
if (memory.kind != ResourceKind::Buffer || memory.dataBits != 32u || memory.formatted || memory.typed || !memory.idxen || read.component >= memory.dataDwords) {
|
||||
return false;
|
||||
}
|
||||
const std::uint64_t position = static_cast<std::uint64_t>(memory.offset) + sizeof(std::uint32_t) * read.component;
|
||||
if (dword == 0u) {
|
||||
first = &memory;
|
||||
firstPosition = position;
|
||||
} else if (memory.offen != first->offen || position != firstPosition + sizeof(std::uint32_t) * dword || !SameBufferAddress(*reads[0].load, *read.load)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (reads[0].other != nullptr) {
|
||||
std::array<IrValue*, 8> others {};
|
||||
std::ranges::transform(reads, others.begin(), &LaneTableRead::other);
|
||||
std::vector<std::array<IrValue*, 8>> visited;
|
||||
if (MayHoldDescriptor(others, 0u, visited)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const IrValue& load = *reads[0].load;
|
||||
std::uint32_t voffset = 0;
|
||||
std::uint32_t soffset = 0;
|
||||
if (!immediateU32(load.Argument(3), soffset) || (first->offen && !immediateU32(load.Argument(2), voffset))) {
|
||||
return false;
|
||||
}
|
||||
const std::uint64_t entryOffset = firstPosition + voffset + soffset;
|
||||
IrValue* key = load.Argument(1)->Resolve();
|
||||
if (entryOffset > std::numeric_limits<std::uint32_t>::max() || key->Type() != IrType::U32) {
|
||||
return false;
|
||||
}
|
||||
DescriptorSource heapSource;
|
||||
std::uint32_t heapSourceIndex = 0;
|
||||
if (!MakeRuntimeBufferSource(*load.Argument(0)->Resolve(), heapSourceIndex, heapSource)) {
|
||||
return false;
|
||||
}
|
||||
if (reads[0].written != nullptr) {
|
||||
const std::array<IrValue*, 3> arguments {reads[0].written, key, &m_builder.Constant(std::numeric_limits<std::uint32_t>::max())};
|
||||
key = EmitBefore(handle, IrOpcode::SelectU32, IrType::U32, arguments);
|
||||
}
|
||||
|
||||
DescriptorSource::IndirectImage table;
|
||||
table.heapSource = heapSourceIndex;
|
||||
table.materialSource = heapSourceIndex;
|
||||
table.entryOffset = static_cast<std::uint32_t>(entryOffset);
|
||||
table.laneKey = true;
|
||||
table.indexed = true;
|
||||
table.implicitDerivatives = implicitDerivatives;
|
||||
DescriptorSource imageSource;
|
||||
imageSource.dwordCount = 8u;
|
||||
std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin());
|
||||
std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin() + 4u);
|
||||
for (auto& known : m_sources) {
|
||||
if (!known.indirectImage.has_value() || !known.indirectImage->laneKey) {
|
||||
continue;
|
||||
}
|
||||
auto shared = *known.indirectImage;
|
||||
shared.implicitDerivatives = table.implicitDerivatives;
|
||||
if (shared == table) {
|
||||
table.implicitDerivatives = known.indirectImage->implicitDerivatives = known.indirectImage->implicitDerivatives || table.implicitDerivatives;
|
||||
}
|
||||
}
|
||||
imageSource.indirectImage = table;
|
||||
|
||||
plan.handle = &handle;
|
||||
plan.source = InternSource(imageSource);
|
||||
plan.key = key;
|
||||
plan.roots = imageSource.dwords;
|
||||
plan.planningReads = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ImplicitDerivatives(const IrValue& inst) const {
|
||||
if (inst.Opcode() == IrOpcode::ImageQueryLod) {
|
||||
return true;
|
||||
}
|
||||
const auto& memoryInfo = m_program.Resources().memoryInfo;
|
||||
const auto index = inst.Flags<MemoryFlags>().index;
|
||||
return inst.Opcode() == IrOpcode::ImageSampleRaw && (index >= memoryInfo.size() || !ImageSampleExplicitLod(memoryInfo[index].imageSampleFlags, m_program.Resources().stage));
|
||||
}
|
||||
|
||||
static bool LaneRead(const IrValue& value) {
|
||||
return (value.Opcode() == IrOpcode::ReadLane || value.Opcode() == IrOpcode::ReadFirstLane) && value.ArgumentCount() == 2u;
|
||||
}
|
||||
|
||||
static bool FoldableImageRead(IrOpcode op) {
|
||||
switch (op) {
|
||||
case IrOpcode::ImageSampleRaw:
|
||||
case IrOpcode::ImageGatherRaw:
|
||||
case IrOpcode::ImageQueryLod:
|
||||
case IrOpcode::ImageQueryDimensions:
|
||||
case IrOpcode::ImageRead: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
static IrValue* ObservingCondition(const IrValue& inst) {
|
||||
IrValue* condition = nullptr;
|
||||
std::vector<const IrValue*> pending {&inst};
|
||||
while (!pending.empty()) {
|
||||
const IrValue* value = pending.back();
|
||||
pending.pop_back();
|
||||
for (const IrUse& use : value->OperandUses()) {
|
||||
const IrValue* user = use.user;
|
||||
if (user->IsIdentity() || (user->Opcode() == IrOpcode::CompositeExtractU32x4 && use.operand == 0u)) {
|
||||
pending.push_back(user);
|
||||
continue;
|
||||
}
|
||||
if (user->Opcode() != IrOpcode::SelectU32 || user->ArgumentCount() != 3u || use.operand != 1u) {
|
||||
return nullptr;
|
||||
}
|
||||
IrValue* selector = user->Argument(0)->Resolve();
|
||||
if (condition != nullptr && selector != condition) {
|
||||
return nullptr;
|
||||
}
|
||||
condition = selector;
|
||||
}
|
||||
}
|
||||
return condition;
|
||||
}
|
||||
|
||||
using Equalities = std::vector<std::pair<const IrValue*, const IrValue*>>;
|
||||
|
||||
static Equalities ImpliedEqualities(IrValue* condition) {
|
||||
constexpr std::size_t limit = 1024u;
|
||||
Equalities equal;
|
||||
std::vector<std::pair<IrValue*, bool>> pending {{condition, true}};
|
||||
for (std::size_t visited = 0; !pending.empty() && visited < limit; visited++) {
|
||||
auto [value, holds] = pending.back();
|
||||
pending.pop_back();
|
||||
value = value->Resolve();
|
||||
const auto op = value->Opcode();
|
||||
if (op == IrOpcode::LogicalNot && value->ArgumentCount() == 1u) {
|
||||
pending.emplace_back(value->Argument(0), !holds);
|
||||
} else if (((op == IrOpcode::LogicalAnd && holds) || (op == IrOpcode::LogicalOr && !holds)) && value->ArgumentCount() == 2u) {
|
||||
pending.emplace_back(value->Argument(0), holds);
|
||||
pending.emplace_back(value->Argument(1), holds);
|
||||
} else if (((op == IrOpcode::IEqual32 && holds) || (op == IrOpcode::INotEqual32 && !holds)) && value->ArgumentCount() == 2u) {
|
||||
equal.emplace_back(value->Argument(0)->Resolve(), value->Argument(1)->Resolve());
|
||||
} else if (((op == IrOpcode::IEqual64 && holds) || (op == IrOpcode::INotEqual64 && !holds)) && value->ArgumentCount() == 2u) {
|
||||
const IrValue* left = value->Argument(0)->Resolve();
|
||||
const IrValue* right = value->Argument(1)->Resolve();
|
||||
if (left->Opcode() != IrOpcode::CompositeConstructU64 || right->Opcode() != IrOpcode::CompositeConstructU64 || left->ArgumentCount() != 2u || right->ArgumentCount() != 2u) {
|
||||
continue;
|
||||
}
|
||||
for (std::size_t half = 0; half < 2u; half++) {
|
||||
equal.emplace_back(left->Argument(half)->Resolve(), right->Argument(half)->Resolve());
|
||||
}
|
||||
}
|
||||
}
|
||||
return equal;
|
||||
}
|
||||
|
||||
void FoldLaneReads(IrValue& inst, std::uint32_t slot, IrOpcode expected, std::size_t width, const Equalities& equal) {
|
||||
IrValue* handle = inst.Argument(slot)->Resolve();
|
||||
if (handle->Opcode() != expected || handle->ArgumentCount() != width) {
|
||||
return;
|
||||
}
|
||||
constexpr SrtWalker walker;
|
||||
std::vector<IrValue*> dwords;
|
||||
bool valid = true;
|
||||
bool folded = false;
|
||||
for (std::size_t dword = 0; dword < width; dword++) {
|
||||
IrValue* value = handle->Argument(dword)->Resolve();
|
||||
valid = valid && value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value);
|
||||
if (!LaneRead(*value)) {
|
||||
dwords.push_back(value);
|
||||
continue;
|
||||
}
|
||||
IrValue* source = value->Argument(0)->Resolve();
|
||||
const bool proven = std::ranges::any_of(equal, [&](const auto& pair) {
|
||||
return (pair.first == value && pair.second == source) || (pair.first == source && pair.second == value);
|
||||
});
|
||||
if (!proven) {
|
||||
return;
|
||||
}
|
||||
dwords.push_back(source);
|
||||
folded = true;
|
||||
}
|
||||
if (valid || !folded) {
|
||||
return;
|
||||
}
|
||||
IrValue* copy = EmitCopy(*handle, dwords, inst);
|
||||
bool accepted = std::ranges::all_of(dwords, [&](IrValue* value) {
|
||||
return value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value);
|
||||
});
|
||||
static const bool tables = std::getenv("APS5_NO_BINDLESS_IMAGES") == nullptr;
|
||||
IndirectImagePlan plan;
|
||||
if (!accepted && tables && expected == IrOpcode::GetImageResource && TryMakeLaneTableImage(*copy, ImplicitDerivatives(inst), plan)) {
|
||||
m_indirectImages.push_back(std::move(plan));
|
||||
accepted = true;
|
||||
}
|
||||
if (!accepted) {
|
||||
copy->Invalidate();
|
||||
copy->Parent()->RemoveInstruction(copy);
|
||||
return;
|
||||
}
|
||||
inst.ReplaceArgument(slot, copy);
|
||||
m_foldedHandles.push_back(copy);
|
||||
}
|
||||
|
||||
void FoldWaterfallDescriptors() {
|
||||
std::vector<IrValue*> candidates;
|
||||
for (auto& block : m_program.Blocks()) {
|
||||
for (IrValue* inst : block->Instructions()) {
|
||||
if (FoldableImageRead(inst->Opcode())) {
|
||||
candidates.push_back(inst);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (IrValue* inst : candidates) {
|
||||
const bool sampler = ImageOpcodeInfoOf(inst->Opcode()).needsSampler;
|
||||
if (inst->Parent() == nullptr || inst->ArgumentCount() < (sampler ? 2u : 1u)) {
|
||||
continue;
|
||||
}
|
||||
IrValue* condition = ObservingCondition(*inst);
|
||||
if (condition == nullptr) {
|
||||
continue;
|
||||
}
|
||||
const Equalities equal = ImpliedEqualities(condition);
|
||||
if (equal.empty()) {
|
||||
continue;
|
||||
}
|
||||
FoldLaneReads(*inst, 0u, IrOpcode::GetImageResource, 8u, equal);
|
||||
if (sampler) {
|
||||
FoldLaneReads(*inst, 1u, IrOpcode::GetSamplerResource, 4u, equal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const IndirectImagePlan* FindIndirectImage(const IrValue& handle) const {
|
||||
const auto found = std::ranges::find_if(m_indirectImages, [&](const IndirectImagePlan& plan) {
|
||||
return plan.handle == &handle;
|
||||
@@ -599,7 +963,7 @@ private:
|
||||
|
||||
bool IsIndirectPlanningMemory(std::uint32_t index) const {
|
||||
return std::ranges::any_of(m_indirectImages, [&](const IndirectImagePlan& plan) {
|
||||
return std::ranges::find(plan.memory, index) != plan.memory.end();
|
||||
return plan.planningReads && std::ranges::find(plan.memory, index) != plan.memory.end();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1117,6 +1481,9 @@ private:
|
||||
}
|
||||
}
|
||||
for (IrValue* inst : candidates) {
|
||||
if (std::ranges::find(m_foldedHandles, inst->Argument(0)->Resolve()) != m_foldedHandles.end()) {
|
||||
continue;
|
||||
}
|
||||
SplitDescriptorPhi(*inst);
|
||||
}
|
||||
}
|
||||
@@ -1494,6 +1861,7 @@ private:
|
||||
std::vector<IndirectImagePlan> m_indirectImages;
|
||||
std::unordered_map<std::uint32_t, std::vector<const IrValue*>> m_memoryUsers;
|
||||
bool m_memoryUsersIndexed = false;
|
||||
std::vector<const IrValue*> m_foldedHandles;
|
||||
struct EdgeSelectorEntry {
|
||||
IrBlock* block = nullptr;
|
||||
std::vector<bool> edges;
|
||||
|
||||
@@ -147,7 +147,7 @@ static_assert(fieldCount<IrResourcePlan>() == 21, "IrResourcePlan changed: updat
|
||||
static_assert(fieldCount<GuestRegister>() == 2, "GuestRegister changed: update the plan value encoder");
|
||||
static_assert(fieldCount<MemoryInfo>() == 30, "MemoryInfo changed: update the plan encoder");
|
||||
static_assert(fieldCount<DescriptorSource>() == 3, "DescriptorSource changed: update the plan encoder");
|
||||
static_assert(fieldCount<DescriptorSource::IndirectImage>() == 7, "IndirectImage changed: update the plan encoder");
|
||||
static_assert(fieldCount<DescriptorSource::IndirectImage>() == 10, "IndirectImage changed: update the plan encoder");
|
||||
static_assert(fieldCount<ResourceBlock>() == 4, "ResourceBlock changed: update the plan encoder");
|
||||
static_assert(fieldCount<SrtRead>() == 2, "SrtRead changed: update the plan encoder");
|
||||
static_assert(fieldCount<UniformFill>() == 5, "UniformFill changed: update the plan encoder");
|
||||
@@ -973,6 +973,9 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) {
|
||||
out.Value(table.keyArg);
|
||||
out.Value(table.entryOffset);
|
||||
out.Value(table.hasMaterial);
|
||||
out.Value(table.laneKey);
|
||||
out.Value(table.indexed);
|
||||
out.Value(table.implicitDerivatives);
|
||||
});
|
||||
writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) {
|
||||
out.Value(valueIndex(block.condition));
|
||||
@@ -1115,6 +1118,9 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) {
|
||||
in.Value(table.keyArg);
|
||||
in.Value(table.entryOffset);
|
||||
in.Value(table.hasMaterial);
|
||||
in.Value(table.laneKey);
|
||||
in.Value(table.indexed);
|
||||
in.Value(table.implicitDerivatives);
|
||||
source.indirectImage = indirect ? std::optional(table) : std::nullopt;
|
||||
});
|
||||
reader.List(plan.controlFlow, 28, [&](Reader& in, ResourceBlock& block) {
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
namespace ShaderRecompiler::ShaderDiskCache {
|
||||
|
||||
inline constexpr std::uint32_t FormatVersion = 31;
|
||||
inline constexpr std::uint32_t FormatVersion = 32;
|
||||
|
||||
enum class LoadStatus {
|
||||
Loaded,
|
||||
|
||||
@@ -72,6 +72,18 @@ std::uint32_t RuntimeMipCount(SpirvEmitterState& state, std::uint32_t resource)
|
||||
return Binary(state, spv::OpIAdd, TypeU32(state), Binary(state, spv::OpISub, TypeU32(state), last, first), ConstantU32(state, 1u));
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
bool TableKeyPerLane(const SpirvEmitterState& state, std::uint32_t resource) {
|
||||
const auto& images = state.program.Info().images;
|
||||
const auto& sources = state.program.Resources().descriptorSources;
|
||||
if (resource >= images.size() || images[resource].source >= sources.size()) return false;
|
||||
const auto& table = sources[images[resource].source].indirectImage;
|
||||
return table.has_value() && table->laneKey;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource) {
|
||||
const bool storage = ImageBindingResourceClass(kind) == ImageResourceClass::Storage;
|
||||
const auto supported = [&](std::uint32_t capability) {
|
||||
@@ -88,7 +100,7 @@ std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind ki
|
||||
state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), metadata, state.shaderDataStorageVariable, ConstantU32(state, 0u), offset);
|
||||
const auto index = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpLoad, TypeU32(state), index, metadata);
|
||||
if (!state.tableIndexNonUniform) {
|
||||
if (!state.tableIndexNonUniform && !TableKeyPerLane(state, resource)) {
|
||||
return index;
|
||||
}
|
||||
const auto nonUniform = storage ? spv::CapabilityStorageImageArrayNonUniformIndexing : spv::CapabilitySampledImageArrayNonUniformIndexing;
|
||||
@@ -104,8 +116,8 @@ std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind ki
|
||||
return index;
|
||||
}
|
||||
|
||||
void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t slotId, std::uint32_t id) {
|
||||
if (slotId != 0 && state.tableIndexNonUniform) state.module.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform);
|
||||
void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t slotId, std::uint32_t id) {
|
||||
if (slotId != 0 && (state.tableIndexNonUniform || TableKeyPerLane(state, resource))) state.module.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform);
|
||||
}
|
||||
|
||||
std::uint32_t ImageScalarType(SpirvEmitterState& state, IrTextureNumericClass numericClass) {
|
||||
@@ -184,11 +196,11 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t
|
||||
} else {
|
||||
pointer = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, variable, TableImageIndex(state, kind, resource));
|
||||
DecorateTableNonUniform(state, slotId, pointer);
|
||||
DecorateTableNonUniform(state, resource, slotId, pointer);
|
||||
}
|
||||
const auto image = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpLoad, imageType, image, pointer);
|
||||
DecorateTableNonUniform(state, slotId, image);
|
||||
DecorateTableNonUniform(state, resource, slotId, image);
|
||||
return image;
|
||||
}
|
||||
|
||||
@@ -210,7 +222,7 @@ std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource,
|
||||
const auto sampledImage = state.module.AllocateId();
|
||||
const auto sampledType = state.module.Type(spv::OpTypeSampledImage, ImageType(state, imageResource));
|
||||
state.module.AddFunction(spv::OpSampledImage, sampledType, sampledImage, image, samplerId);
|
||||
DecorateTableNonUniform(state, slotId, sampledImage);
|
||||
DecorateTableNonUniform(state, resource, slotId, sampledImage);
|
||||
return sampledImage;
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
- [8_SRGB and 8_8_SRGB loads](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) on a device that cannot sample R8_SRGB or R8G8_SRGB return the sRGB EOTF rounded to nearest even at bf16 precision, as adriadam10 measured on RDNA2 (gfx1035) in #866; the PS5's gfx1013 is not measured.
|
||||
- [Samples and gathers](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB.
|
||||
- [Bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB.
|
||||
- A [waterfall loop](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) over T#s is compiled as one access per lane only when each lane loaded its T# with `buffer_load_dword*` (IDXEN, immediate VOFFSET and SOFFSET) from one V# that the SRT walker evaluates: the access becomes a [bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) keyed by the lane's index, with the NonUniform decoration. The loop is recognised when every T# dword of an image read, sample, gather or query comes from `v_readlane_b32` or `v_readfirstlane_b32` and the EXEC its results are written under implies that each of them equals the lane's own VGPR. Waterfalls over other per-lane T#s, over S#s, around image stores or atomics, and over T# VGPRs that hold another descriptor in lanes the load skipped (in all eight dwords alike, directly or through the same masked writes and phis) still throw. A lane the load skipped whose T# VGPRs hold any other value reads zeros instead of using that value as a T#. Every lane runs the access with its own record, so a sample with implicit derivatives or an LOD query throws at dispatch when the bound records need different runtime image modes. Lanes whose record is null or outside the table take the zero path of the per-lane mode switch under record 0's mode, the first mode when record 0 is null, so around such a sample or query the derivatives of quad neighbours whose records need another mode are still undefined. A table V# with records must be structured (stride at least the record offset plus 32 bytes, `OOB_SELECT` 0 or 1) without swizzling or `ADD_TID`, else the dispatch throws. One with `num_records` 0, a null V# included, binds no record, and every lane that runs the access reads zeros: with no records every `OOB_SELECT` of the GFX10 table in Mesa [ac_descriptors.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/ac_descriptors.c) puts each load out of range. The table is bound whole, so a table V# with more than `BindlessTableSlots` (16) records throws at dispatch. None of this was measured on hardware.
|
||||
- [Shader MODE register access](../../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) reads the initial rounding fields from `GuestContext::floatMode`; missing metadata retains the legacy round-to-nearest-even assumption. Immediate rounding writes are accepted only when they preserve that state. State-changing or SGPR-sourced MODE writes throw until runtime mode tracking and affected arithmetic are implemented. Non-rounding register reads still throw; `s_setreg_b32` writes to other hardware registers are still ignored. Tests follow the public RDNA2/LLVM contract, not console measurements.
|
||||
- A [flat SRT read](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read.
|
||||
- Repeated `s_load`s of one [flat SRT](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own.
|
||||
|
||||
Reference in new issue
Block a user