From 5c716de97340b56f6e112fd766189c4deef6420e Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:26:48 +0400 Subject: [PATCH 001/103] perf(shader): rewrite high-fan-out IR uses in linear time --- CMakeLists.txt | 11 + .../src/IrValue.cpp | 26 +- .../tests/UseRewrite.cpp | 244 ++++++++++++++++++ .../tests/UseRewriteAllocation.cpp | 20 ++ 4 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp create mode 100644 core/shader/recompiler/IntermediateRepresentation/tests/UseRewriteAllocation.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5cb4fb85c8..dc6dbf6924 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,17 @@ endif() if(BUILD_TESTING) find_package(Python3 QUIET COMPONENTS Interpreter) + add_test_executable(shader_ir_use_rewrite_tests + core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp + core/shader/recompiler/IntermediateRepresentation/tests/UseRewriteAllocation.cpp + core/shader/recompiler/IntermediateRepresentation/src/IrValue.cpp + core/shader/recompiler/IntermediateRepresentation/src/IrBlock.cpp + core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp + core/shader/recompiler/IntermediateRepresentation/src/GuestRegister.cpp) + target_include_directories(shader_ir_use_rewrite_tests PRIVATE + core/shader/recompiler/IntermediateRepresentation/include) + add_test(NAME shader_ir_use_rewrite COMMAND shader_ir_use_rewrite_tests) + set_tests_properties(shader_ir_use_rewrite PROPERTIES TIMEOUT 20) if(Python3_Interpreter_FOUND) add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py) diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrValue.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrValue.cpp index 4a1a662b69..33273410cf 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrValue.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrValue.cpp @@ -65,10 +65,30 @@ void IrValue::ReplaceAllUsesWith(IrValue* replacement) { if (replacement == this) { return; } - while (!operandUses.empty()) { - const IrUse use = operandUses.back(); - use.user->ReplaceArgument(use.operand, replacement); + if (replacement != nullptr) { + const auto reserveAdditional = [count = operandUses.size()](auto& list) { + const auto maximum = list.max_size(); + if (count > maximum - list.size()) { + throw std::length_error("IrValue::ReplaceAllUsesWith use list is too large"); + } + const auto required = list.size() + count; + if (required > list.capacity()) { + const auto grown = list.capacity() > maximum / 2 ? maximum : list.capacity() * 2; + list.reserve(std::max(required, grown)); + } + }; + reserveAdditional(replacement->uses); + reserveAdditional(replacement->operandUses); } + for (auto use = operandUses.rbegin(); use != operandUses.rend(); ++use) { + use->user->arguments[use->operand] = replacement; + if (replacement != nullptr) { + replacement->uses.push_back(use->user); + replacement->operandUses.push_back(*use); + } + } + uses.clear(); + operandUses.clear(); } void IrValue::SetImmediateU32(std::uint32_t value) { diff --git a/core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp b/core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp new file mode 100644 index 0000000000..3308434316 --- /dev/null +++ b/core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp @@ -0,0 +1,244 @@ +#include "IntermediateRepresentation/IrBlock.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace IrRewriteAllocation { +extern std::ptrdiff_t failureAfter; +extern std::size_t allocations; +extern bool countAllocations; +} +using namespace IrRewriteAllocation; + +void Require(bool condition) { + if (!condition) throw std::runtime_error("IR use rewrite assertion failed"); +} + +struct Graph { + IrBlock block{1}; + std::vector> values; + + explicit Graph(std::size_t count) { + for (std::size_t index = 0; index < count; ++index) { + auto value = std::make_unique(IrOpcode::IAdd32, IrType::U32, static_cast(index)); + value->SetFlags(static_cast(index * 17)); + block.AppendInstruction(value.get()); + values.push_back(std::move(value)); + } + } + + IrValue* Value(std::size_t index) const { return values.at(index).get(); } +}; + +void ReferenceReplace(IrValue* source, IrValue* replacement) { + if (source == replacement) return; + while (!source->OperandUses().empty()) { + const IrUse use = source->OperandUses().back(); + use.user->ReplaceArgument(use.operand, replacement); + } +} + +void EqualGraphs(const Graph& first, const Graph& second) { + Require(first.values.size() == second.values.size()); + for (std::size_t index = 0; index < first.values.size(); ++index) { + const auto* left = first.Value(index); + const auto* right = second.Value(index); + Require(left->Opcode() == right->Opcode() && left->Type() == right->Type()); + Require(left->Id() == right->Id() && left->Flags() == right->Flags()); + Require((left->Parent() == nullptr) == (right->Parent() == nullptr)); + if (left->Parent()) Require(left->Parent()->Id() == right->Parent()->Id()); + Require(left->ArgumentCount() == right->ArgumentCount()); + for (std::size_t operand = 0; operand < left->ArgumentCount(); ++operand) { + const auto* a = left->Argument(operand); + const auto* b = right->Argument(operand); + Require((a == nullptr) == (b == nullptr)); + if (a) Require(a->Id() == b->Id()); + } + Require(left->Uses().size() == right->Uses().size()); + for (std::size_t use = 0; use < left->Uses().size(); ++use) { + Require(left->Uses()[use]->Id() == right->Uses()[use]->Id()); + } + Require(left->OperandUses().size() == right->OperandUses().size()); + for (std::size_t use = 0; use < left->OperandUses().size(); ++use) { + const auto& a = left->OperandUses()[use]; + const auto& b = right->OperandUses()[use]; + Require(a.user->Id() == b.user->Id() && a.operand == b.operand); + } + } +} + +void CheckSimpleReplacement() { + Graph graph(5); + auto* source = graph.Value(0); + auto* target = graph.Value(1); + auto* first = graph.Value(2); + auto* second = graph.Value(3); + auto* existing = graph.Value(4); + existing->AddArgument(target); + first->AddArgument(source); + second->AddArgument(source); + first->AddArgument(source); + first->AddArgument(target); + source->ReplaceAllUsesWith(target); + Require(!source->HasUses() && source->OperandUses().empty()); + Require(first->Argument(0) == target && first->Argument(1) == target && first->Argument(2) == target); + Require(second->Argument(0) == target); + const std::vector expected{{existing, 0}, {first, 2}, {first, 1}, {second, 0}, {first, 0}}; + Require(target->OperandUses() == expected); + Require(target->Uses() == std::vector({existing, first, first, second, first})); + first->ReplaceArgument(1, source); + Require(source->UseCount() == 1); + source->ReplaceAllUsesWith(nullptr); + Require(first->Argument(1) == nullptr && target->UseCount() == 4); + Require(source->Opcode() == IrOpcode::IAdd32 && source->Parent() == &graph.block); +} + +void CheckSelfAndEmpty() { + Graph graph(3); + graph.Value(2)->AddArgument(graph.Value(0)); + graph.Value(0)->ReplaceAllUsesWith(graph.Value(0)); + Require(graph.Value(0)->UseCount() == 1 && graph.Value(2)->Argument(0) == graph.Value(0)); + graph.Value(1)->ReplaceAllUsesWith(graph.Value(0)); + Require(graph.Value(0)->UseCount() == 1); + graph.Value(1)->ReplaceAllUsesWith(nullptr); + Require(!graph.Value(1)->HasUses()); +} + +void CheckPhiAndIdentity() { + Graph graph(4); + auto* source = graph.Value(0); + auto* target = graph.Value(1); + auto* phi = graph.Value(2); + phi->ReplaceOpcode(IrOpcode::Phi); + phi->AddPhiOperand(&graph.block, source); + phi->AddPhiOperand(&graph.block, source); + graph.Value(3)->ReplaceOpcode(IrOpcode::Identity); + graph.Value(3)->AddArgument(source); + source->ReplaceUsesWith(target, false); + Require(source->Opcode() == IrOpcode::Unreachable); + Require(phi->Argument(0) == target && phi->Argument(1) == target); + Require(phi->PhiBlockCount() == 2 && phi->PhiBlock(0) == &graph.block && phi->PhiBlock(1) == &graph.block); + Require(graph.Value(3)->Resolve() == target); + Require(target->UseCount() == 3); +} + +void CheckAllocationFailure() { + for (std::ptrdiff_t failure = 0; failure < 2; ++failure) { + Graph graph(12); + for (std::size_t index = 2; index < 12; ++index) { + graph.Value(index)->AddArgument(graph.Value(0)); + } + bool caught = false; + failureAfter = failure; + try { + graph.Value(0)->ReplaceAllUsesWith(graph.Value(1)); + } catch (const std::bad_alloc&) { + caught = true; + } catch (...) { + failureAfter = -1; + throw; + } + failureAfter = -1; + Require(caught && graph.Value(0)->UseCount() == 10); + Require(graph.Value(0)->OperandUses().size() == 10); + Require(!graph.Value(1)->HasUses() && graph.Value(1)->OperandUses().empty()); + for (std::size_t index = 2; index < 12; ++index) { + Require(graph.Value(index)->Argument(0) == graph.Value(0)); + } + graph.Value(0)->ReplaceAllUsesWith(graph.Value(1)); + Require(graph.Value(1)->UseCount() == 10); + } +} + +void CheckGeometricGrowth() { + constexpr std::size_t count = 4096; + Graph graph(count + 2); + std::vector> sources; + for (std::size_t index = 2; index < count + 2; ++index) { + sources.push_back(std::make_unique(IrOpcode::UndefU32, IrType::U32, index + count + 2)); + graph.Value(index)->AddArgument(sources.back().get()); + } + allocations = 0; + countAllocations = true; + for (auto& source : sources) source->ReplaceAllUsesWith(graph.Value(1)); + countAllocations = false; + Require(allocations <= 2 * (std::bit_width(count) + 1)); + Require(graph.Value(1)->UseCount() == count); + for (std::size_t index = 2; index < count + 2; ++index) Require(graph.Value(index)->Argument(0) == graph.Value(1)); +} + +void CheckLargeFanOut() { + constexpr std::size_t count = 100000; + Graph graph(count + 2); + for (std::size_t index = 2; index < count + 2; ++index) { + graph.Value(index)->AddArgument(graph.Value(0)); + graph.Value(index)->AddArgument(graph.Value(0)); + } + graph.Value(0)->ReplaceAllUsesWith(graph.Value(1)); + Require(graph.Value(1)->UseCount() == count * 2 && !graph.Value(0)->HasUses()); + for (std::size_t index = 2; index < count + 2; ++index) { + Require(graph.Value(index)->Argument(0) == graph.Value(1) && graph.Value(index)->Argument(1) == graph.Value(1)); + } +} + +void CheckDifferentialPrograms() { + std::mt19937 random(0x4f23a771); + constexpr std::size_t count = 32; + for (std::size_t trial = 0; trial < 2000; ++trial) { + Graph actual(count), reference(count); + for (std::size_t index = 0; index < count; ++index) { + const auto operands = random() % 5; + for (std::size_t operand = 0; operand < operands; ++operand) { + const auto argument = random() % (count + 1); + actual.Value(index)->AddArgument(argument == count ? nullptr : actual.Value(argument)); + reference.Value(index)->AddArgument(argument == count ? nullptr : reference.Value(argument)); + } + } + for (std::size_t step = 0; step < 24; ++step) { + const auto source = random() % count; + const auto replacement = random() % (count + 1); + if (random() % 4 == 0 && actual.Value(source)->ArgumentCount() > 0) { + const auto operand = random() % actual.Value(source)->ArgumentCount(); + actual.Value(source)->ReplaceArgument(operand, replacement == count ? nullptr : actual.Value(replacement)); + reference.Value(source)->ReplaceArgument(operand, replacement == count ? nullptr : reference.Value(replacement)); + } else { + actual.Value(source)->ReplaceAllUsesWith(replacement == count ? nullptr : actual.Value(replacement)); + ReferenceReplace(reference.Value(source), replacement == count ? nullptr : reference.Value(replacement)); + } + EqualGraphs(actual, reference); + } + } +} + +int main() { + const char* current = "setup"; + try { + const std::pair cases[] = { + {"ordered duplicate operands", CheckSimpleReplacement}, + {"self and empty replacement", CheckSelfAndEmpty}, + {"phi and identity rewriting", CheckPhiAndIdentity}, + {"allocation failure integrity", CheckAllocationFailure}, + {"geometric use-list growth", CheckGeometricGrowth}, + {"large fan-out", CheckLargeFanOut}, + {"differential use graphs", CheckDifferentialPrograms} + }; + for (const auto& [name, test] : cases) { + current = name; + test(); + } + std::puts("7 IR rewrite suites passed; 2000 differential use graphs and 48000 mutation comparisons"); + return 0; + } catch (const std::exception& error) { + failureAfter = -1; + countAllocations = false; + std::fprintf(stderr, "%s: %s\n", current, error.what()); + return 1; + } +} diff --git a/core/shader/recompiler/IntermediateRepresentation/tests/UseRewriteAllocation.cpp b/core/shader/recompiler/IntermediateRepresentation/tests/UseRewriteAllocation.cpp new file mode 100644 index 0000000000..4681d4b223 --- /dev/null +++ b/core/shader/recompiler/IntermediateRepresentation/tests/UseRewriteAllocation.cpp @@ -0,0 +1,20 @@ +#include +#include +#include + +namespace IrRewriteAllocation { +std::ptrdiff_t failureAfter = -1; +std::size_t allocations = 0; +bool countAllocations = false; +} + +void* operator new(std::size_t size) { + if (IrRewriteAllocation::failureAfter == 0) throw std::bad_alloc(); + if (IrRewriteAllocation::failureAfter > 0) --IrRewriteAllocation::failureAfter; + if (IrRewriteAllocation::countAllocations) ++IrRewriteAllocation::allocations; + if (void* memory = std::malloc(size == 0 ? 1 : size)) return memory; + throw std::bad_alloc(); +} + +void operator delete(void* memory) noexcept { std::free(memory); } +void operator delete(void* memory, std::size_t) noexcept { std::free(memory); } From 784cb4d3dbe7f8248186e8f5948fa2fc98aef731 Mon Sep 17 00:00:00 2001 From: notaxeltv Date: Wed, 14 Oct 2026 23:00:30 +0200 Subject: [PATCH 002/103] feat(libScePlayerSelectionDialog): run common-dialog state machine without UI Co-authored-by: Cursor --- .../CMakeLists.txt | 14 ++++++ .../libScePlayerSelectionDialog/Export.cpp | 49 ++++++++++++++++++ .../libs/tests/GuestPlayerSelectionDialog.cpp | 50 +++++++++++++++++++ docs/dev/debt/libScePlayerSelectionDialog.md | 5 ++ 4 files changed, 118 insertions(+) create mode 100644 core/libs/prx/libScePlayerSelectionDialog/CMakeLists.txt create mode 100644 core/libs/tests/GuestPlayerSelectionDialog.cpp create mode 100644 docs/dev/debt/libScePlayerSelectionDialog.md diff --git a/core/libs/prx/libScePlayerSelectionDialog/CMakeLists.txt b/core/libs/prx/libScePlayerSelectionDialog/CMakeLists.txt new file mode 100644 index 0000000000..02e53fdc16 --- /dev/null +++ b/core/libs/prx/libScePlayerSelectionDialog/CMakeLists.txt @@ -0,0 +1,14 @@ +add_library(libScePlayerSelectionDialog SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libScePlayerSelectionDialog PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libScePlayerSelectionDialog) + +if(BUILD_TESTING) + add_test_executable(guest_player_selection_dialog_tests ${LIBS_INCLUDE_DIR}/tests/GuestPlayerSelectionDialog.cpp) + target_include_directories(guest_player_selection_dialog_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_player_selection_dialog_tests PRIVATE libScePlayerSelectionDialog) + add_test(NAME guest_player_selection_dialog COMMAND guest_player_selection_dialog_tests) + if(WIN32) + set_tests_properties(guest_player_selection_dialog PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libScePlayerSelectionDialog/Export.cpp b/core/libs/prx/libScePlayerSelectionDialog/Export.cpp index e32913a7ad..6093f69d67 100644 --- a/core/libs/prx/libScePlayerSelectionDialog/Export.cpp +++ b/core/libs/prx/libScePlayerSelectionDialog/Export.cpp @@ -1,11 +1,60 @@ +#include #include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +constexpr int COMMON_DIALOG_STATUS_NONE = 0; +constexpr int COMMON_DIALOG_STATUS_INITIALIZED = 1; +constexpr int COMMON_DIALOG_STATUS_RUNNING = 2; +constexpr int COMMON_DIALOG_STATUS_FINISHED = 3; +constexpr int COMMON_DIALOG_ERROR_NOT_INITIALIZED = static_cast(0x80B80003u); +constexpr int COMMON_DIALOG_ERROR_ALREADY_INITIALIZED = static_cast(0x80B80004u); +constexpr int COMMON_DIALOG_ERROR_NOT_FINISHED = static_cast(0x80B80005u); +constexpr int COMMON_DIALOG_ERROR_BUSY = static_cast(0x80B80007u); +constexpr int COMMON_DIALOG_ERROR_ARG_NULL = static_cast(0x80B8000Du); +constexpr int COMMON_DIALOG_RESULT_USER_CANCELED = 1; + +std::atomic g_status{COMMON_DIALOG_STATUS_NONE}; + +} + extern "C" { +int APS5_VABI scePlayerSelectionDialogInitialize(void) { + int expected = COMMON_DIALOG_STATUS_NONE; + if (!g_status.compare_exchange_strong(expected, COMMON_DIALOG_STATUS_INITIALIZED)) return COMMON_DIALOG_ERROR_ALREADY_INITIALIZED; + return 0; +} + +int APS5_VABI scePlayerSelectionDialogOpen(const void* param) { + const int status = g_status.load(); + if (status == COMMON_DIALOG_STATUS_NONE) return COMMON_DIALOG_ERROR_NOT_INITIALIZED; + if (status == COMMON_DIALOG_STATUS_RUNNING) return COMMON_DIALOG_ERROR_BUSY; + if (param == nullptr) return COMMON_DIALOG_ERROR_ARG_NULL; + g_status = COMMON_DIALOG_STATUS_FINISHED; + return 0; +} + +int APS5_VABI scePlayerSelectionDialogUpdateStatus(void) { + return g_status.load(); +} + +int APS5_VABI scePlayerSelectionDialogGetResult(void* result) { + const int status = g_status.load(); + if (status == COMMON_DIALOG_STATUS_NONE) return COMMON_DIALOG_ERROR_NOT_INITIALIZED; + if (result == nullptr) return COMMON_DIALOG_ERROR_ARG_NULL; + if (status != COMMON_DIALOG_STATUS_FINISHED) return COMMON_DIALOG_ERROR_NOT_FINISHED; + const std::int32_t canceled = COMMON_DIALOG_RESULT_USER_CANCELED; + std::memcpy(result, &canceled, sizeof(canceled)); + return 0; +} + int APS5_VABI scePlayerSelectionDialogTerminate(void) { + if (g_status.exchange(COMMON_DIALOG_STATUS_NONE) == COMMON_DIALOG_STATUS_NONE) return COMMON_DIALOG_ERROR_NOT_INITIALIZED; return 0; } diff --git a/core/libs/tests/GuestPlayerSelectionDialog.cpp b/core/libs/tests/GuestPlayerSelectionDialog.cpp new file mode 100644 index 0000000000..cd5d62693e --- /dev/null +++ b/core/libs/tests/GuestPlayerSelectionDialog.cpp @@ -0,0 +1,50 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include + +extern "C" { +int APS5_VABI scePlayerSelectionDialogInitialize(void); +int APS5_VABI scePlayerSelectionDialogOpen(const void* param); +int APS5_VABI scePlayerSelectionDialogUpdateStatus(void); +int APS5_VABI scePlayerSelectionDialogGetResult(void* result); +int APS5_VABI scePlayerSelectionDialogTerminate(void); +} + +namespace { + +void Require(bool value) { if (!value) std::abort(); } + +constexpr int kStatusFinished = 3; +constexpr int kErrNotInitialized = static_cast(0x80B80003); +constexpr int kErrAlreadyInitialized = static_cast(0x80B80004); +constexpr int kErrNotFinished = static_cast(0x80B80005); +constexpr int kErrArgNull = static_cast(0x80B8000D); +constexpr int kResultUserCanceled = 1; + +} + +int main() { + std::int64_t param = 0; + std::int32_t result[2] = {-1, -1}; + + Require(scePlayerSelectionDialogTerminate() == kErrNotInitialized); + Require(scePlayerSelectionDialogOpen(¶m) == kErrNotInitialized); + Require(scePlayerSelectionDialogGetResult(result) == kErrNotInitialized); + + Require(scePlayerSelectionDialogInitialize() == 0); + Require(scePlayerSelectionDialogInitialize() == kErrAlreadyInitialized); + + Require(scePlayerSelectionDialogOpen(nullptr) == kErrArgNull); + Require(scePlayerSelectionDialogGetResult(result) == kErrNotFinished); + + Require(scePlayerSelectionDialogOpen(¶m) == 0); + Require(scePlayerSelectionDialogUpdateStatus() == kStatusFinished); + + Require(scePlayerSelectionDialogGetResult(nullptr) == kErrArgNull); + Require(scePlayerSelectionDialogGetResult(result) == 0); + Require(result[0] == kResultUserCanceled); + Require(result[1] == -1); + + Require(scePlayerSelectionDialogTerminate() == 0); + Require(scePlayerSelectionDialogTerminate() == kErrNotInitialized); +} diff --git a/docs/dev/debt/libScePlayerSelectionDialog.md b/docs/dev/debt/libScePlayerSelectionDialog.md new file mode 100644 index 0000000000..602e740063 --- /dev/null +++ b/docs/dev/debt/libScePlayerSelectionDialog.md @@ -0,0 +1,5 @@ +# libScePlayerSelectionDialog technical debt + +### Silent stubs + +- [libScePlayerSelectionDialog](../../../core/libs/prx/libScePlayerSelectionDialog/Export.cpp) - the player-selection dialog runs the common-dialog state machine but shows nothing and reports the user as having cancelled. `GetResult` writes only the first `int32` (`USER_CANCELED`); the rest of `ScePlayerSelectionDialogResult` (player list pointer and length) is not filled. EA SPORTS FC 27 (PPSA34015) fails to load `libScePlayerSelectionDialog.prx` without `Initialize`, `Open`, `UpdateStatus` and `GetResult` (`unresolved ELF import` / `GetLastError: 127`) From 0aa3b87448926dbfa853fc7bd4cc0c0ced5bc52b Mon Sep 17 00:00:00 2001 From: valenciana-nc Date: Thu, 15 Oct 2026 02:36:05 +0000 Subject: [PATCH 003/103] fix(libSceAudioOut): check the audio buffer size for 32-bit overflow prepareBuffer and queueAudio computed frames * channels * bytesPerSample in 32-bit arithmetic. A large samplesNum from sceAudioOutOpen (which only rejects len == 0) wraps the product, so buf.resize allocates a truncated buffer while the mixing loops write frames * channels samples past it. Compute the size in 64 bits and throw past INT32_MAX, which also keeps the downstream int casts well-defined. --- core/libs/prx/libSceAudioOut/src/AudioOut.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut.cpp index 2f63a8fb03..ac7919ef81 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut.cpp @@ -94,6 +94,14 @@ static std::uint32_t bytesPerSample(Format f) { return formatIsFloat(f) ? sizeof(float) : sizeof(std::int16_t); } +static std::uint32_t checkedBufferSize(std::uint32_t frames, std::uint32_t channels, std::uint32_t bps) { + const std::uint64_t size = static_cast(frames) * channels * bps; + if (size > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("checkedBufferSize: audio buffer size overflows"); + } + return static_cast(size); +} + struct Port { bool used = false; int type = 0; @@ -160,7 +168,7 @@ static const void* prepareBuffer(const Port& port, const void* data, std::vector const auto frames = port.samplesNum; const auto ch = static_cast(port.channels); const auto bps = bytesPerSample(port.format); - const auto size = frames * ch * bps; + const auto size = checkedBufferSize(frames, ch, bps); bool volumeChanged = port.mixLevel != DEFAULT_VOLUME; for (std::uint32_t i = 0; i < ch; i++) { @@ -227,8 +235,8 @@ static void queueAudio(Port& port, const void* data, std::unique_lock prepareBuf; const void* prepared = prepareBuffer(port, data, prepareBuf); - const std::uint32_t preparedSize = port.samplesNum * - static_cast(port.channels) * bytesPerSample(port.format); + const std::uint32_t preparedSize = checkedBufferSize(port.samplesNum, + static_cast(port.channels), bytesPerSample(port.format)); SDL_AudioCVT cvt{}; const int cvtResult = SDL_BuildAudioCVT( From d4b33b0ae859433a5d15e0911054767663a0ac7e Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:48:55 +0000 Subject: [PATCH 004/103] perf(shader): select the record of a T# whose offset the CPU evaluates Since #757, a T# loaded from a buffer heap at a runtime offset is an image table column, and every capture snapshots the whole heap, up to 65,536 records, even when user data or an SRT read fixes the offset. When the T#'s eight dwords and its offset are runtime values the SRT walker evaluates, the column is now selected: the offset is kept as a descriptor source, the shader indexes entry 0, and the snapshot reads only the record at the evaluated offset, with the heap base moved to that record. The column keeps a table's semantics. An invalid T#, an unmapped record or an offset that carries past 32 bits is poison that faults only when a lane selects it; an offset the walker cannot read faults the table instead of throwing; the record's mode is picked per entry, so no pipeline variant is added; and a dispatch that writes the record faults it as written by its dispatch. Dwords past the heap read zeros, as the scalar loads would, and a record past the key cap is read rather than poisoned. A column stays a whole table when its offset is chosen per lane, or when the shader may write the memory the offset is read from at run time: the buffer it is read from (by V#), or any memory when the read or a write goes through a pointer or a GPU-only descriptor. Samplers are unchanged: an S# the walker evaluates is already a direct sampler. The plan stores the selection, so the disk cache format is 36. Five agc_shader_memory and agc_shader_bindless checks asserted the old planning and now assert the selected record: - "image table SRT key": a key read through the SRT narrowed a 4-key table to records 0 and 2. It now selects record 2 alone, captures no other record, and a changed key moves the capture without changing the module or its disk key. - "image table key union" and "image table late key": two keys of one heap shared one column with two value key domains. Each key now selects its own record, as two one-record tables. - "image table unkeyed access": a read keyed through the SRT shared a whole column with a read keyed per lane. The first now selects its record; the second keeps the whole, unnarrowed table. - "uniform table image": the T# at a user data offset was one key of a full table. It is now the one selected record, and the record the user data does not select is not read. A value key domain only arises for a key the walker evaluates, so it no longer narrows an image column; key buffer domains (keys loaded per lane) still do. --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 85 +++++++++++-------- .../IrMetadata/ControlFlowInfo.hpp | 4 +- .../Optimization/src/ResourceMaterializer.cpp | 52 +++++++++++- .../Optimization/src/ResourceTracker.cpp | 72 +++++++++++++++- core/shader/recompiler/ShaderDiskCache.cpp | 12 ++- core/shader/recompiler/ShaderDiskCache.hpp | 2 +- docs/dev/ARCHITECTURE.md | 6 +- docs/dev/debt/libSceAgcDriver.md | 2 +- docs/dev/debt/shader.md | 4 +- 9 files changed, 188 insertions(+), 51 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 462bbcd8ff..8ba52c6cc2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1054,6 +1054,13 @@ TableCompiled CompileTable(ShaderRecompiler::RecompileRequest request) { return out; } +std::array TableSelected(const ShaderRecompiler::ImageTableSnapshot& tables, std::uint32_t table, const char* message) { + namespace Abi = ShaderRecompiler::ImageTableAbi; + const auto& column = tables.tables.at(table); + require(column.records == 1u && column.keys == 1u && column.codes.size() == 1u && (column.codes[0] & Abi::PoisonFlag) == 0u, message); + return tables.words.at(column.codes[0]).dwords; +} + void verifyImageTableKeys() { using namespace ShaderRecompiler; namespace Abi = ImageTableAbi; @@ -1359,7 +1366,6 @@ void verifyImageTables() { void verifyImageTableKeyUnion() { using namespace ShaderRecompiler; - namespace Abi = ImageTableAbi; static TableTexture textures[3]; alignas(4096) static TableGuest guest; auto& heap = guest.heap; @@ -1374,24 +1380,25 @@ void verifyImageTableKeyUnion() { const auto userData = TableUserData(guest); const auto request = TableRequest(TableTwoKeyCode, userData); const auto plan = GetResourcePlan(request); - require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "image table key union: two keyed reads of one table were not planned as one image of one column"); - const auto& column = plan->descriptorSources.at(plan->info.images[0].source).tableColumn; - require(column.has_value() && column->keyDomains.size() == 2u && column->keyDomains[0].value && column->keyDomains[1].value && column->keyDomains[0].source != column->keyDomains[1].source, "image table key union: the column does not carry both value key domains"); + require(plan->info.images.size() == 2u && plan->info.images[0].table == 0u && plan->info.images[1].table == 1u, "image table key union: two keyed reads of one heap were not planned as two tables"); + const auto& first = plan->descriptorSources.at(plan->info.images[0].source).tableColumn; + const auto& second = plan->descriptorSources.at(plan->info.images[1].source).tableColumn; + require(first.has_value() && second.has_value() && first->selected && second->selected && first->heapSource == second->heapSource && first->selection != second->selection, "image table key union: the two keys do not each select a record of the one heap"); + const auto held = [](const ImageTableSnapshot& tables, std::uint32_t table) { return TableSelected(tables, table, "image table key union: a table does not hold one readable selected record"); }; const auto compiled = CompileTable(request); - const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0); - const auto outside = Abi::PoisonCode(static_cast(Abi::PoisonReason::OutsideDomain)); - require(compiled.capture->snapshot.tables.tables.size() == 1u && snapshot.keys == 4u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && (snapshot.codes[1] & Abi::PoisonFlag) == 0u && snapshot.codes[2] == outside && (snapshot.codes[3] & Abi::PoisonFlag) == 0u, "image table key union: the snapshot does not hold the union of the two keys' records"); + const auto& tables = compiled.capture->snapshot.tables; + require(tables.tables.size() == 2u && held(tables, 0u) == heap[1] && held(tables, 1u) == heap[3], "image table key union: the tables do not hold the two keys' records"); + require(TableCovered(compiled.regions, &heap[1], sizeof(heap[1])) && TableCovered(compiled.regions, &heap[3], sizeof(heap[3])) && !TableCovered(compiled.regions, &heap[0], sizeof(std::uint32_t)) && !TableCovered(compiled.regions, &heap[2], sizeof(std::uint32_t)), "image table key union: records no key selects were captured"); const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable); - require(images != nullptr && images->count == 3u, "image table key union: the binding does not hold the three distinct images of the selected records"); + require(images != nullptr && images->count == 2u, "image table key union: the binding does not hold the two distinct images of the selected records"); guest.srt[17] = 2u; const auto moved = CompileTable(request); - const auto& movedSnapshot = moved.capture->snapshot.tables.tables.at(0); - require((movedSnapshot.codes[2] & Abi::PoisonFlag) == 0u && movedSnapshot.codes[3] == outside && (movedSnapshot.codes[1] & Abi::PoisonFlag) == 0u && moved.result->spirv == compiled.result->spirv, "image table key union: a changed second key did not move the union or changed the module"); + const auto& movedTables = moved.capture->snapshot.tables; + require(held(movedTables, 0u) == heap[1] && held(movedTables, 1u) == heap[2] && TableCovered(moved.regions, &heap[2], sizeof(heap[2])) && !TableCovered(moved.regions, &heap[3], sizeof(std::uint32_t)) && moved.result->spirv == compiled.result->spirv, "image table key union: a changed second key did not move its record alone or changed the module"); } void verifyImageTableLateKey() { using namespace ShaderRecompiler; - namespace Abi = ImageTableAbi; static TableTexture textures[3]; alignas(4096) static TableGuest guest; auto& heap = guest.heap; @@ -1406,15 +1413,16 @@ void verifyImageTableLateKey() { const auto userData = TableUserData(guest); const auto request = TableRequest(TableLateKeyCode, userData); const auto plan = GetResourcePlan(request); - require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "image table late key: two keyed reads of one table were not planned as one image of one column"); - const auto& column = plan->descriptorSources.at(plan->info.images[0].source).tableColumn; - require(column.has_value() && column->keyDomains.size() == 2u && column->keyDomains[0].value && column->keyDomains[1].value && column->keyDomains[0].source != column->keyDomains[1].source, "image table late key: the column does not carry both value key domains"); + require(plan->info.images.size() == 2u && plan->info.images[0].table == 0u && plan->info.images[1].table == 1u, "image table late key: two keyed reads of one heap were not planned as two tables"); + const auto& first = plan->descriptorSources.at(plan->info.images[0].source).tableColumn; + const auto& second = plan->descriptorSources.at(plan->info.images[1].source).tableColumn; + require(first.has_value() && second.has_value() && first->selected && second->selected && first->selection != second->selection, "image table late key: a key read after the first image operation does not select its own record"); + const auto held = [](const ImageTableSnapshot& tables, std::uint32_t table) { return TableSelected(tables, table, "image table late key: a table does not hold one readable selected record"); }; const auto compiled = CompileTable(request); - const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0); - const auto outside = Abi::PoisonCode(static_cast(Abi::PoisonReason::OutsideDomain)); - require(compiled.capture->snapshot.tables.tables.size() == 1u && snapshot.keys == 4u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && (snapshot.codes[1] & Abi::PoisonFlag) == 0u && snapshot.codes[2] == outside && (snapshot.codes[3] & Abi::PoisonFlag) == 0u, "image table late key: the snapshot does not hold the union of the two keys' records"); + const auto& tables = compiled.capture->snapshot.tables; + require(tables.tables.size() == 2u && held(tables, 0u) == heap[1] && held(tables, 1u) == heap[3], "image table late key: the tables do not hold the two keys' records"); const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable); - require(images != nullptr && images->count == 3u, "image table late key: the binding does not hold the three distinct images of the selected records"); + require(images != nullptr && images->count == 2u, "image table late key: the binding does not hold the two distinct images of the selected records"); } void verifyImageTableUnkeyedAccess() { @@ -1437,13 +1445,19 @@ void verifyImageTableUnkeyedAccess() { const auto userData = TableUserData(guest); const auto request = TableRequest(*code, userData); const auto plan = GetResourcePlan(request); - require(!plan->info.images.empty() && std::all_of(plan->info.images.begin(), plan->info.images.end(), [](const ImageResource& image) { return image.table == 0u; }), "image table unkeyed access: the two reads were not planned as one table"); - const auto& column = plan->descriptorSources.at(plan->info.images[0].source).tableColumn; - require(column.has_value() && column->unkeyed && column->keyDomains.empty(), "image table unkeyed access: a column shared with a read that has no key domain kept a key domain"); + const auto& planned = plan->info.images; + require(planned.size() == 2u && std::all_of(planned.begin(), planned.end(), [](const ImageResource& image) { return image.table != NoTable; }), "image table unkeyed access: the two reads were not planned as two tables"); + const auto columnOf = [&](const ImageResource& image) { return *plan->descriptorSources.at(image.source).tableColumn; }; + const auto selected = std::find_if(planned.begin(), planned.end(), [&](const ImageResource& image) { return columnOf(image).selected; }); + const auto whole = std::find_if(planned.begin(), planned.end(), [&](const ImageResource& image) { return !columnOf(image).selected; }); + require(selected != planned.end() && whole != planned.end(), "image table unkeyed access: the read keyed through the SRT does not select its record beside the read keyed per lane"); + require(columnOf(*whole).unkeyed && columnOf(*whole).keyDomains.empty(), "image table unkeyed access: the read keyed per lane kept a key domain"); const auto compiled = CompileTable(request); - const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0); + const auto& tables = compiled.capture->snapshot.tables; + require(TableSelected(tables, selected->table, "image table unkeyed access: a table does not hold one readable selected record") == heap[2], "image table unkeyed access: the read keyed through the SRT does not hold its record"); + const auto& snapshot = tables.tables.at(whole->table); const auto outside = Abi::PoisonCode(static_cast(Abi::PoisonReason::OutsideDomain)); - require(snapshot.keys == 4u && std::none_of(snapshot.codes.begin(), snapshot.codes.end(), [&](std::uint32_t code) { return code == outside; }), "image table unkeyed access: the shared column was narrowed by the keyed read"); + require(snapshot.keys == 4u && std::none_of(snapshot.codes.begin(), snapshot.codes.end(), [&](std::uint32_t entry) { return entry == outside; }), "image table unkeyed access: the table keyed per lane was narrowed"); const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable); require(images != nullptr && images->count == 4u, "image table unkeyed access: the binding does not hold the whole domain"); } @@ -1467,18 +1481,19 @@ void verifyImageTableSrtKey() { const auto plan = GetResourcePlan(request); require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "image table SRT key: the T# was not planned as a table"); const auto& column = plan->descriptorSources.at(plan->info.images[0].source).tableColumn; - require(column.has_value() && !column->keyDomains.empty(), "image table SRT key: a key read through the SRT has no key domain"); + require(column.has_value() && column->selected && column->selection < plan->descriptorSources.size(), "image table SRT key: a T# whose key is read through the SRT does not select its record"); const auto compiled = CompileTable(request); - const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0); - const auto outside = Abi::PoisonCode(static_cast(Abi::PoisonReason::OutsideDomain)); - require(snapshot.records == 4u && snapshot.keys == 4u && snapshot.codes.size() == 4u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && snapshot.codes[1] == outside && (snapshot.codes[2] & Abi::PoisonFlag) == 0u && snapshot.codes[3] == outside, "image table SRT key: the snapshot was not narrowed to the key read through the SRT"); + const auto& tables = compiled.capture->snapshot.tables; + const auto& snapshot = tables.tables.at(0); + require(snapshot.records == 1u && snapshot.keys == 1u && snapshot.codes.size() == 1u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && tables.words.at(snapshot.codes[0]).dwords == heap[2], "image table SRT key: the snapshot does not hold the selected record alone"); const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable); - require(images != nullptr && images->count == 2u, "image table SRT key: the binding holds other entries than the selected one and the first"); - require(TableCovered(compiled.regions, &guest.srt[16], sizeof(std::uint32_t)), "image table SRT key: the key was not captured"); + require(images != nullptr && images->count == 1u, "image table SRT key: the binding holds other entries than the selected one"); + require(TableCovered(compiled.regions, &guest.srt[16], sizeof(std::uint32_t)) && TableCovered(compiled.regions, &heap[2], sizeof(heap[2])), "image table SRT key: the key or the selected record was not captured"); + require(!TableCovered(compiled.regions, &heap[0], sizeof(std::uint32_t)) && !TableCovered(compiled.regions, &heap[1], sizeof(std::uint32_t)) && !TableCovered(compiled.regions, &heap[3], sizeof(std::uint32_t)), "image table SRT key: records the key does not select were captured"); guest.srt[16] = 3u; const auto moved = CompileTable(request); - const auto& movedSnapshot = moved.capture->snapshot.tables.tables.at(0); - require((movedSnapshot.codes[3] & Abi::PoisonFlag) == 0u && movedSnapshot.codes[2] == outside && movedSnapshot.codes[1] == outside && moved.result->spirv == compiled.result->spirv && moved.key == compiled.key, "image table SRT key: a changed key did not move the snapshot alone"); + const auto& movedTables = moved.capture->snapshot.tables; + require(movedTables.words.at(movedTables.tables.at(0).codes.at(0)).dwords == heap[3] && TableCovered(moved.regions, &heap[3], sizeof(heap[3])) && !TableCovered(moved.regions, &heap[2], sizeof(std::uint32_t)) && moved.result->spirv == compiled.result->spirv && moved.key == compiled.key, "image table SRT key: a changed key did not move the selected record alone"); } void verifyImageTableMaskedOffset() { @@ -1987,15 +2002,15 @@ void verifyUniformTableImage() { request.layout.pushConstantSizeBytes = 128; const auto plan = GetResourcePlan(request); const auto& column = plan->descriptorSources.at(plan->info.images.at(0).source).tableColumn; - require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u && column.has_value() && column->stride == 48u && column->dwordCount == 8u, "uniform table image: the T# read from a constant buffer at a user data offset was not planned as one table column"); + require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u && column.has_value() && column->stride == 48u && column->dwordCount == 8u && column->selected, "uniform table image: the T# read from a constant buffer at a user data offset does not select its record"); for (const auto& [key, expected] : {std::pair{2u, second}, std::pair{1u, first}}) { userData[2] = key; AgcDriver::ShaderMemory memory({}); const auto capture = memory.Capture(request); const auto& tables = capture->snapshot.tables; - require(tables.tables.size() == 1u && key < tables.tables[0].codes.size(), "uniform table image: the capture has no table for the user data key"); - const auto code = tables.tables[0].codes[key]; - require((code & ImageTableAbi::PoisonFlag) == 0u && std::equal(expected.begin(), expected.end(), tables.words.at(code).dwords.begin()), "uniform table image: the T# of the record the user data selects was not read"); + require(tables.tables.size() == 1u && TableSelected(tables, 0u, "uniform table image: a table does not hold one readable selected record") == expected, "uniform table image: the T# of the record the user data selects was not read"); + const auto regions = memory.Regions(); + require(TableCovered(regions, guest.table[key].data() + 4, sizeof(expected)) && !TableCovered(regions, guest.table[3u - key].data() + 4, sizeof(std::uint32_t)), "uniform table image: the record the user data does not select was read"); } } diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index c402bbee7e..9be02f1133 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -44,9 +44,11 @@ struct TableColumn { std::uint32_t indirectImmediate = 0; std::uint32_t foldedAddend = 0; std::uint32_t foldedCarry = 0; + bool selected = false; + std::uint32_t selection = 0; bool operator==(const TableColumn& other) const { - return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry; + return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection; } }; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 3ed3870d7b..2f041a1804 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -314,6 +314,7 @@ struct TableCounters { std::array, PoisonReasonCount> poison{}; std::atomic narrowed{0}; std::atomic narrowSkipped{0}; + std::atomic selected{0}; std::atomic nanoseconds{0}; std::atomic lastReport{0}; }; @@ -341,8 +342,8 @@ void reportTables() { if (count == 0) continue; poison += " " + std::string(ImageTableAbi::PoisonReasonName(static_cast(reason))) + "=" + std::to_string(count); } - std::fprintf(stderr, "[image-table] (10 s): %llu snapshots, %llu tables, %llu keys, structural poison:%s, narrowing %llu applied / %llu skipped, %.1f ms\n", - static_cast(snapshots), static_cast(counters.tables.exchange(0, std::memory_order_relaxed)), static_cast(counters.keys.exchange(0, std::memory_order_relaxed)), poison.empty() ? " none" : poison.c_str(), + std::fprintf(stderr, "[image-table] (10 s): %llu snapshots, %llu tables, %llu keys, %llu selected records, structural poison:%s, narrowing %llu applied / %llu skipped, %.1f ms\n", + static_cast(snapshots), static_cast(counters.tables.exchange(0, std::memory_order_relaxed)), static_cast(counters.keys.exchange(0, std::memory_order_relaxed)), static_cast(counters.selected.exchange(0, std::memory_order_relaxed)), poison.empty() ? " none" : poison.c_str(), static_cast(counters.narrowed.exchange(0, std::memory_order_relaxed)), static_cast(counters.narrowSkipped.exchange(0, std::memory_order_relaxed)), static_cast(counters.nanoseconds.exchange(0, std::memory_order_relaxed)) / 1e6); } @@ -526,6 +527,49 @@ private: tableCounters().narrowed.fetch_add(1, std::memory_order_relaxed); } + void Select(const TableColumn& column, ColumnRecords& records) { + const auto base = records.base; + const auto size = records.size; + DescriptorValue selection; + try { + walker.EvaluateDescriptorSource(plan, column.selection, runtime, selection); + } catch (const std::exception&) { + records.fault = ImageTableAbi::PoisonCode(static_cast(PoisonReason::Unmapped)); + return; + } + tableCounters().selected.fetch_add(1, std::memory_order_relaxed); + const auto offset = selection.dwords[0]; + const auto entry = (column.addend + column.offset) & ~3u; + records.base = base + ((offset + column.offset) & ~3u) - entry; + records.size = entry + static_cast(column.dwordCount) * sizeof(std::uint32_t); + records.records = 1u; + records.keys = 1u; + records.states.assign(1u, RecordState::Read); + records.words.assign(1u, 0u); + if (offset >= column.foldedAddend && offset > 0xffffffffu - column.foldedCarry) { + records.states.front() = RecordState::Carried; + return; + } + DescriptorValue value; + value.dwordCount = 8u; + std::uint64_t low = std::numeric_limits::max(); + std::uint64_t high = 0; + for (std::uint32_t dword = 0; dword < column.dwordCount; dword++) { + const auto position = ImageTableAbi::ScalarBufferDword(offset, column.offset + dword * 4u, size); + if (!position.has_value()) continue; + const auto address = base + *position; + if (!PageAccessible(address, base, base + size)) { + records.states.front() = RecordState::Unmapped; + return; + } + value.dwords[dword] = Read(address); + low = std::min(low, address); + high = std::max(high, address + sizeof(std::uint32_t)); + } + records.words.front() = WordsIndex(value); + if (low < high) tables.ranges.emplace_back(low, high - low); + } + std::uint32_t WordsIndex(const DescriptorValue& value) { const auto [found, inserted] = wordsIndex.try_emplace({value.dwords, value.dwordCount}, static_cast(tables.words.size())); if (inserted) tables.words.push_back(value); @@ -556,6 +600,10 @@ private: } records.base = heap.Base48(); records.size = heap.GetSize(); + if (column.selected) { + Select(column, records); + return records; + } records.records = ImageTableAbi::ScalarBufferRecords(column.addend, column.stride, column.offset + (column.indirect ? 4u : 0u), records.size); if (records.size != 0u) tables.ranges.emplace_back(records.base, records.size); } diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index d8634e78ca..580eac4d16 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -146,11 +146,15 @@ public: } std::vector referenced; for (const auto& plan : m_tableColumns) { - for (std::uint32_t argument = 0; argument < plan.handle->ArgumentCount(); argument++) { - plan.handle->ReplaceArgument(argument, plan.offset); - } const auto& column = *m_sources[plan.source].tableColumn; + IrValue* offset = column.selected ? &m_builder.Constant(column.addend) : plan.offset; + for (std::uint32_t argument = 0; argument < plan.handle->ArgumentCount(); argument++) { + plan.handle->ReplaceArgument(argument, offset); + } KeepSourceAlive(*plan.handle, m_sources[column.heapSource], referenced); + if (column.selected) { + KeepSourceAlive(*plan.handle, m_sources[column.selection], referenced); + } if (plan.keyDomain.has_value()) { KeepSourceAlive(*plan.handle, m_sources[plan.keyDomain->source], referenced); } @@ -650,6 +654,18 @@ private: column.address = address; column.foldedAddend = folded ? baseAddend : 0u; column.foldedCarry = folded ? carry : 0u; + if (!address && !sampler) { + DescriptorSource direct; + MakeSource(handle, width, false, false, direct); + DescriptorSource selection; + selection.dwordCount = 1u; + selection.dwords[0] = offset; + std::uint32_t badDword = 0; + if (ValidateSource(direct, badDword) && ValidateSource(selection, badDword)) { + column.selected = true; + column.selection = InternSource(selection); + } + } const auto domain = masked ? std::optional{} : MatchKeyDomain(key); DescriptorSource columnSource = heapSource; columnSource.tableColumn = column; @@ -1336,8 +1352,58 @@ private: return source < m_sources.size() && m_sources[source].tableColumn.has_value(); } + bool SameBufferSource(const IrValue& handle, std::uint32_t source) const { + const auto& current = m_sources[source]; + if (current.tableColumn.has_value() || current.dwordCount != 4u || handle.ArgumentCount() != 4u) { + return false; + } + for (std::uint32_t dword = 0; dword < 4u; dword++) { + if (!EquivalentValue(m_program.Resources(), current.dwords[dword], handle.Argument(dword))) { + return false; + } + } + return true; + } + + bool SelectionWritten(const TableColumn& column) const { + const auto written = [&](const auto& matches) { + return std::any_of(m_info.buffers.begin(), m_info.buffers.end(), [&](const auto& buffer) { return (buffer.written || buffer.atomic) && matches(buffer.source); }); + }; + std::vector pending{m_sources[column.selection].dwords[0]}; + std::vector seen; + while (!pending.empty()) { + const IrValue* current = pending.back()->Resolve(); + pending.pop_back(); + if (std::ranges::find(seen, current) != seen.end()) { + continue; + } + seen.push_back(current); + const bool pointer = current->Opcode() == IrOpcode::LoadAddressU32; + const bool buffer = current->Opcode() == IrOpcode::ReadConstBuffer && current->ArgumentCount() != 0u; + if ((pointer || buffer) && m_info.bdaWrites) { + return true; + } + if (pointer && written([](std::uint32_t) { return true; })) { + return true; + } + if (buffer) { + const IrValue* handle = current->Argument(0)->Resolve(); + if (written([&](std::uint32_t source) { return SameBufferSource(*handle, source); })) { + return true; + } + } + for (std::size_t index = 0; index < current->ArgumentCount(); index++) { + pending.push_back(current->Argument(index)); + } + } + return false; + } + void CheckTableStores() { for (auto& source : m_sources) { + if (source.tableColumn.has_value() && source.tableColumn->selected && SelectionWritten(*source.tableColumn)) { + source.tableColumn->selected = false; + } if (!source.tableColumn.has_value() || source.tableColumn->keyDomains.empty()) { continue; } diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index c0a118aeb8..05c6eea6e1 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -108,8 +108,8 @@ static_assert(sizeof(IrResourcePlan) == 584, "IrResourcePlan changed: update Enc static_assert(sizeof(IrValue) == 152, "IrValue changed: update the plan value encoder"); static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder"); static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder"); -static_assert(sizeof(DescriptorSource) == 160, "DescriptorSource changed: update the plan encoder"); -static_assert(sizeof(TableColumn) == 80, "TableColumn changed: update the plan encoder"); +static_assert(sizeof(DescriptorSource) == 168, "DescriptorSource changed: update the plan encoder"); +static_assert(sizeof(TableColumn) == 88, "TableColumn changed: update the plan encoder"); static_assert(sizeof(KeyDomain) == 16, "KeyDomain changed: update the plan encoder"); static_assert(sizeof(ResourceBlock) == 80, "ResourceBlock changed: update the plan encoder"); static_assert(sizeof(SrtRead) == 16, "SrtRead changed: update the plan encoder"); @@ -149,7 +149,7 @@ static_assert(fieldCount() == 21, "IrResourcePlan changed: updat static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); static_assert(fieldCount() == 31, "MemoryInfo changed: update the plan encoder"); static_assert(fieldCount() == 3, "DescriptorSource changed: update the plan encoder"); -static_assert(fieldCount() == 15, "TableColumn changed: update the plan encoder"); +static_assert(fieldCount() == 17, "TableColumn changed: update the plan encoder"); static_assert(fieldCount() == 4, "KeyDomain changed: update the plan encoder"); static_assert(fieldCount() == 4, "ResourceBlock changed: update the plan encoder"); static_assert(fieldCount() == 2, "SrtRead changed: update the plan encoder"); @@ -1041,6 +1041,8 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) { out.Value(column.indirectImmediate); out.Value(column.foldedAddend); out.Value(column.foldedCarry); + out.Value(column.selected); + out.Value(column.selection); }); writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) { out.Value(valueIndex(block.condition)); @@ -1195,6 +1197,8 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { in.Value(column.indirectImmediate); in.Value(column.foldedAddend); in.Value(column.foldedCarry); + in.Value(column.selected); + in.Value(column.selection); source.tableColumn = table ? std::optional(std::move(column)) : std::nullopt; }); reader.List(plan.controlFlow, 28, [&](Reader& in, ResourceBlock& block) { @@ -1226,7 +1230,7 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { if (!valid || !reader.Done() || fill.words > plan.uniformFill.values.size()) return false; const auto knownSource = [&](std::uint32_t source) { return source < plan.descriptorSources.size(); }; if (!std::ranges::all_of(plan.materializationSources, knownSource)) return false; - if (!std::ranges::all_of(plan.descriptorSources, [&](const DescriptorSource& source) { return source.dwordCount <= source.dwords.size() && (!source.tableColumn.has_value() || (knownSource(source.tableColumn->heapSource) && std::ranges::all_of(source.tableColumn->keyDomains, [&](const KeyDomain& domain) { return knownSource(domain.source); }))); })) return false; + if (!std::ranges::all_of(plan.descriptorSources, [&](const DescriptorSource& source) { return source.dwordCount <= source.dwords.size() && (!source.tableColumn.has_value() || (knownSource(source.tableColumn->heapSource) && (!source.tableColumn->selected || knownSource(source.tableColumn->selection)) && std::ranges::all_of(source.tableColumn->keyDomains, [&](const KeyDomain& domain) { return knownSource(domain.source); }))); })) return false; if (!std::ranges::all_of(plan.info.buffers, [&](const BufferResource& buffer) { return knownSource(buffer.source); })) return false; if (!std::ranges::all_of(plan.info.images, [&](const ImageResource& image) { return knownSource(image.source); })) return false; if (!std::ranges::all_of(plan.info.samplers, [&](const SamplerResource& sampler) { return knownSource(sampler.source); })) return false; diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 9e59550c01..b03d85a809 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 35; +inline constexpr std::uint32_t FormatVersion = 36; enum class LoadStatus { Loaded, diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index ffbc3207e3..573bf0ac92 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -111,11 +111,13 @@ flowchart LR A T# or S# that a shader loads from a buffer at a runtime offset (`key * stride + addend`, or `(key << n) & mask`) is not a direct descriptor. [`ResourceTracker.cpp`](../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) plans it as a table column (`TableColumn`): the heap V#'s descriptor source, the record stride and addend, the descriptor's offset in the record, and the key domains that may narrow it. Accesses with the same heap and record shape share one column. The shader keeps computing the offset and indexes a table binding: images use one variable-count `SAMPLED_IMAGE` array (`RuntimeAbi::ImageTableBinding`, in the fragment set when pipeline libraries split the sets), samplers a `SAMPLER` array (`Binding::SamplerTable`). Devices without `runtimeDescriptorArray`, `descriptorBindingPartiallyBound` and `descriptorBindingVariableDescriptorCount` refuse such shaders. -At every invocation, `TableSnapshotter` in [`ResourceMaterializer.cpp`](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) reads each column's records from guest memory into `ResourceSnapshot::tables`: heap base and size, the record count (at most `ImageTableAbi::MaxKeys`), one code per record (an index into the distinct descriptor words, or a poison code) and the ranges it read. A column narrows to the records its keys select, plus record 0, when every access has a key domain: a value the SRT walker evaluates from user data, SRT reads and constants, or a key buffer of at most 256 positions (`APS5_IMAGE_TABLE_KEY_SCAN`). A loop-carried, lane-reduced or masked key keeps the whole column, so the cost of an invocation grows with the column's records and its distinct T#s. The snapshot is part of the invocation, not of the compiled variant: a changed table changes neither the SPIR-V nor the variant's disk key. +A T# whose eight dwords and record offset the SRT walker can evaluate is planned as a selected column (`TableColumn::selected`): the offset is kept as a descriptor source (`TableColumn::selection`) and the shader indexes entry 0 of a one-record table. The column stays a whole table when the shader may write the memory its offset is read from at run time: the buffer it is read from, or any memory when the read or a write goes through a pointer or a GPU-only descriptor. Samplers are not selected: an S# the walker can evaluate is a direct sampler. + +At every invocation, `TableSnapshotter` in [`ResourceMaterializer.cpp`](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) reads each column's records from guest memory into `ResourceSnapshot::tables`: heap base and size, the record count (at most `ImageTableAbi::MaxKeys`), one code per record (an index into the distinct descriptor words, or a poison code) and the ranges it read. A selected column evaluates its offset and reads only that record, as the scalar loads would: dwords past the heap read zeros, an unmapped record or an offset that carries past 32 bits is poison, and an offset the walker cannot read faults the table instead of throwing. Its heap base moves to the record, so the shader's fault address and the written-heap check cover that record alone. Any other column narrows to the records its keys select, plus record 0, when every access has a key domain: a key buffer of at most 256 positions (`APS5_IMAGE_TABLE_KEY_SCAN`), or a value the SRT walker evaluates from user data, SRT reads and constants (which, for an image, selects its record before it could narrow). A loop-carried, lane-reduced or masked key keeps the whole column, so the cost of an invocation grows with the column's records and its distinct T#s. The snapshot is part of the invocation, not of the compiled variant: a changed table or key changes neither the SPIR-V nor the variant's disk key. `ResolveImageTables` turns the snapshot into the map buffer (`DescriptorRole::ImageTableMap`, layout in [`ImageTableAbi.hpp`](../../core/shader/recompiler/ImageTableAbi.hpp)): a header, eight words per table (map start, key count, heap base and size, the code for keys past the snapshot, a table fault) and one record per key (flags with the decode mode, the swizzle dword and the element for images; flags and element for samplers). The shader reads `map[key]` and switches over the image's prepared runtime modes. Each distinct T# becomes one element, up to `ImageElementBudget` and the device's capacity. An entry that does not decode, is multisampled or over budget, pairs a reducing sampler with a point-only mode, is an unnormalized or reducing S#, or whose surface has no mapped byte is poisoned: it reads zeros and reports a fault through the BDA fault buffer only when a lane selects it (`APS5_IMAGE_TABLE_STRICT=1` throws at the snapshot instead). The driver's `resolveImageTables` in [`ShaderResources.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) looks up or creates a texture per element and keeps the last four resolved tables per thread for reuse within one collect epoch while their memory is unchanged. Cached objects grow with their tables, so the resource cache keeps at most `ResourceCache::TableElementBudget()` table elements (65,536, `APS5_RESOURCE_CACHE_TABLE_ELEMENTS`) and evicts its least recently used table objects past that, and the dispatch cache evicts its least recently used entries while their captured words exceed `Driver::DispatchCacheBytes()` (256 MiB, `APS5_DISPATCH_CACHE_BYTES`). -The disk cache (`FormatVersion` 33) stores the table and table operation of each image and sampler, the image shapes of the table binding, the invocation's table poison list, read ranges and fault count, and the table column of each descriptor source in the front-end plan, with its key domains. +The disk cache (`FormatVersion` 36) stores the table and table operation of each image and sampler, the image shapes of the table binding, the invocation's table poison list, read ranges and fault count, and the table column of each descriptor source in the front-end plan, with its key domains and its selection. ### Untracked storage image comparisons diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 720fc6ccbd..8c381cdf4f 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -129,7 +129,7 @@ - [Table S#s](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) that clamp to border fault only when selected: four variants cannot encode dynamic integer/float pairing. This also rejects otherwise-valid float pairs. - [Table T#s](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) paired with direct S#s fault when a paired border class differs from the decoded image class. Depth-bit views use float borders. Matching float pairs and inactive mismatches do not fault. - [Table S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) DISABLE_CUBE_WRAP faults on selection without `VK_EXT_non_seamless_cube_map`: no runtime dimension check, including valid non-cube direct/table T#s. Unselected entries do not fault; with the feature this bit alone is accepted. Table T#/direct S# faults only for cubes needing the missing feature. -- An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) whose record range overlaps a buffer or storage image written by its own dispatch faults (`table written by its dispatch`) when any entry is selected, even if the write misses the selected record. +- An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) whose record range overlaps a buffer or storage image written by its own dispatch faults (`table written by its dispatch`) when any entry is selected, even if the write misses the selected record. A selected column's range is its record and the bytes of the record before the descriptor. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) T# whose surface has no mapped byte faults when selected instead of becoming a zero texture as a direct T# does; what the console returns for it is not measured. - [User-data callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped from the recompiled code; in the shaders seen they are metadata or constants the caller reads relative to its program counter, and those reads go to the captured guest code, which linking does not change. A caller that jumps through a table among those words, which the control flow analysis reads from the recompiled code, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. - [Table callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is two consecutive dwords of an `s_buffer_load_dword*`, or a copy of them made with `s_mov_b32` or `s_mov_b64` in the same block, calls one of the targets that buffer holds when the load's buffer descriptor comes from an `s_load_dwordx4` without SOFFSET from a user-data pointer and the load's SOFFSET from an `s_lshl_b32` by 2 to 24 bits. At dispatch the descriptor and the buffer are read from guest memory, and every non-zero 64-bit value at the loaded offset plus a multiple of the shift's stride, below the descriptor's size (stride times num_records, or num_records bytes with stride 0), is read as a user-data callee and appended once per call. The call becomes a branch to a chain that compares the source pair with each target (`s_cmp_lg_u32` with a literal and `s_cbranch_scc1` per dword) and `s_call_b64`s the copy that matches; no match runs the original `s_swappc_b64` as an unlinked call. There is one chain for each value SCC has at the call, and the chain sets SCC back to that value (`s_cmp_eq_u32 0, 0` or `s_cmp_lg_u32 0, 0`) before the callee or the unlinked call runs. The searched range is the descriptor's whole size, which covers the range the load can reach whether it is checked in bytes or in strides; which one the PS5 uses for `s_buffer_load` was not measured. More than 64 distinct targets, a buffer above 64 MiB or a copy out of `s_call_b64` range throws. As with user-data callees, the buffer is read again only when a dispatch misses the dispatch cache, so a target a title adds to the buffer later is an unlinked call when a cached program runs. diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index bbae61a42b..61ceaa79aa 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -30,9 +30,9 @@ - [Image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures are decoded by `image_load` like a direct T#; a sample, gather or LOD query of one faults (`sRGB decode`) on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Image tables](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) (T#s and S#s that `s_buffer_load` reads at `key * stride + addend` from a V#) are snapshotted when the dispatch or draw is recorded; a later CPU write to a record is not seen. - An [image table](../../../core/shader/recompiler/ImageTableAbi.hpp) entry that cannot be bound exactly reads zeros, and an active lane that selects it raises a BDA fault (reason 9) naming the record's address, words and reason; the dispatch's write-back reports the first one. -- An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) snapshot holds at most 65536 keys per table, 65536 distinct images and 256 S# entries per shader; a key past a longer table's snapshot, or an entry over the budget, faults when selected. +- An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) snapshot holds at most 65536 keys per table, 65536 distinct images and 256 S# entries per shader; a key past a longer table's snapshot, or an entry over the budget, faults when selected. A selected column (a T# the SRT walker evaluates) holds its one record wherever it lies. - [Image tables](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read by `s_load` through a 64-bit pointer hold the keys the key's known bits allow, at most 65536; a key past them, or an entry at a wrapping, unaligned or unmapped address, faults when selected. -- An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) whose key is neither a V#-relative scalar read nor a value the SRT walker evaluates from user data, SRT reads and constants alone (for example a shifted or masked runtime-offset V# read, a per-lane value made uniform by `v_readfirstlane`, or a loop-carried phi) binds every record of the snapshot domain instead of the selected ones. So does a table read at `(key << n) & mask`: a key domain would select records by the unmasked key, so that key is not narrowed. +- An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) whose key is neither a V#-relative scalar read nor a value the SRT walker evaluates (for example a shifted or masked runtime-offset V# read, a per-lane value made uniform by `v_readfirstlane`, or a loop-carried phi) binds every record of the snapshot domain instead of the selected ones. So does a table read at `(key << n) & mask` with such a key: a key domain would select records by the unmasked key, so that key is not narrowed. A key the walker evaluates selects its one record instead, unless the shader may write the memory the key is read from at run time (the same buffer, or any memory when the read or a write goes through a pointer or a GPU-only descriptor); then the column binds the whole domain too. A write to the key's memory through another V# over the same bytes is not detected. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) T# of an MSAA or FMASK image, or of a color format under a depth compare, faults when selected: tables have no multisampled descriptors and no emulated color compare. - `image_store`, image atomics, PCK2/PCK4 and MIMG BY2/BY4 through an [image table](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) T# fault (`unsupported operation`) when an active lane executes them; the store is dropped and the load reads zeros. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) S# with a min or max reduction or FORCE_UNNORMALIZED faults when selected, as does an integer, converted or depth-bits table T# paired with a direct min or max S# that filters between texels. From e1472723680f6b16c856f01542350ac8c5044360 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:49:16 +0000 Subject: [PATCH 005/103] feat(shader): name why image table narrowing is skipped The [image-table] line under APS5_PROFILE_DRAW now splits the skipped narrowings by reason: a value key that cannot be evaluated, a key descriptor that is not a buffer, a key buffer with more positions than APS5_IMAGE_TABLE_KEY_SCAN (with the most positions seen), pending GPU writes over the key buffer, and an unmapped key page. Selected records never narrow, so the line counts them apart. --- .../Optimization/src/ResourceMaterializer.cpp | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 2f041a1804..95b984bda9 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -307,6 +307,9 @@ bool TableTraced() { constexpr std::size_t PoisonReasonCount = static_cast(PoisonReason::Count); +enum class NarrowSkip : std::size_t { Value, NotBuffer, KeyScan, Pending, Unmapped, Count }; +constexpr std::array(NarrowSkip::Count)> NarrowSkipNames{"value", "not-buffer", "key-scan", "pending-write", "unmapped"}; + struct TableCounters { std::atomic snapshots{0}; std::atomic tables{0}; @@ -314,6 +317,8 @@ struct TableCounters { std::array, PoisonReasonCount> poison{}; std::atomic narrowed{0}; std::atomic narrowSkipped{0}; + std::array, static_cast(NarrowSkip::Count)> narrowSkips{}; + std::atomic narrowPositions{0}; std::atomic selected{0}; std::atomic nanoseconds{0}; std::atomic lastReport{0}; @@ -342,9 +347,16 @@ void reportTables() { if (count == 0) continue; poison += " " + std::string(ImageTableAbi::PoisonReasonName(static_cast(reason))) + "=" + std::to_string(count); } - std::fprintf(stderr, "[image-table] (10 s): %llu snapshots, %llu tables, %llu keys, %llu selected records, structural poison:%s, narrowing %llu applied / %llu skipped, %.1f ms\n", + std::string skips; + for (std::size_t reason = 0; reason < counters.narrowSkips.size(); reason++) { + const auto count = counters.narrowSkips[reason].exchange(0, std::memory_order_relaxed); + if (count == 0) continue; + skips += " " + std::string(NarrowSkipNames[reason]) + "=" + std::to_string(count); + if (reason == static_cast(NarrowSkip::KeyScan)) skips += " (up to " + std::to_string(counters.narrowPositions.exchange(0, std::memory_order_relaxed)) + " positions, APS5_IMAGE_TABLE_KEY_SCAN " + std::to_string(TableKeyScanLimit()) + ")"; + } + std::fprintf(stderr, "[image-table] (10 s): %llu snapshots, %llu tables, %llu keys, %llu selected records, structural poison:%s, narrowing %llu applied / %llu skipped%s%s, %.1f ms\n", static_cast(snapshots), static_cast(counters.tables.exchange(0, std::memory_order_relaxed)), static_cast(counters.keys.exchange(0, std::memory_order_relaxed)), static_cast(counters.selected.exchange(0, std::memory_order_relaxed)), poison.empty() ? " none" : poison.c_str(), - static_cast(counters.narrowed.exchange(0, std::memory_order_relaxed)), static_cast(counters.narrowSkipped.exchange(0, std::memory_order_relaxed)), static_cast(counters.nanoseconds.exchange(0, std::memory_order_relaxed)) / 1e6); + static_cast(counters.narrowed.exchange(0, std::memory_order_relaxed)), static_cast(counters.narrowSkipped.exchange(0, std::memory_order_relaxed)), skips.empty() ? "" : ":", skips.c_str(), static_cast(counters.nanoseconds.exchange(0, std::memory_order_relaxed)) / 1e6); } enum class RecordState : std::uint8_t { OutsideDomain, Read, Unmapped, Carried }; @@ -466,8 +478,10 @@ private: } void Narrow(const TableColumn& column, ColumnRecords& records) { - const auto skip = [&] { - tableCounters().narrowSkipped.fetch_add(1, std::memory_order_relaxed); + const auto skip = [&](NarrowSkip reason) { + auto& counters = tableCounters(); + counters.narrowSkipped.fetch_add(1, std::memory_order_relaxed); + counters.narrowSkips[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); }; std::vector values{0u}; std::vector> scanned; @@ -477,7 +491,7 @@ private: try { walker.EvaluateDescriptorSource(plan, domain.source, runtime, key); } catch (const std::exception&) { - skip(); + skip(NarrowSkip::Value); return; } values.push_back(key.dwords[0]); @@ -485,7 +499,7 @@ private: } const auto keyBuffer = EvaluateBuffer(domain.source); if (keyBuffer.Type() != 0u) { - skip(); + skip(NarrowSkip::NotBuffer); return; } const auto base = keyBuffer.Base48(); @@ -493,17 +507,20 @@ private: const std::uint64_t first = domain.offset & ~3u; const std::uint64_t positions = size >= first + 4u ? (size - first - 4u) / domain.stride + 1u : 0u; if (positions > TableKeyScanLimit() || first > 0xffffffffull) { - skip(); + auto& widest = tableCounters().narrowPositions; + auto seen = widest.load(std::memory_order_relaxed); + while (seen < positions && !widest.compare_exchange_weak(seen, positions, std::memory_order_relaxed)) {} + skip(NarrowSkip::KeyScan); return; } if (positions != 0u && runtime.pendingWrite != nullptr && runtime.pendingWrite(runtime.userContext, base + first, size - first)) { - skip(); + skip(NarrowSkip::Pending); return; } for (std::uint64_t index = 0; index < positions; index++) { const auto address = base + first + index * domain.stride; if (!PageAccessible(address, base, base + size)) { - skip(); + skip(NarrowSkip::Unmapped); return; } values.push_back(Read(address)); From 43855cb5e8c5dd8cf3f8051d89ffc39f3916bd75 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:49:29 +0000 Subject: [PATCH 006/103] test(shader): cover selected image table records and their fallbacks agc_shader_memory and agc_shader_bindless: an invalid selected T# is poisoned with its words without changing the artifact; a key past the heap selects a null image without reading the heap; an unmapped selected record is Unmapped poison and is not read; a key loaded from a constant buffer at an SRT offset selects its record, and one that cannot be read faults the table instead of throwing; a key loaded from a buffer the shader writes keeps the whole table; a key chosen per lane keeps the table; a record past the 65,536-key cap is selected; and a selected record whose second load carries past 32 bits is OffsetCarry poison while the record past the wrap is read. agc_driver_image_table: a dispatch keyed from a constant buffer at an SRT offset samples three records exactly as their direct bindings do, with one module and one bound image; samples zeros for a key past the table; does not fault for an invalid record under a clear EXEC; reports the invalid record's address, words and reason when it selects it; and faults when it stores over its selected record. agc_shader_disk_cache: a selected column round-trips in the hand-built plan, a fixture plans one and materializes it after a round trip, and a selection past the sources is rejected. --- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 8 ++ .../libSceAgcDriver/tests/ShaderMemory.cpp | 127 ++++++++++++++++++ .../tests/execution/ImageTable.cpp | 52 +++++++ 3 files changed, 187 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 6d79855198..0b8ce3301d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1414,6 +1414,10 @@ std::vector planFixtures() { add("bindless material table", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x93109010u, 0xf4200406u, 0x20000004u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}}); add("bindless whole table", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}}); add("bindless sample in a loop latch", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xbea80380u, 0xf09c0f08u, 0x00450000u, 0x80288128u, 0xbf0a8428u, 0xbf85fffbu, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}}); + auto keyed = tables; + keyed.resize(20u, 0u); + keyed[16] = 3u; + add("bindless selected record", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0xf4000400u, 0xfa000040u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, keyed}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}}); std::vector phis(32, 0u); place(phis, 0, imageWords(PlanTextureAddress)); @@ -1559,6 +1563,8 @@ IrResourcePlan handBuiltPlan() { table.dwordCount = 8u; table.dwords[0] = &readConst(3); table.tableColumn = TableColumn{2u, 32u, 16u, 4u, 8u, 0xffu, false, true, {KeyDomain{3u, 4u, 16u, false}, KeyDomain{2u, 8u, 4u, true}}, false}; + table.tableColumn->selected = true; + table.tableColumn->selection = 2u; DescriptorSource heap; heap.dwordCount = 4u; heap.dwords[1] = &readConst(6); @@ -1807,6 +1813,7 @@ void verifyMalformedPlans() { rejects("a uniform fill of five words", [](IrResourcePlan& plan) { plan.uniformFill.fill.words = 5u; }); rejects("an image table heap past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->heapSource = 4u; }); rejects("an image table key past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->keyDomains[1].source = 4u; }); + rejects("an image table selection past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->selection = 4u; }); auto foreign = handBuiltPlan(); IrValue stray(IrOpcode::Void, IrType::U32, 99u); foreign.srtReads[0].value = &stray; @@ -1866,6 +1873,7 @@ void verifyPlanRoundTrip() { require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.tableColumn.has_value() && source.tableColumn->keyDomains.empty(); }); }), "no fixture plans a whole image table"); require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.tableColumn.has_value() && !source.tableColumn->keyDomains.empty(); }); }), "no fixture plans an image table with a key domain"); require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.tableColumn.has_value() && source.tableColumn->address; }); }), "no fixture plans an image table through a pointer"); + require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.tableColumn.has_value() && source.tableColumn->selected; }); }), "no fixture plans a selected image table record"); require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.controlFlow.begin(), plan.controlFlow.end(), [](const ResourceBlock& block) { return block.condition != nullptr; }); }), "no fixture plans a control-flow condition"); require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.valueStorage.begin(), plan.valueStorage.end(), [](const auto& value) { return value.IsPhi(); }); }), "no fixture plans a phi"); require(covers([](const IrResourcePlan& plan) { return !plan.guardedSrtSlots.empty(); }), "no fixture plans a guarded SRT read"); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 8ba52c6cc2..3980d639f4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1781,6 +1781,131 @@ void verifyPointerTableOffsetCarry() { #endif } +void verifySelectedRecordFallbacks() { + using namespace ShaderRecompiler; + namespace Abi = ImageTableAbi; + static TableTexture textures[2]; + alignas(4096) static TableGuest guest; + auto& heap = guest.heap; + std::fill(heap.begin(), heap.end(), std::array{}); + heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D); + heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D); + heap[2] = heap[1]; + heap[3] = heap[0]; + FillTableSrt(guest, heap.data(), 32u, 4u); + guest.materials = {}; + guest.srt[16] = 2u; + const auto userData = TableUserData(guest); + const auto request = TableRequest(TableSrtKeyCode, userData); + const auto valid = CompileTable(request); + + heap[2][3] &= 0x0fffffffu; + const auto invalid = CompileTable(request); + const auto& invalidMap = TableMap(*invalid.result); + const auto invalidCodes = TableCodes(invalidMap, 0); + require(invalidCodes.size() == 1u && invalidCodes[0] != Abi::NullCode && (invalidCodes[0] & Abi::PoisonFlag) == 0u, "selected record fallbacks: an invalid selected T# has no entry"); + const auto invalidFlags = invalidMap.at(invalidCodes[0] + Abi::ImageRecordFlags); + require((invalidFlags & Abi::PoisonFlag) != 0u, "selected record fallbacks: an invalid selected T# was bound"); + const auto& poison = invalid.result->imageTablePoison.at(invalidFlags & ~Abi::PoisonFlag); + require(poison.reason == static_cast(Abi::PoisonReason::NotImage) && poison.dwordCount == 8u && poison.words[3] == heap[2][3] && invalid.result->imageTableFaults != 0u, "selected record fallbacks: an invalid selected T# was not poisoned with its words"); + require(invalid.result->variantId == valid.result->variantId, "selected record fallbacks: an invalid selected T# changed the artifact"); + heap[2] = heap[1]; + + guest.srt[16] = 200u; + const auto beyond = CompileTable(request); + const auto beyondCodes = TableCodes(TableMap(*beyond.result), 0); + require(beyondCodes.size() == 1u && beyondCodes[0] == Abi::NullCode && beyond.result->imageTableFaults == 0u && !TableTouched(beyond.regions, heap.data(), sizeof(heap)), "selected record fallbacks: a key past the heap did not select a null image without reading the heap"); + guest.srt[16] = 2u; + + void* reserved = nullptr; +#ifdef _WIN32 + reserved = VirtualAlloc(nullptr, 4096, MEM_RESERVE, PAGE_NOACCESS); +#else + reserved = mmap(nullptr, 4096, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) reserved = nullptr; +#endif + require(reserved != nullptr, "selected record fallbacks: cannot reserve an inaccessible page"); + FillTableSrt(guest, reserved, 32u, 4u); + const auto unmapped = CompileTable(request); + const auto& unmappedTable = unmapped.capture->snapshot.tables.tables.at(0); + require(unmappedTable.keys == 1u && unmappedTable.codes.at(0) == Abi::PoisonCode(static_cast(Abi::PoisonReason::Unmapped)) && !TableTouched(unmapped.regions, reserved, 4096u), "selected record fallbacks: an unmapped selected record is not Unmapped poison or was read"); + FillTableSrt(guest, heap.data(), 32u, 4u); + + std::vector loadedKey(TableSrtKeyCode); + const auto shift = std::find(loadedKey.begin(), loadedKey.end(), 0x8f108510u); + require(shift != loadedKey.end(), "selected record fallbacks: the program has no s_lshl_b32"); + loadedKey.insert(shift, {0xf4200406u, 0x20000000u}); + guest.materials[0][1] = 3u; + guest.srt[16] = 4u; + const auto loadedRequest = TableRequest(loadedKey, userData); + const auto loadedPlan = GetResourcePlan(loadedRequest); + require(loadedPlan->info.images.size() == 1u && loadedPlan->descriptorSources.at(loadedPlan->info.images[0].source).tableColumn->selected, "selected record fallbacks: a key loaded from a constant buffer at an SRT offset does not select its record"); + const auto loaded = CompileTable(loadedRequest); + require(TableSelected(loaded.capture->snapshot.tables, 0u, "selected record fallbacks: a loaded key does not hold one readable selected record") == heap[3] && TableCovered(loaded.regions, &guest.materials[0][1], sizeof(std::uint32_t)), "selected record fallbacks: the record of a loaded key was not selected or its key was not captured"); + const auto materials = TableV(guest.materials.data(), 16u, 3u); + const auto unreadable = TableV(reserved, 16u, 3u); + std::copy(unreadable.begin(), unreadable.end(), guest.srt.begin() + 8); + const auto lost = CompileTable(loadedRequest); + const auto& lostTable = lost.capture->snapshot.tables.tables.at(0); + require(lostTable.keys == 0u && lostTable.fault == Abi::PoisonCode(static_cast(Abi::PoisonReason::Unmapped)) && lost.result->imageTableFaults != 0u && lost.result->variantId == loaded.result->variantId, "selected record fallbacks: a key that cannot be read did not fault its table"); + std::copy(materials.begin(), materials.end(), guest.srt.begin() + 8); +#ifdef _WIN32 + VirtualFree(reserved, 0, MEM_RELEASE); +#else + munmap(reserved, 4096); +#endif + + auto writtenKey = loadedKey; + const auto store = std::find(writtenKey.begin(), writtenKey.end(), 0x80070000u); + require(store != writtenKey.end(), "selected record fallbacks: the program has no buffer_store_dword"); + *store = 0x80030000u; + const auto writtenRequest = TableRequest(writtenKey, userData); + const auto writtenPlan = GetResourcePlan(writtenRequest); + const auto& writtenColumn = writtenPlan->descriptorSources.at(writtenPlan->info.images.at(0).source).tableColumn; + require(writtenColumn.has_value() && !writtenColumn->selected && writtenColumn->keyDomains.empty(), "selected record fallbacks: a key loaded from a buffer the shader writes selected its record"); + const auto written = CompileTable(writtenRequest); + require(written.capture->snapshot.tables.tables.at(0).keys == 4u, "selected record fallbacks: a key loaded from a buffer the shader writes did not keep the whole table"); + + const auto lanePlan = GetResourcePlan(TableRequest(TableWholeCode, userData)); + require(!lanePlan->info.images.empty() && !lanePlan->descriptorSources.at(lanePlan->info.images[0].source).tableColumn->selected, "selected record fallbacks: a T# selected per lane left the table"); + + alignas(4096) static std::array, 70000> large{}; + large.back() = heap[1]; + FillTableSrt(guest, large.data(), 32u, static_cast(large.size())); + guest.srt[16] = static_cast(large.size() - 1u); + const auto pastCap = CompileTable(request); + require(TableSelected(pastCap.capture->snapshot.tables, 0u, "selected record fallbacks: a record past the key cap does not hold one readable selected record") == heap[1] && !pastCap.capture->snapshot.tables.tables[0].outside && pastCap.result->imageTableFaults == 0u, "selected record fallbacks: a record past the key cap was not selected"); + FillTableSrt(guest, heap.data(), 32u, 4u); +} + +void verifySelectedRecordOffsetCarry() { + using namespace ShaderRecompiler; + namespace Abi = ImageTableAbi; + static TableTexture texture; + alignas(4096) static TableGuest guest; + std::fill(guest.heap.begin(), guest.heap.end(), std::array{}); + const auto image = TableT(texture.bytes.data(), TableFormat8888UNorm, TableType2D); + auto* words = guest.heap.front().data(); + std::copy(image.begin(), image.end(), words + 0x50u / 4u); + FillTableSrt(guest, guest.heap.data(), 0u, static_cast(sizeof(guest.heap))); + const auto userData = TableUserData(guest); + const std::vector code{0xf4080100u, 0xfa000000u, 0xf4080700u, 0xfa000030u, 0xf4080800u, 0xfa000010u, 0xf4000440u, 0xfa000040u, 0x8711ff11u, + 0x000000ffu, 0x8f128511u, 0x8113ff12u, 0xffffff10u, 0x8114ff12u, 0xffffff20u, 0xf4280902u, 0x26000040u, 0xf4280a02u, 0x28000040u, 0xf09c0f08u, + 0x01090000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; + const auto request = TableRequest(code, userData); + const auto plan = GetResourcePlan(request); + require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "selected record offset carry: a T# read by two loads of its record was not planned as a table"); + const auto& column = *plan->descriptorSources.at(plan->info.images[0].source).tableColumn; + require(column.selected && !column.address && column.stride == 32u && column.addend == 0xffffff10u && column.offset == 0x40u && column.foldedAddend == 0xffffff10u && column.foldedCarry == 16u, "selected record offset carry: the column is wrong"); + guest.srt[16] = 7u; + const auto carried = CompileTable(request); + const auto& carriedTable = carried.capture->snapshot.tables.tables.at(0); + require(carriedTable.keys == 1u && carriedTable.codes.at(0) == Abi::PoisonCode(static_cast(Abi::PoisonReason::OffsetCarry)) && !TableTouched(carried.regions, guest.heap.data(), sizeof(guest.heap)), "selected record offset carry: the record whose second load wraps past 32 bits is not OffsetCarry poison or was read"); + guest.srt[16] = 8u; + const auto wrapped = CompileTable(request); + require(TableSelected(wrapped.capture->snapshot.tables, 0u, "selected record offset carry: the record past the wrap does not hold one readable selected record") == image && TableCovered(wrapped.regions, words + 0x50u / 4u, sizeof(image)), "selected record offset carry: the record past the wrap was not selected"); +} + void verifyDescriptorPhis() { using namespace ShaderRecompiler; constexpr std::uint32_t Format8888UNorm = 56; @@ -5071,6 +5196,8 @@ int main(int argc, char** argv) { verifyScalarAddressRule(); verifyPointerImageTable(); verifyPointerTableOffsetCarry(); + verifySelectedRecordFallbacks(); + verifySelectedRecordOffsetCarry(); std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp index 43f1799617..4c0e9d8d54 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp @@ -110,6 +110,20 @@ alignas(256) constexpr std::array TableWave32Code{ 0x01090401u, 0x7e120202u, 0x34101287u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, }; +alignas(256) constexpr std::array SelectedTableCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4000400u, 0xfa000060u, + 0xf4200444u, 0x20000000u, 0x9312b011u, 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x7e0402ffu, 0x3e000000u, 0xf09c8f08u, 0x01090401u, 0x7e120202u, 0x34101286u, + 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array SelectedTableCutCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4000400u, 0xfa000060u, + 0xf4200444u, 0x20000000u, 0x9312b011u, 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x7e0402ffu, 0x3e000000u, 0x7da400ffu, 0x00001000u, 0xf09c8f08u, 0x01090401u, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + alignas(256) constexpr std::array DirectCode{ 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, 0xf4080900u, 0xfa000040u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x7e0402ffu, 0x3e000000u, 0xf09c8f08u, 0x01090401u, 0x7e120202u, @@ -934,6 +948,43 @@ void RunGuardedTableTests(AgcDriver::VulkanDevice& device) { SetClassRecords(); } +void RunSelectedRecordTests(AgcDriver::VulkanDevice& device) { + Fill(); + SetClassRecords(); + Keys = {}; + Srt[24] = 12u; + Outcome first; + for (const auto record : {2u, 0u, 1u}) { + Keys[3] = record; + const auto selected = Run(device, SelectedTableCode, 1, Lanes, 64); + Require(!Faulted(selected) && selected.imageTableCount == 1u, "image table selected record: record " + std::to_string(record) + " faulted or bound other entries:\n" + selected.log); + const auto* words = Table.data() + record * (Stride / 4u); + const auto control = RunDirect(device, {words[0], words[1], words[2], words[3]}, {words[4], words[5], words[6], words[7]}); + Require(Group(selected, 0) == Group(control, 0), "image table selected record: record " + std::to_string(record) + " differs from the direct binding of the same words"); + if (record == 2u) first = selected; + Require(selected.artifactId == first.artifactId && selected.variantId == first.variantId, "image table selected record: another key changed the module"); + } + + Keys[3] = 1000u; + const auto beyond = Run(device, SelectedTableCode, 1, Lanes, 64); + Require(!Faulted(beyond) && Zeros(beyond), "image table selected record: a key past the table faulted or did not sample zeros:\n" + beyond.log); + + Keys[3] = 7u; + const auto cut = Run(device, SelectedTableCutCode, 1, Lanes, 64); + Require(!Faulted(cut), "image table selected record: an invalid record selected with EXEC clear faulted:\n" + cut.log); + const auto invalid = Run(device, SelectedTableCode, 1, Lanes, 64); + const auto tableAddress = static_cast(reinterpret_cast(Table.data())) + 7u * Stride + 16u; + char address[32]; + std::snprintf(address, sizeof(address), "0x%llx", static_cast(tableAddress)); + Require(Faulted(invalid) && invalid.log.find("not an image") != std::string::npos && invalid.log.find(address) != std::string::npos && invalid.log.find("00001234") != std::string::npos && Zeros(invalid), "image table selected record: an invalid record selected with EXEC set did not report its address, words and reason:\n" + invalid.log); + + Keys[3] = 0u; + const auto aliased = Run(device, SelectedTableCode, 1, Lanes, 64, true); + Require(Faulted(aliased) && aliased.log.find("is written by its own dispatch") != std::string::npos, "image table selected record: a dispatch storing over its selected record did not fault on use:\n" + aliased.log); + SetClassRecords(); + Srt[24] = 0u; +} + template void RequireModeEntry(AgcDriver::VulkanDevice& device, const std::array& tableCode, const std::array& directCode, std::uint32_t record, std::span image, const std::string& what) { Keys = {}; @@ -1325,6 +1376,7 @@ int main() { RunReadTests(*device); RunLoopKeyTests(*device); RunGuardedTableTests(*device); + RunSelectedRecordTests(*device); RunModeTests(*device); RunStoreTests(*device); RunPointerTests(*device); From f9f6872b8d293ee0353e2f2dd95c49840da209ad Mon Sep 17 00:00:00 2001 From: ArjenS-LA Date: Wed, 14 Oct 2026 23:38:52 -0700 Subject: [PATCH 007/103] fix(tools): audit ordinary homebrew import names --- docs/user/USAGE.md | 2 ++ tools/import_audit.py | 6 ++---- tools/tests/test_import_audit.py | 31 ++++++++++++++++++++++++++++--- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index b448949fe2..c38899b148 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -237,6 +237,8 @@ The report also lists needed libraries that have no file in `--libs` or `--modul | `--names ` | NID and name per line, as in `aerolib.csv` from [`tools/nid_names.py`](../../tools/nid_names.py); names the absent imports. Never downloaded; a name that does not hash to its NID is marked. | | `--json ` | Full result, stamped with the repository's commit and whether its tree was modified. | +Registry `nid` fields may contain either encoded NIDs or ordinary ELF symbol names such as `strcmp`. Ordinary names are matched against literal exports, without hashing them into NIDs. + Exit codes: `0` nothing blocks loading, `1` there are `absent` imports or needed libraries without a file, `2` an input could not be read; the message names the file and the value. A module registry lists `nid`, `library` and `targetOffset` for each relocation against a function that no other bundled module exports, without the `#` suffix of the executable's NIDs. diff --git a/tools/import_audit.py b/tools/import_audit.py index bc819fc9bf..4059b91a3a 100644 --- a/tools/import_audit.py +++ b/tools/import_audit.py @@ -1,7 +1,6 @@ import argparse import hashlib import json -import re import struct import subprocess import sys @@ -14,7 +13,6 @@ import progress ROOT = Path(__file__).resolve().parent.parent PRX = ROOT / "core" / "libs" / "prx" -NID = re.compile(r"[A-Za-z0-9+\-]{11}") NID_POSTFIX = "_nid_postfix" MODULE_SUFFIX = "#" CLASSES = ("implemented", "stub", "absent", "module") @@ -409,8 +407,8 @@ def read_registry(path): if not isinstance(entry, dict) or not isinstance(entry.get("nid"), str) or not isinstance(entry.get("library"), str): raise AuditError(f"{path}: entry {index} needs string fields 'nid' and 'library': {entry!r}") nid = entry["nid"].split("#", 1)[0] - if not NID.fullmatch(nid): - raise AuditError(f"{path}: entry {index} has an invalid NID {entry['nid']!r}") + if not nid or any(character.isspace() or ord(character) < 32 or ord(character) == 127 for character in nid): + raise AuditError(f"{path}: entry {index} has an invalid import symbol {entry['nid']!r}") imports.append((nid, entry["library"], bool(entry.get("weak", False)))) return imports diff --git a/tools/tests/test_import_audit.py b/tools/tests/test_import_audit.py index 2b6bf042f6..993cf3a107 100644 --- a/tools/tests/test_import_audit.py +++ b/tools/tests/test_import_audit.py @@ -178,6 +178,31 @@ class ImportAuditTests(unittest.TestCase): self.work = Workspace() self.addCleanup(self.work.close) + def test_named_imports_use_literal_exports_and_keep_unresolved_names(self): + work = self.work + (work.libs / "libc.prx").write_bytes(exporting("strcmp", "_Z3foov", "AAAAAAAAAAA")) + registry, report = work.root / "homebrew.registry.json", work.root / "audit.json" + write_registry(registry, [("strcmp", ""), ("strcmp", ""), ("_Z3foov", "libc.prx"), + ("AAAAAAAAAAA#A#B", "libc.prx"), ("missing_function", "", True)]) + code, _, err = work.run(registry, "--json", str(report)) + self.assertEqual((code, err), (1, "")) + data = json.loads(report.read_text(encoding="utf-8")) + records = {record["nid"]: record for record in data["imports"]} + self.assertEqual(data["references"], 5) + self.assertEqual(data["unique_by_class"], {"implemented": 3, "stub": 0, "absent": 1, "module": 0}) + self.assertEqual(records["strcmp"]["references"], 2) + self.assertEqual(records["strcmp"]["providers"], ["libc.prx"]) + self.assertTrue(records["missing_function"]["weak"]) + self.assertIsNone(data["stops"][0]["nid"]) + + def test_empty_and_control_character_imports_remain_invalid(self): + registry = self.work.root / "homebrew.registry.json" + for name in ("", "#A#B", "bad\x00name", "bad\nname", "bad\x7fname", "bad name"): + with self.subTest(name=name): + write_registry(registry, [(name, "")]) + with self.assertRaisesRegex(import_audit.AuditError, "invalid import symbol"): + import_audit.read_registry(registry) + def test_failed_json_write_preserves_existing_report_and_leaves_no_partial_report(self): work = self.work (work.libs / "libSceA.prx").write_bytes(exporting("AAAAAAAAAAA")) @@ -1192,7 +1217,7 @@ class ImportAuditTests(unittest.TestCase): with self.assertRaisesRegex(import_audit.AuditError, r"entry 0 needs string fields"): import_audit.read_registry(path) write_registry(path, [("AAAAAAAAAAA", ""), ("bad\x01nid", "")]) - with self.assertRaisesRegex(import_audit.AuditError, r"entry 1 has an invalid NID 'bad\\x01nid'"): + with self.assertRaisesRegex(import_audit.AuditError, r"entry 1 has an invalid import symbol 'bad\\x01nid'"): import_audit.read_registry(path) write_registry(path, [("AAAAAAAAAAA#A#B", "libSceA.prx")], encoding="utf-8-sig") self.assertEqual(import_audit.read_registry(path), [("AAAAAAAAAAA", "libSceA.prx", False)]) @@ -1226,10 +1251,10 @@ class ImportAuditTests(unittest.TestCase): work = self.work (work.libs / "libSceA.prx").write_bytes(exporting("AAAAAAAAAAA")) registry = work.root / "game.json" - write_registry(registry, [("short", "libSceA.prx")]) + write_registry(registry, [("bad name", "libSceA.prx")]) code, out, err = work.run(registry) self.assertEqual((code, out), (2, "")) - self.assertIn("game.json: entry 0 has an invalid NID 'short'", err) + self.assertIn("game.json: entry 0 has an invalid import symbol 'bad name'", err) def test_unreadable_library_and_unwritable_json_exit_two(self): work = self.work From fe56f6056184441a7382f18d53d16ddfc933cb1a Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:50:15 +0000 Subject: [PATCH 008/103] fix(shader): bind a T# whose record the CPU evaluates directly when a table cannot express its operation An image table cannot store, update atomically or read PCK and BY2/BY4 through its records (#757): those operations fault and read zeros. #6243 turns a heap T# whose words and record offset the SRT walker evaluates into a selected one-record table, which kept that limitation although such a T# is one known descriptor, as #5188's direct images were. PlanTableColumns now binds such a T# directly, for every operation of the shader on it, when any of them is a store, an atomic, a packed or a BY2/BY4 access. A T# keyed per lane, or by a value the walker cannot evaluate, keeps its table and its unsupported-operation fault. The disk format moves to 37 so that plans cached by #6243's builds are not reused. --- .../Optimization/src/ResourceTracker.cpp | 54 ++++++++++++++++++- core/shader/recompiler/ShaderDiskCache.hpp | 2 +- docs/dev/debt/shader.md | 2 +- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 580eac4d16..a83d9ee2f7 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -1289,6 +1289,7 @@ private: return; } IndexMemoryUsers(); + const auto directImages = DirectImageHandles(); for (auto& block : m_program.Blocks()) { for (IrValue* inst : block->Instructions()) { const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode()); @@ -1300,7 +1301,10 @@ private: continue; } const auto memory = m_program.Resources().memoryInfo[flags.index]; - static_cast(PlanTableColumn(*inst->Argument(0)->Resolve(), false, memory.imageR128)); + IrValue& image = *inst->Argument(0)->Resolve(); + if (std::ranges::none_of(directImages, [&](const IrValue* handle) { return SameHandle(*handle, image); })) { + static_cast(PlanTableColumn(image, false, memory.imageR128)); + } if (imageInfo.needsSampler && inst->ArgumentCount() > 1u && !DirectSampler(*inst->Argument(1)->Resolve())) { static_cast(PlanTableColumn(*inst->Argument(1)->Resolve(), true, false)); } @@ -1344,6 +1348,54 @@ private: return ValidateSource(descriptor, badDword); } + bool DirectImage(const IrValue& handle) { + if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u) { + return false; + } + DescriptorSource descriptor; + MakeSource(handle, 8u, false, false, descriptor); + std::uint32_t badDword = 0; + return ValidateSource(descriptor, badDword); + } + + std::vector DirectImageHandles() { + std::vector handles; + for (auto& block : m_program.Blocks()) { + for (IrValue* inst : block->Instructions()) { + const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode()); + if (imageInfo.access == ImageAccess::None || inst->ArgumentCount() == 0u) { + continue; + } + const auto flags = inst->Flags(); + if (flags.index >= m_program.Resources().memoryInfo.size()) { + continue; + } + const auto& memory = m_program.Resources().memoryInfo[flags.index]; + const bool tableUnsupported = imageInfo.resourceClass == ImageResourceClass::Storage || memory.imagePacked || memory.imageByElements != 0u; + const IrValue* handle = inst->Argument(0)->Resolve(); + if (tableUnsupported && DirectImage(*handle)) { + handles.push_back(handle); + } + } + } + return handles; + } + + bool SameHandle(const IrValue& left, const IrValue& right) const { + if (&left == &right) { + return true; + } + if (left.Opcode() != right.Opcode() || left.ArgumentCount() != right.ArgumentCount()) { + return false; + } + for (std::size_t index = 0; index < left.ArgumentCount(); index++) { + if (!EquivalentValue(m_program.Resources(), left.Argument(index), right.Argument(index))) { + return false; + } + } + return true; + } + bool IsTablePlanningMemory(std::uint32_t index) const { return std::ranges::find(m_tablePlanningMemory, index) != m_tablePlanningMemory.end(); } diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index b03d85a809..1cce383ff3 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 36; +inline constexpr std::uint32_t FormatVersion = 37; enum class LoadStatus { Loaded, diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 61ceaa79aa..b69e3157ae 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -34,7 +34,7 @@ - [Image tables](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read by `s_load` through a 64-bit pointer hold the keys the key's known bits allow, at most 65536; a key past them, or an entry at a wrapping, unaligned or unmapped address, faults when selected. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) whose key is neither a V#-relative scalar read nor a value the SRT walker evaluates (for example a shifted or masked runtime-offset V# read, a per-lane value made uniform by `v_readfirstlane`, or a loop-carried phi) binds every record of the snapshot domain instead of the selected ones. So does a table read at `(key << n) & mask` with such a key: a key domain would select records by the unmasked key, so that key is not narrowed. A key the walker evaluates selects its one record instead, unless the shader may write the memory the key is read from at run time (the same buffer, or any memory when the read or a write goes through a pointer or a GPU-only descriptor); then the column binds the whole domain too. A write to the key's memory through another V# over the same bytes is not detected. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) T# of an MSAA or FMASK image, or of a color format under a depth compare, faults when selected: tables have no multisampled descriptors and no emulated color compare. -- `image_store`, image atomics, PCK2/PCK4 and MIMG BY2/BY4 through an [image table](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) T# fault (`unsupported operation`) when an active lane executes them; the store is dropped and the load reads zeros. +- `image_store`, image atomics, PCK2/PCK4 and MIMG BY2/BY4 through an [image table](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) T# fault (`unsupported operation`) when an active lane executes them; the store is dropped and the load reads zeros. A T# whose words the SRT walker evaluates is bound directly instead, for every operation of the shader on it (`DirectImageHandles`), so only T#s keyed per lane or by a value the walker cannot evaluate reach a table with these operations; such a direct T# is read when the work is recorded, so a key its own dispatch writes is read before that write. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) S# with a min or max reduction or FORCE_UNNORMALIZED faults when selected, as does an integer, converted or depth-bits table T# paired with a direct min or max S# that filters between texels. - An image-table T#/S# pair with anisotropic `ANISO_OVERRIDE` faults on selection: dynamic mip-level pairing is unimplemented, including otherwise-valid single-level and mipmapped tables. Direct pairs keep their override behavior. Unselected entries, anisotropy without the override and the override without anisotropy stay supported. - Uint and Sint [image table](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp) entries and direct T#s under a table S# sample through a point-filtered copy of the S#: Vulkan leaves linear filtering of integer formats undefined; the console is assumed to point-sample. From e87f3ae920a9ee5d934a91eac61becfd5a6f678d Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:50:15 +0000 Subject: [PATCH 009/103] test(shader): store, update atomically and read packed through a T# whose record the CPU evaluates verifySelectedRecordDirectOperations plans image_store, image_atomic_add, image_load_pck and image_load_by2 through a heap T# keyed by an SRT value, which must be bound directly, and through a T# keyed per lane, which must stay a table with its unsupported mark. A T# that is both sampled and stored is bound directly for both, and one that is only sampled still selects its record. RunSelectedStoreTests stores through a record keyed by an SRT value on the GPU and reads the texels back. Before the fix both failed: the T# was planned through an image table, and the store faulted as an unsupported operation. --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 48 +++++++++++++++++++ .../tests/execution/ImageTable.cpp | 33 +++++++++++++ 2 files changed, 81 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 3980d639f4..66e1365008 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -975,6 +975,7 @@ struct TableGuest { std::array srt{}; }; +constexpr std::uint32_t TableFormat16UNorm = 7; constexpr std::uint32_t TableFormat8888UNorm = 56; constexpr std::uint32_t TableFormat32UInt = 20; constexpr std::uint32_t TableFormat16SInt = 12; @@ -1906,6 +1907,52 @@ void verifySelectedRecordOffsetCarry() { require(TableSelected(wrapped.capture->snapshot.tables, 0u, "selected record offset carry: the record past the wrap does not hold one readable selected record") == image && TableCovered(wrapped.regions, words + 0x50u / 4u, sizeof(image)), "selected record offset carry: the record past the wrap was not selected"); } +std::vector WithImageOperation(std::vector code, std::uint32_t word0, bool keepSample) { + const auto sample = std::find(code.begin(), code.end(), 0xf09c0f08u); + require(sample != code.end() && sample + 1 != code.end(), "image operation variant: the program has no image_sample_lz"); + if (keepSample) { + code.insert(sample + 2, {word0, 0x00050000u}); + } else { + sample[0] = word0; + sample[1] = 0x00050000u; + } + return code; +} + +void verifySelectedRecordDirectOperations() { + using namespace ShaderRecompiler; + static TableTexture textures[2]; + alignas(4096) static TableGuest guest; + auto& heap = guest.heap; + std::fill(heap.begin(), heap.end(), std::array{}); + heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D); + heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D); + heap[2] = heap[1]; + heap[3] = heap[0]; + FillTableSrt(guest, heap.data(), 32u, 4u); + guest.srt[16] = 2u; + const auto userData = TableUserData(guest); + const std::array, 4> operations{{{"image_store", 0xf0200f08u, TableFormat32UInt}, {"image_atomic_add", 0xf0440108u, TableFormat32UInt}, {"image_load_pck", 0xf0080f08u, TableFormat32UInt}, {"image_load_by2", 0xf1080308u, TableFormat16UNorm}}}; + for (const auto& [name, word0, format] : operations) { + const auto what = std::string("selected record direct operations: ") + name; + heap[2] = TableT(textures[1].bytes.data(), format, TableType2D); + const auto code = WithImageOperation(TableSrtKeyCode, word0, false); + const auto request = TableRequest(code, userData); + const auto plan = GetResourcePlan(request); + require(plan->info.images.size() == 1u && plan->info.images[0].table == NoTable && !plan->descriptorSources.at(plan->info.images[0].source).tableColumn.has_value(), (what + " through a T# whose record the CPU evaluates was planned through an image table").c_str()); + const auto compiled = CompileTable(request); + require(TableBinding(*compiled.result, DescriptorRole::ImageTable) == nullptr && compiled.result->imageTableFaults == 0u && TableCovered(compiled.regions, &heap[2], sizeof(heap[2])), (what + " through a T# whose record the CPU evaluates was not bound from its record").c_str()); + const auto wholeCode = WithImageOperation(TableWholeCode, word0, false); + const auto whole = GetResourcePlan(TableRequest(wholeCode, userData)); + require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == TableOperation::Unsupported, (what + " through a T# keyed per lane left the image table or lost its unsupported mark").c_str()); + } + const auto mixedCode = WithImageOperation(TableSrtKeyCode, 0xf0200f08u, true); + const auto mixed = GetResourcePlan(TableRequest(mixedCode, userData)); + require(mixed->info.images.size() == 2u && std::ranges::all_of(mixed->info.images, [&](const ImageResource& image) { return image.table == NoTable && !mixed->descriptorSources.at(image.source).tableColumn.has_value(); }), "selected record direct operations: a T# sampled and stored was planned through an image table for one of its operations"); + const auto sampled = GetResourcePlan(TableRequest(TableSrtKeyCode, userData)); + require(sampled->info.images.size() == 1u && sampled->info.images[0].table == 0u && sampled->descriptorSources.at(sampled->info.images[0].source).tableColumn->selected, "selected record direct operations: a T# that is only sampled no longer selects its record"); +} + void verifyDescriptorPhis() { using namespace ShaderRecompiler; constexpr std::uint32_t Format8888UNorm = 56; @@ -5198,6 +5245,7 @@ int main(int argc, char** argv) { verifyPointerTableOffsetCarry(); verifySelectedRecordFallbacks(); verifySelectedRecordOffsetCarry(); + verifySelectedRecordDirectOperations(); std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp index 4c0e9d8d54..a6038714bb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp @@ -204,6 +204,14 @@ alignas(256) constexpr std::array StoreTableCutCode{ 0xbf810000u, }; +constexpr std::uint32_t SelectedStoreValue = 0xc0de5eedu; + +alignas(256) constexpr std::array SelectedStoreCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4000400u, 0xfa000060u, 0xf4200444u, 0x20000000u, + 0x9312b011u, 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e0802ffu, SelectedStoreValue, 0xf0208f08u, + 0x00090401u, 0xbf810000u, +}; + alignas(256) constexpr std::array PointerCode{ 0xf4080200u, 0xfa000000u, 0xf4080700u, 0xfa000010u, 0xf4080800u, 0xfa000020u, 0x8f108202u, 0xf4200444u, 0x20000000u, 0x8711ff11u, 0x000000ffu, 0x8f128511u, 0xf40c0900u, 0x24000040u, 0x360200bfu, 0x7e020d01u, @@ -1050,6 +1058,30 @@ void RunStoreTests(AgcDriver::VulkanDevice& device) { Require(UintTexels == saved, "image table store: an unsupported store reached the image"); } +void RunSelectedStoreTests(AgcDriver::VulkanDevice& device) { + alignas(256) static std::array storeTexels{}; + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + } registered(storeTexels.data(), sizeof(storeTexels)); + for (std::uint32_t texel = 0; texel < storeTexels.size(); ++texel) storeTexels[texel] = 0x5000u + texel; + const auto saved = storeTexels; + SetClassRecords(); + SetRecord(9, PointClamp, Image(storeTexels.data(), Format32UInt)); + Keys = {}; + Keys[3] = 9u; + Srt[24] = 12u; + const auto stored = Run(device, SelectedStoreCode, 1, Lanes, 64); + AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast(storeTexels.data()), sizeof(storeTexels), nullptr, "test"); + device.WaitIdle(); + Require(!Faulted(stored) && stored.imageTableCount == 0u, "image table selected store: an image_store through a T# whose record the CPU evaluates faulted or was bound through an image table:\n" + stored.log); + for (std::uint32_t texel = 0; texel < storeTexels.size(); ++texel) { + const auto expected = texel < Lanes ? SelectedStoreValue : saved[texel]; + Require(storeTexels[texel] == expected, "image table selected store: texel " + std::to_string(texel) + " holds " + std::to_string(storeTexels[texel]) + " after an image_store through a T# whose record the CPU evaluates, expected " + std::to_string(expected)); + } +} + void RunPointerTests(AgcDriver::VulkanDevice& device) { const auto floatImage = Image(FloatTexels.data(), Format32Float); const auto uintImage = Image(UintTexels.data(), Format32UInt); @@ -1379,6 +1411,7 @@ int main() { RunSelectedRecordTests(*device); RunModeTests(*device); RunStoreTests(*device); + RunSelectedStoreTests(*device); RunPointerTests(*device); RunPointerRecordTests(*device); RunSplitRecordTests(*device); From 3b914c84b40d4322b1c15ddb89246b5c6baec4c3 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:50:15 +0000 Subject: [PATCH 010/103] test(shader): pin that an S# whose record the CPU evaluates is bound directly A sampler column is planned only when DirectSampler cannot evaluate the S#, so no sampler record is ever selected: an S# read from a heap at an offset the SRT walker evaluates is a direct sampler, and one keyed per lane is a sampler table. --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 66e1365008..5322874596 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1953,6 +1953,36 @@ void verifySelectedRecordDirectOperations() { require(sampled->info.images.size() == 1u && sampled->info.images[0].table == 0u && sampled->descriptorSources.at(sampled->info.images[0].source).tableColumn->selected, "selected record direct operations: a T# that is only sampled no longer selects its record"); } +void verifyEvaluableHeapSamplerDirect() { + using namespace ShaderRecompiler; + static TableTexture textures[2]; + alignas(4096) static TableGuest guest; + auto& heap = guest.heap; + std::fill(heap.begin(), heap.end(), std::array{}); + heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D); + heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D); + heap[2] = heap[1]; + heap[3] = heap[0]; + FillTableSrt(guest, heap.data(), 32u, 4u); + guest.srt[16] = 2u; + const auto userData = TableUserData(guest); + const auto heapSampler = [](std::vector code) { + const auto direct = std::find(code.begin(), code.end(), 0xf4080200u); + require(direct != code.end() && direct[1] == 0xfa000010u, "evaluable heap sampler: the program has no S# load from the SRT"); + code.erase(direct, direct + 2); + const auto shift = std::find(code.begin(), code.end(), 0x8f108510u); + require(shift != code.end(), "evaluable heap sampler: the program has no key shift"); + code.insert(shift + 1, {0xf4280202u, 0x20000000u}); + return code; + }; + const auto evaluatedCode = heapSampler(TableSrtKeyCode); + const auto evaluated = GetResourcePlan(TableRequest(evaluatedCode, userData)); + require(evaluated->info.samplers.size() == 1u && evaluated->info.samplers[0].table == NoTable && !evaluated->descriptorSources.at(evaluated->info.samplers[0].source).tableColumn.has_value(), "evaluable heap sampler: an S# read from a heap at an offset the SRT walker evaluates was planned through a sampler table"); + const auto perLaneCode = heapSampler(TableWholeCode); + const auto perLane = GetResourcePlan(TableRequest(perLaneCode, userData)); + require(perLane->info.samplers.size() == 1u && perLane->info.samplers[0].table != NoTable && perLane->descriptorSources.at(perLane->info.samplers[0].source).tableColumn.has_value(), "evaluable heap sampler: an S# read from a heap at an offset keyed per lane was not planned through a sampler table"); +} + void verifyDescriptorPhis() { using namespace ShaderRecompiler; constexpr std::uint32_t Format8888UNorm = 56; @@ -5246,6 +5276,7 @@ int main(int argc, char** argv) { verifySelectedRecordFallbacks(); verifySelectedRecordOffsetCarry(); verifySelectedRecordDirectOperations(); + verifyEvaluableHeapSamplerDirect(); std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } From 033406265762d56024ebc3d9c45a6b966e956697 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:50:15 +0000 Subject: [PATCH 011/103] perf(shader): drop NonUniform from the index of a selected image table record A selected record is one map entry for the whole dispatch, so its descriptor index is dynamically uniform. The emitter decorated it NonUniform like any table index whenever the workgroup spans more than one subgroup, and on every graphics stage, which makes drivers scalarize the access. The element index, its access chain, the loaded image and the sampled image of a selected record are no longer decorated; table samplers and tables keyed per lane keep the decoration. --- .../SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp | 1 + .../recompiler/SpirvBackend/src/SpirvImageEmitter.cpp | 4 +++- .../SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index b85eb66e9f..dfc86a7f4b 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -83,6 +83,7 @@ struct SpirvTableAccess { std::uint32_t dword3 = 0; std::uint32_t samplerElement = 0; bool active = false; + bool uniform = false; }; struct SpirvEmitterState { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 0328487362..b54e5d2469 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -1703,7 +1703,9 @@ TableAccessResult EmitTableAccess(SpirvValueEmitContext& ctx, const IrValue& ins result.flags = flags; live = selected; state.tableAccess.element = Binary(state, spv::OpIAdd, u32, LoadTableMap(state, ConstantU32(state, ImageTableAbi::HeaderElementBase)), element); - DecorateTableNonUniform(state, state.tableAccess.element); + const auto& sources = state.program.Resources().descriptorSources; + state.tableAccess.uniform = base.source < sources.size() && sources[base.source].tableColumn.has_value() && sources[base.source].tableColumn->selected; + if (!state.tableAccess.uniform) DecorateTableNonUniform(state, state.tableAccess.element); state.tableAccess.dword3 = dword3; } if (tableSampler) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp index 10a18de9ff..51c6548c8e 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp @@ -156,10 +156,10 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t if (found == state.tableImageVariables.end() || !state.tableAccess.active) ExitDescriptorBindingFailure(state, kind, resource, "image table array was not emitted for the mode"); const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, found->variable, state.tableAccess.element); - DecorateTableNonUniform(state, pointer); + if (!state.tableAccess.uniform) DecorateTableNonUniform(state, pointer); const auto image = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, imageType, image, pointer); - DecorateTableNonUniform(state, image); + if (!state.tableAccess.uniform) DecorateTableNonUniform(state, image); return image; } const auto arrayIndex = ResourceForDescriptor(state, kind, resource); @@ -206,7 +206,7 @@ std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, const auto sampledImage = state.module.AllocateId(); const auto sampledType = state.module.Type(spv::OpTypeSampledImage, ImageType(state, imageResource)); state.module.AddFunction(spv::OpSampledImage, sampledType, sampledImage, image, samplerId); - if (imageResource.table != NoTable || state.program.Info().samplers.at(sampler).table != NoTable) DecorateTableNonUniform(state, sampledImage); + if ((imageResource.table != NoTable && !state.tableAccess.uniform) || state.program.Info().samplers.at(sampler).table != NoTable) DecorateTableNonUniform(state, sampledImage); return sampledImage; } From 55a09385ab16d42f8a29ae0dae6627b328507a0b Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:50:15 +0000 Subject: [PATCH 012/103] test(shader): check that a selected record's table index is not NonUniform verifySelectedRecordUniformIndex compiles the program keyed by an SRT value and the one keyed per lane with a two-wave workgroup. The selected record's table index carries no NonUniform decoration and the per-lane table's still does. Before the change the selected record's was decorated. --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 5322874596..2625e6699d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1983,6 +1983,38 @@ void verifyEvaluableHeapSamplerDirect() { require(perLane->info.samplers.size() == 1u && perLane->info.samplers[0].table != NoTable && perLane->descriptorSources.at(perLane->info.samplers[0].source).tableColumn.has_value(), "evaluable heap sampler: an S# read from a heap at an offset keyed per lane was not planned through a sampler table"); } +void verifySelectedRecordUniformIndex() { + using namespace ShaderRecompiler; + static TableTexture textures[2]; + alignas(4096) static TableGuest guest; + auto& heap = guest.heap; + std::fill(heap.begin(), heap.end(), std::array{}); + heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D); + heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D); + heap[2] = heap[1]; + heap[3] = heap[0]; + FillTableSrt(guest, heap.data(), 32u, 4u); + guest.srt[16] = 2u; + const auto userData = TableUserData(guest); + const auto twoWaves = [&](const std::vector& code) { + auto request = TableRequest(code, userData); + request.context.compute->numThreads = {128u, 1u, 1u}; + return CompileTable(request); + }; + const auto nonUniform = [](const TableCompiled& compiled) { + const auto spirv = compiled.result->spirv.Words(); + std::uint32_t count = 0; + for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { + if ((spirv[cursor] & 0xffffu) == spv::OpDecorate && (spirv[cursor] >> 16u) == 3u && spirv[cursor + 2u] == spv::DecorationNonUniform) ++count; + } + return count; + }; + const auto selected = twoWaves(TableSrtKeyCode); + require(TableBinding(*selected.result, DescriptorRole::ImageTable) != nullptr && nonUniform(selected) == 0u, "selected record uniform index: the image table index of a selected record in a two-wave workgroup is decorated NonUniform"); + const auto whole = twoWaves(TableWholeCode); + require(nonUniform(whole) != 0u, "selected record uniform index: the image table index of a table keyed per lane in a two-wave workgroup is not decorated NonUniform"); +} + void verifyDescriptorPhis() { using namespace ShaderRecompiler; constexpr std::uint32_t Format8888UNorm = 56; @@ -5277,6 +5309,7 @@ int main(int argc, char** argv) { verifySelectedRecordOffsetCarry(); verifySelectedRecordDirectOperations(); verifyEvaluableHeapSamplerDirect(); + verifySelectedRecordUniformIndex(); std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } From 6aacf0bab5434289d60a072f549cdd071444f974 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:50:15 +0000 Subject: [PATCH 013/103] fix(shader): clear the key domains of a column deselected because its key may be written CheckTableStores deselects a selected column whose key the dispatch may write: a key read through a pointer or a V# while the shader stores through a pointer, or through a V# the shader writes. The column kept the key domains narrowed from that same key buffer, which the domain rule clears only when the domain's own V# is written, so a shader storing through a pointer still narrowed its whole table on a key it may overwrite. Deselection now clears them as well. --- core/shader/recompiler/Optimization/src/ResourceTracker.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index a83d9ee2f7..ee01ec50ad 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -1455,6 +1455,7 @@ private: for (auto& source : m_sources) { if (source.tableColumn.has_value() && source.tableColumn->selected && SelectionWritten(*source.tableColumn)) { source.tableColumn->selected = false; + source.tableColumn->keyDomains.clear(); } if (!source.tableColumn.has_value() || source.tableColumn->keyDomains.empty()) { continue; From 23d6cf6848d1bc8e57fddc2b5df7d366eddd3471 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:50:15 +0000 Subject: [PATCH 014/103] test(shader): check that a key a pointer store may write keeps no key domain The key loaded from a constant buffer at an SRT offset has a key domain. With a global_store_dword added, the column is deselected and keeps no key domain. Before the fix it kept the domain. --- core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 2625e6699d..4f85e07306 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1866,6 +1866,14 @@ void verifySelectedRecordFallbacks() { require(writtenColumn.has_value() && !writtenColumn->selected && writtenColumn->keyDomains.empty(), "selected record fallbacks: a key loaded from a buffer the shader writes selected its record"); const auto written = CompileTable(writtenRequest); require(written.capture->snapshot.tables.tables.at(0).keys == 4u, "selected record fallbacks: a key loaded from a buffer the shader writes did not keep the whole table"); + require(!loadedPlan->descriptorSources.at(loadedPlan->info.images[0].source).tableColumn->keyDomains.empty(), "selected record fallbacks: a key loaded from a constant buffer has no key domain"); + auto pointerWritten = loadedKey; + const auto end = std::find(pointerWritten.begin(), pointerWritten.end(), 0xbf810000u); + require(end != pointerWritten.end(), "selected record fallbacks: the program has no s_endpgm"); + pointerWritten.insert(end, {0xdc708000u, 0x001c0100u}); + const auto pointerPlan = GetResourcePlan(TableRequest(pointerWritten, userData)); + const auto& pointerColumn = pointerPlan->descriptorSources.at(pointerPlan->info.images.at(0).source).tableColumn; + require(pointerPlan->info.bdaWrites && pointerColumn.has_value() && !pointerColumn->selected && pointerColumn->keyDomains.empty(), "selected record fallbacks: a key loaded from a constant buffer in a shader that stores through a pointer selected its record or kept its key domain"); const auto lanePlan = GetResourcePlan(TableRequest(TableWholeCode, userData)); require(!lanePlan->info.images.empty() && !lanePlan->descriptorSources.at(lanePlan->info.images[0].source).tableColumn->selected, "selected record fallbacks: a T# selected per lane left the table"); From e25bb140f9ac2fa674e9f2d367b6adccfe823b3e Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:09:22 -0400 Subject: [PATCH 015/103] fix(agc): preserve null images during stencil feedback checks Co-Authored-By: Codex --- .../prx/libSceAgcDriver/Graphics/src/Draw.cpp | 1 + .../tests/execution/StencilNullFeedback.cpp | 265 ++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 7fb4593c6e..1039fa0538 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -2078,6 +2078,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages && binding.role != ShaderRecompiler::DescriptorRole::ImageTable) continue; for (std::size_t offset = 0; offset + 8u <= binding.guestDescriptor.size(); offset += 8u) { const auto words = std::span(binding.guestDescriptor).subspan(offset, 8u); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; const auto texture = DecodeTextureResource(words); Require(!DepthSurfaceAt(context, words, texture), "stencil replay refuses sampled depth-surface feedback"); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp new file mode 100644 index 0000000000..3c721781dd --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp @@ -0,0 +1,265 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver { + +struct VulkanDeviceTestAccess { + static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); } +}; + +} + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +PFN_vkCmdBindPipeline NativeBind = nullptr; +std::atomic GraphicsBinds{0u}; + +VKAPI_ATTR void VKAPI_CALL CountBind(VkCommandBuffer commands, VkPipelineBindPoint point, VkPipeline pipeline) { + if (point == VK_PIPELINE_BIND_POINT_GRAPHICS) ++GraphicsBinds; + NativeBind(commands, point, pipeline); +} + +struct BindTrace { + AgcDriver::Graphics::DeviceFunctions* functions; + explicit BindTrace(const AgcDriver::VulkanDevice& device) { + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + functions = const_cast(context.functions); + Require(functions != nullptr && functions->cmdBindPipeline != nullptr, "native pipeline bind table is unavailable"); + NativeBind = functions->cmdBindPipeline; + functions->cmdBindPipeline = CountBind; + } + ~BindTrace() { functions->cmdBindPipeline = NativeBind; } +}; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 16; +constexpr std::uint32_t WaveSize = 64; +constexpr std::byte Kept{0x40}; +alignas(256) std::array Pixels{}; +alignas(256) std::array NullBindingTexture{}; +alignas(256) std::array DepthMemory{}; +alignas(256) std::array StencilMemory{}; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +alignas(256) constexpr std::array PlainPixelCode{ + 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +alignas(256) constexpr std::array ColumnPixelCode{ + 0x7e040f00, 0x7e0e02f2, 0xf8000082, 0x00000200, 0xf800180f, 0x07070707, 0xbf810000, +}; + +alignas(256) constexpr std::array RowPixelCode{ + 0x7e040f01, 0x7e0e02f2, 0xf8000082, 0x00000200, 0xf800180f, 0x07070707, 0xbf810000, +}; + +alignas(256) constexpr std::array, 6> Vertices{{ + {-1.0f, -1.0f, 0.25f, 1.0f}, {3.0f, -1.0f, 0.25f, 1.0f}, {-1.0f, 3.0f, 0.25f, 1.0f}, + {-1.0f, -1.0f, 0.75f, 1.0f}, {3.0f, -1.0f, 0.75f, 1.0f}, {-1.0f, 3.0f, 0.75f, 1.0f}, +}}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +ShaderRecompiler::ShaderPixelStageInfo PixelInfo(std::uint32_t shaderControl) { + const AgcDriver::Registers context{ + {0x1b3, 0x700u}, {0x1b4, 0x700u}, {0x1b6, 0u}, {0x1c5, 9u}, {0x203, shaderControl}, + }; + return AgcDriver::Graphics::DecodePixelStageInfo(context, {0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u}); +} + +void AttachNullImages(ShaderRecompiler::RecompileResult& result) { + namespace Runtime = ShaderRecompiler::RuntimeAbi; + const auto address = static_cast(reinterpret_cast(NullBindingTexture.data())); + const std::array words{static_cast(address >> 8u), static_cast(address >> 40u) | (56u << 20u) | (((Width - 1u) & 3u) << 30u), ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), 0xfacu | (9u << 28u), 0u, 0u, 0u, 0u}; + constexpr auto binding = Runtime::FirstStorageImageBinding + 1u; + ShaderRecompiler::DescriptorBinding images{ShaderRecompiler::DescriptorKind::StorageImage, ShaderRecompiler::DescriptorRole::GuestImages, Runtime::FragmentDescriptorSet, binding, 2u, std::vector(16u)}; + std::copy(words.begin(), words.end(), images.guestDescriptor.begin()); + images.imageShape = ShaderRecompiler::DescriptorImageShape::Image2D; + images.imageWritten = {false, false}; + images.imageAtomic = {false, false}; + images.imageAtomic64 = {false, false}; + result.bindings.push_back(std::move(images)); + auto module = result.spirv.Words(); + auto id = module[3]; + std::uint32_t scalar = 0u, floating = 0u; + std::size_t types = 0u, functions = 0u; + for (std::size_t offset = 5u; offset < module.size(); offset += module[offset] >> 16u) { + const auto opcode = module[offset] & 0xffffu; + if (types == 0u && opcode == spv::OpTypeVoid) types = offset; + if (opcode == spv::OpTypeInt && module[offset + 2u] == 32u && module[offset + 3u] == 0u) scalar = module[offset + 1u]; + if (opcode == spv::OpTypeFloat && module[offset + 2u] == 32u) floating = module[offset + 1u]; + if (opcode == spv::OpFunction) { functions = offset; break; } + } + Require(types != 0u && functions != 0u && floating != 0u, "the null-image fixture has no scalar shader types"); + const bool declareScalar = scalar == 0u; + if (declareScalar) scalar = id++; + const auto count = id++, image = id++, array = id++, pointer = id++, variable = id++; + const auto emit = [](std::vector& output, spv::Op opcode, std::initializer_list operands) { + output.push_back((static_cast(operands.size() + 1u) << 16u) | static_cast(opcode)); + output.insert(output.end(), operands.begin(), operands.end()); + }; + std::vector annotations; + emit(annotations, spv::OpDecorate, {variable, spv::DecorationDescriptorSet, Runtime::FragmentDescriptorSet}); + emit(annotations, spv::OpDecorate, {variable, spv::DecorationBinding, binding}); + emit(annotations, spv::OpDecorate, {variable, spv::DecorationNonWritable}); + std::vector declarations; + if (declareScalar) emit(declarations, spv::OpTypeInt, {scalar, 32u, 0u}); + emit(declarations, spv::OpConstant, {scalar, count, 2u}); + emit(declarations, spv::OpTypeImage, {image, floating, spv::Dim2D, 0u, 0u, 0u, 2u, spv::ImageFormatRgba8}); + emit(declarations, spv::OpTypeArray, {array, image, count}); + emit(declarations, spv::OpTypePointer, {pointer, spv::StorageClassUniformConstant, array}); + emit(declarations, spv::OpVariable, {pointer, variable, spv::StorageClassUniformConstant}); + module.insert(module.begin() + functions, declarations.begin(), declarations.end()); + module.insert(module.begin() + types, annotations.begin(), annotations.end()); + module[3] = id; + result.spirv = std::move(module); +} + + +void Draw(AgcDriver::VulkanDevice& device, std::span pixelCode, std::uint32_t shaderControl, const VkStencilOpState& stencil, std::uint32_t vertices = 3u, std::uint32_t instances = 1u, bool depthFail = false, const std::function& alter = {}, const std::function& alterDraw = {}, std::span> vertexData = Vertices, std::span vertexCode = VertexCode, std::span pixelData = {}, std::span pixelSrt = {}, bool nullImages = false) { + Pixels.fill(Kept); + auto target = device.Target(); + std::vector capabilities(target.supportedCapabilities.begin(), target.supportedCapabilities.end()); + std::erase(capabilities, static_cast(spv::CapabilityStencilExportEXT)); + target.supportedCapabilities = capabilities; + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(vertexData.data(), 16u, static_cast(vertexData.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(vertexCode.data()), std::as_bytes(vertexCode)}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(vertexCode.data()), vertexCode, 0, {}}, + {WaveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + auto pixelUserData = pixelData.empty() ? std::vector(8, 0u) : std::vector(pixelData.begin(), pixelData.end()); + std::vector pixelMemory{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}; + pixelMemory.insert(pixelMemory.end(), pixelSrt.begin(), pixelSrt.end()); + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {WaveSize, 0, pixelUserData, std::nullopt, PixelInfo(shaderControl), std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + auto pixelResult = ShaderRecompiler::Recompile(fragment); + if (nullImages) AttachNullImages(pixelResult); + if (!pixelResult.stencilReferences.empty()) { + Require(pixelResult.stencilReferenceDword * sizeof(std::uint32_t) + sizeof(std::uint32_t) == pixelResult.pushConstants.size(), "dynamic reference is not reserved after the stage payload"); + for (std::size_t cursor = 5; cursor < pixelResult.spirv.size();) { + const auto count = pixelResult.spirv[cursor] >> 16u; + Require(count != 0u && count <= pixelResult.spirv.size() - cursor, "invalid materialized replay module"); + Require((pixelResult.spirv[cursor] & 0xffffu) != spv::OpSpecConstant, "expected reference remains a pipeline specialization"); + cursor += count; + } + } + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, WaveSize, WaveSize, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.depth = AgcDriver::Graphics::DepthTarget{reinterpret_cast(DepthMemory.data()), reinterpret_cast(StencilMemory.data()), {Width, Height}, VK_FORMAT_D32_SFLOAT_S8_UINT, 1.0f, 0}; + state.depthTest = depthFail; + state.depthCompare = depthFail ? VK_COMPARE_OP_NEVER : VK_COMPARE_OP_ALWAYS; + state.stencilTest = true; + state.stencilFront = stencil; + state.stencilBack = stencil; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + if (alter) alter(state); + AgcDriver::Pm4::DrawParameters draw{0, vertices, 0, instances, 0, false}; + if (alterDraw) alterDraw(draw); + const auto bindsBefore = GraphicsBinds.load(); + device.Draw(state, draw, shaders); + device.WaitIdle(); + if (!pixelResult.stencilReferences.empty()) Require(GraphicsBinds.load() - bindsBefore == 1u, "stencil replay rebound its shader pipeline within an original draw"); +} + +void Check(const std::function& drawn, std::string_view what) { + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + const auto expected = drawn(x, y) ? std::byte{255} : Kept; + const auto actual = Pixels[(y * Width + x) * 4u]; + Require(actual == expected, std::string(what) + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") is " + std::to_string(static_cast(actual)) + ", expected " + (drawn(x, y) ? "drawn" : "stencil-rejected")); + } + } +} + +} + + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const BindTrace binds(*device); + constexpr std::uint32_t exported = 0x52u; + constexpr std::uint32_t plain = 0x50u; + constexpr std::array alternating{ + 0x7e040f00u, 0x36060481u, + 0x7c000000u | (0xc2u << 17u) | (3u << 9u) | 0x80u, + 0x7e0802ffu, 0xffu, + (1u << 25u) | (5u << 17u) | (4u << 9u) | 0x80u, + 0x7e0e02f2u, 0xf8000082u, 5u << 8u, + 0xf800180fu, 0x07070707u, 0xbf810000u + }; + const auto replace = [](std::uint32_t reference = 0u, std::uint32_t mask = 0xffu) { return VkStencilOpState{VK_STENCIL_OP_KEEP, VK_STENCIL_OP_REPLACE, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_ALWAYS, 0xffu, mask, reference}; }; + const auto equal = [](std::uint32_t reference, std::uint32_t mask = 0xffu) { return VkStencilOpState{VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_EQUAL, mask, 0u, reference}; }; + const auto all = [](std::uint32_t, std::uint32_t) { return true; }; + const auto even = [](std::uint32_t x, std::uint32_t) { return x % 2u == 0u; }; + const auto odd = [](std::uint32_t x, std::uint32_t) { return x % 2u != 0u; }; + Draw(*device, PlainPixelCode, plain, replace(), 3u, 1u, false, {}, {}, Vertices, VertexCode, {}, {}, true); + Check(all, "an empty typed image element remains supported without stencil export"); + Draw(*device, alternating, exported, replace(), 6u, 2u, false, {}, {}, Vertices, VertexCode, {}, {}, true); + Check(all, "an empty typed image element preserves stencil replay colors"); + Draw(*device, PlainPixelCode, plain, equal(255u)); + Check(even, "an empty typed image element preserves the even stencil reference"); + Draw(*device, PlainPixelCode, plain, equal(0u)); + Check(odd, "an empty typed image element preserves the odd stencil reference"); + std::puts("stencil null feedback tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From af8a8b28a0170ba235bf5697f8a1c60804c2c326 Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:28:41 -0400 Subject: [PATCH 016/103] test(agc): copy stencil null fixture words before mutation Co-Authored-By: Codex --- .../prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp index 3c721781dd..7101343cf9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp @@ -101,7 +101,7 @@ void AttachNullImages(ShaderRecompiler::RecompileResult& result) { images.imageAtomic = {false, false}; images.imageAtomic64 = {false, false}; result.bindings.push_back(std::move(images)); - auto module = result.spirv.Words(); + auto module = result.spirv.Copy(); auto id = module[3]; std::uint32_t scalar = 0u, floating = 0u; std::size_t types = 0u, functions = 0u; From da0d5b2be3d5c32147b34f83fc046b7b30ffaa10 Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:31:41 -0400 Subject: [PATCH 017/103] test(agc): retain non-null depth feedback refusal Co-Authored-By: Codex --- .../tests/execution/StencilNullFeedback.cpp | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp index 7101343cf9..befa0444a8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp @@ -89,10 +89,11 @@ ShaderRecompiler::ShaderPixelStageInfo PixelInfo(std::uint32_t shaderControl) { return AgcDriver::Graphics::DecodePixelStageInfo(context, {0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u}); } -void AttachNullImages(ShaderRecompiler::RecompileResult& result) { +void AttachNullImages(ShaderRecompiler::RecompileResult& result, bool depthAlias = false) { namespace Runtime = ShaderRecompiler::RuntimeAbi; - const auto address = static_cast(reinterpret_cast(NullBindingTexture.data())); - const std::array words{static_cast(address >> 8u), static_cast(address >> 40u) | (56u << 20u) | (((Width - 1u) & 3u) << 30u), ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), 0xfacu | (9u << 28u), 0u, 0u, 0u, 0u}; + const auto address = static_cast(reinterpret_cast(depthAlias ? DepthMemory.data() : NullBindingTexture.data())); + const auto format = depthAlias ? 22u : 56u; + const std::array words{static_cast(address >> 8u), static_cast(address >> 40u) | (format << 20u) | (((Width - 1u) & 3u) << 30u), ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), 0xfacu | (9u << 28u), 0u, 0u, 0u, 0u}; constexpr auto binding = Runtime::FirstStorageImageBinding + 1u; ShaderRecompiler::DescriptorBinding images{ShaderRecompiler::DescriptorKind::StorageImage, ShaderRecompiler::DescriptorRole::GuestImages, Runtime::FragmentDescriptorSet, binding, 2u, std::vector(16u)}; std::copy(words.begin(), words.end(), images.guestDescriptor.begin()); @@ -127,7 +128,7 @@ void AttachNullImages(ShaderRecompiler::RecompileResult& result) { std::vector declarations; if (declareScalar) emit(declarations, spv::OpTypeInt, {scalar, 32u, 0u}); emit(declarations, spv::OpConstant, {scalar, count, 2u}); - emit(declarations, spv::OpTypeImage, {image, floating, spv::Dim2D, 0u, 0u, 0u, 2u, spv::ImageFormatRgba8}); + emit(declarations, spv::OpTypeImage, {image, floating, spv::Dim2D, 0u, 0u, 0u, 2u, depthAlias ? spv::ImageFormatR32f : spv::ImageFormatRgba8}); emit(declarations, spv::OpTypeArray, {array, image, count}); emit(declarations, spv::OpTypePointer, {pointer, spv::StorageClassUniformConstant, array}); emit(declarations, spv::OpVariable, {pointer, variable, spv::StorageClassUniformConstant}); @@ -138,7 +139,7 @@ void AttachNullImages(ShaderRecompiler::RecompileResult& result) { } -void Draw(AgcDriver::VulkanDevice& device, std::span pixelCode, std::uint32_t shaderControl, const VkStencilOpState& stencil, std::uint32_t vertices = 3u, std::uint32_t instances = 1u, bool depthFail = false, const std::function& alter = {}, const std::function& alterDraw = {}, std::span> vertexData = Vertices, std::span vertexCode = VertexCode, std::span pixelData = {}, std::span pixelSrt = {}, bool nullImages = false) { +void Draw(AgcDriver::VulkanDevice& device, std::span pixelCode, std::uint32_t shaderControl, const VkStencilOpState& stencil, std::uint32_t vertices = 3u, std::uint32_t instances = 1u, bool depthFail = false, const std::function& alter = {}, const std::function& alterDraw = {}, std::span> vertexData = Vertices, std::span vertexCode = VertexCode, std::span pixelData = {}, std::span pixelSrt = {}, bool nullImages = false, bool depthAlias = false) { Pixels.fill(Kept); auto target = device.Target(); std::vector capabilities(target.supportedCapabilities.begin(), target.supportedCapabilities.end()); @@ -170,7 +171,7 @@ void Draw(AgcDriver::VulkanDevice& device, std::span pixelC }; fragment.useCache = false; auto pixelResult = ShaderRecompiler::Recompile(fragment); - if (nullImages) AttachNullImages(pixelResult); + if (nullImages) AttachNullImages(pixelResult, depthAlias); if (!pixelResult.stencilReferences.empty()) { Require(pixelResult.stencilReferenceDword * sizeof(std::uint32_t) + sizeof(std::uint32_t) == pixelResult.pushConstants.size(), "dynamic reference is not reserved after the stage payload"); for (std::size_t cursor = 5; cursor < pixelResult.spirv.size();) { @@ -256,6 +257,16 @@ int main() { Check(even, "an empty typed image element preserves the even stencil reference"); Draw(*device, PlainPixelCode, plain, equal(0u)); Check(odd, "an empty typed image element preserves the odd stencil reference"); + bool refused = false; + try { + Draw(*device, alternating, exported, replace(), 3u, 1u, false, {}, {}, Vertices, VertexCode, {}, {}, true, true); + } catch (const std::exception& error) { + const std::string_view reason(error.what()); + refused = reason.find("stencil replay refuses") != std::string_view::npos && reason.find("feedback") != std::string_view::npos; + if (!refused) throw; + } + Require(refused, "a non-null image aliasing the depth surface bypassed replay feedback checks"); + Require(std::ranges::all_of(Pixels, [](std::byte value) { return value == Kept; }), "refused depth feedback published color results"); std::puts("stencil null feedback tests passed"); return 0; } catch (const std::exception& error) { From 090c0dad15122bd004a6a4b246b1209b59a0498c Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:39:33 -0700 Subject: [PATCH 018/103] fix(shader): count a selected record's dwords past its buffer as read Since 682b07eb0, an image whose dword 3 wasn't captured is poisoned (OutsideSnapshot). A record selected past the end of its heap reads nothing there, so it was poisoned instead of selecting a null image, and agc_shader_bindless and agc_driver_image_table failed after the merge of main. A scalar buffer load returns zeros past the end of the buffer, so those dwords now count as read zeros, and the dwords the selection reads count as captured. Every other path keeps main's rule. --- .../recompiler/Optimization/src/ResourceMaterializer.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 505e0aead6..0cc812e22b 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -580,13 +580,17 @@ private: std::uint64_t high = 0; for (std::uint32_t dword = 0; dword < column.dwordCount; dword++) { const auto position = ImageTableAbi::ScalarBufferDword(offset, column.offset + dword * 4u, size); - if (!position.has_value()) continue; + if (!position.has_value()) { + value.availableDwordMask |= static_cast(1u << dword); + continue; + } const auto address = base + *position; if (!PageAccessible(address, base, base + size)) { records.states.front() = RecordState::Unmapped; return; } value.dwords[dword] = Read(address); + value.availableDwordMask |= static_cast(1u << dword); low = std::min(low, address); high = std::max(high, address + sizeof(std::uint32_t)); } From 9800f3ab92c058e649c50c771a131fe641106d96 Mon Sep 17 00:00:00 2001 From: billybobpicklio Date: Thu, 15 Oct 2026 12:45:54 -0400 Subject: [PATCH 019/103] fix(agc): keep each set's image table writes in its own draw snapshot set The vertex and fragment image tables both sit at ImageTableBinding, in sets 0 and 1. PrepareDrawBindings matched the recorded writes to their bindings by binding number alone, so the set 0 table also took the fragment table's writes: they went into the snapshot's set 0, past its variable descriptor count and into the next set in the pool, and the snapshot's set 1 table was left unwritten. Match a write by its original set as well. The recorder test device now enables the descriptor indexing features image tables need. Co-Authored-By: Claude Opus 5.5 --- .../Graphics/src/ShaderResources.cpp | 3 +- .../prx/libSceAgcDriver/tests/Recorder.cpp | 107 ++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 15a0f3c2d2..8ba6f9b60a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -4606,7 +4606,8 @@ std::shared_ptr ShaderResources::PrepareDrawBindi for (std::size_t index = 0; index < selected.size(); ++index) snapshotIndices[selected[index]] = index; auto nextWrite = writes.begin(); for (const auto& binding : bindings) { - while (nextWrite != writes.end() && nextWrite->dstBinding == binding.layout.binding) { + const auto original = setOf(binding) == 0 ? _set : fragment.set; + while (nextWrite != writes.end() && nextWrite->dstBinding == binding.layout.binding && nextWrite->dstSet == original) { auto& write = *nextWrite++; write.dstSet = setOf(binding) == 0 ? result->allocation.set : result->fragmentAllocation.set; if (binding.allocations.empty()) continue; diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 040c2dd195..47569eb2be 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -28,6 +28,7 @@ #include "prx/libc/include/GuestWriteWatch.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include "ResidentPresent.hpp" +#include "ImageTableAbi.hpp" #include "SampleLod_spv.h" #include "BufferUpdate_spv.h" #include "SampleArray_spv.h" @@ -264,6 +265,28 @@ public: extensionsEnabled.insert(extensionsEnabled.end(), libraryExtensions.begin(), libraryExtensions.end()); address.pNext = &library; } + VkPhysicalDeviceDescriptorIndexingFeaturesEXT indexing{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT}; + if (hasExtension(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &indexing}; + function("vkGetPhysicalDeviceFeatures2")(context.physical, &features); + context.descriptorIndexing = indexing.shaderSampledImageArrayNonUniformIndexing == VK_TRUE && indexing.shaderStorageImageArrayNonUniformIndexing == VK_TRUE; + context.partiallyBound = indexing.descriptorBindingPartiallyBound == VK_TRUE; + context.variableDescriptorCount = indexing.descriptorBindingVariableDescriptorCount == VK_TRUE; + context.runtimeDescriptorArray = indexing.runtimeDescriptorArray == VK_TRUE; + } + indexing = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT}; + if (context.descriptorIndexing && context.partiallyBound && context.variableDescriptorCount && context.runtimeDescriptorArray) { + extensionsEnabled.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME); + indexing.shaderSampledImageArrayNonUniformIndexing = VK_TRUE; + indexing.shaderStorageImageArrayNonUniformIndexing = VK_TRUE; + indexing.descriptorBindingPartiallyBound = VK_TRUE; + indexing.descriptorBindingVariableDescriptorCount = VK_TRUE; + indexing.runtimeDescriptorArray = VK_TRUE; + indexing.pNext = address.pNext; + address.pNext = &indexing; + } else { + context.descriptorIndexing = context.partiallyBound = context.variableDescriptorCount = context.runtimeDescriptorArray = false; + } VkPhysicalDeviceRobustness2FeaturesEXT robustness{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT}; if (hasExtension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME)) { VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness}; @@ -292,6 +315,7 @@ public: std::cout << "Vulkan device: " << properties.deviceName << '\n'; context.limits = properties.limits; context.bufferDeviceAddress = true; + context.imageTableCapacity = ImageTableCapacity(context); context.formatProperties = function("vkGetPhysicalDeviceFormatProperties"); context.imageFormatProperties = function("vkGetPhysicalDeviceImageFormatProperties"); context.Function("vkGetDeviceQueue")(context.device, family, 0, &context.queue); @@ -2434,6 +2458,7 @@ public: struct Contents { std::map, VkDescriptorBufferInfo> buffers; std::map, VkDescriptorImageInfo> images; + std::map, std::uint32_t> written; }; explicit DescriptorUpdates(Context& context) : context(context), previous(context.functions), update(context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")) { @@ -2452,6 +2477,7 @@ private: Contents contents; for (std::uint32_t i = 0; i < count; ++i) { const auto& write = writes[i]; + contents.written[{write.dstSet, write.dstBinding}] += write.descriptorCount; for (std::uint32_t element = 0; element < write.descriptorCount; ++element) { const auto key = std::pair{write.dstBinding, write.dstArrayElement + element}; if (write.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) contents.buffers[key] = write.pBufferInfo[element]; @@ -2522,6 +2548,85 @@ void drawDescriptorWriteTests(const Device& device, Recorder& recorder) { recorder.Activate(); } +void drawSnapshotTableSetTests(const Device& device, Recorder& recorder) { + namespace Abi = ShaderRecompiler::ImageTableAbi; + using ShaderRecompiler::DescriptorKind; + using ShaderRecompiler::DescriptorRole; + const auto& base = device.GetContext(); + TextureDetiler detiler(base); + auto context = base; + context.detiler = &detiler; + if (context.imageTableCapacity < 2u) { + std::cout << "image tables unavailable: draw snapshot table sets not tested\n"; + return; + } + constexpr std::uint32_t width = 4; + constexpr std::uint32_t height = 4; + alignas(256) static std::array texels{}; + texels.fill(1.0f); + const auto address = static_cast(reinterpret_cast(texels.data())); + const std::array texture{static_cast(address >> 8u), static_cast((address >> 40u) & 0xffu) | (22u << 20u) | (((width - 1u) & 3u) << 30u), ((width - 1u) >> 2u) | ((height - 1u) << 14u), 0xfacu | (9u << 28u), 0u, 0u, 0u, 0u}; + constexpr auto bindingCount = static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count); + const auto addTable = [&](ShaderRecompiler::RecompileResult& program, ShaderRecompiler::RuntimeAbi::Stage stage, std::uint32_t set, std::uint32_t elements) { + ShaderRecompiler::DescriptorBinding map{}; + map.kind = DescriptorKind::StorageBuffer; + map.role = DescriptorRole::ImageTableMap; + map.descriptorSet = set; + map.binding = static_cast(stage) * bindingCount + static_cast(ShaderRecompiler::RuntimeAbi::Binding::ImageTableMap); + map.count = 1; + map.guestDescriptor.assign(Abi::HeaderWords + elements, 0u); + map.guestDescriptor[Abi::HeaderVersion] = Abi::Version; + map.guestDescriptor[Abi::HeaderElementCount] = elements; + map.guestDescriptor[Abi::HeaderElementFaults] = Abi::HeaderWords; + map.guestDescriptor[Abi::HeaderSamplerFaults] = Abi::HeaderWords + elements; + program.bindings.push_back(map); + ShaderRecompiler::DescriptorBinding images{}; + images.kind = DescriptorKind::SampledImage; + images.role = DescriptorRole::ImageTable; + images.descriptorSet = set; + images.binding = ShaderRecompiler::RuntimeAbi::ImageTableBinding; + images.count = elements; + for (std::uint32_t element = 0; element < elements; ++element) images.guestDescriptor.insert(images.guestDescriptor.end(), texture.begin(), texture.end()); + images.imageShapes.assign(elements, ShaderRecompiler::DescriptorImageShape::Image2D); + images.imageDepthCompare.assign(elements, false); + images.imageSamplers.assign(elements, 0u); + program.bindings.push_back(std::move(images)); + }; + DescriptorUpdates updates(context); + { + DescriptorCache cache(context); + context.descriptorCache = &cache; + Recorder snapshots(context); + snapshots.Activate(); + ShaderRecompiler::RecompileResult vertex, fragment; + ShaderRecompiler::DescriptorBinding data{}; + data.kind = DescriptorKind::StorageBuffer; + data.role = DescriptorRole::FlattenedSrt; + data.binding = static_cast(ShaderRecompiler::RuntimeAbi::Binding::FlattenedSrt); + data.count = 1; + data.guestDescriptor = {1, 2, 3, 4}; + vertex.bindings.push_back(data); + addTable(vertex, ShaderRecompiler::RuntimeAbi::Stage::Main, 0u, 1u); + addTable(fragment, ShaderRecompiler::RuntimeAbi::Stage::Fragment, ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 2u); + const ColorTarget target{}; + ShaderResources resources(context, vertex, fragment, target, 0, 0); + Require(resources.SplitSets(), "a fragment image table did not get its own descriptor set"); + const ShaderResources::MovedBuffer moved{0, 0, 16, {5, 6, 7, 8}}; + const auto draw = resources.PrepareDrawBindings(snapshots, std::span(&moved, 1)); + Require(draw != nullptr && draw->snapshots.size() == 1, "moved shader data did not create a draw snapshot"); + const auto& written = updates.calls.back().written; + const auto tableWrites = [&](VkDescriptorSet set) { + const auto it = written.find({set, ShaderRecompiler::RuntimeAbi::ImageTableBinding}); + return it == written.end() ? 0u : it->second; + }; + Require(tableWrites(draw->allocation.set) == 1u, "the draw snapshot wrote the wrong number of image table descriptors into the vertex set"); + Require(tableWrites(draw->fragmentAllocation.set) == 2u, "the draw snapshot wrote the wrong number of image table descriptors into the fragment set"); + snapshots.Sync(); + } + ClearCachedTextures(context.device); + recorder.Activate(); +} + void drawSnapshotReuseTests(const Device& device, Recorder& recorder) { const auto& context = device.GetContext(); constexpr std::size_t bytes = 65536; @@ -8199,6 +8304,7 @@ int main(int argc, char** argv) { } if (argc == 2 && std::string_view(argv[1]) == "--draw-descriptors-only") { drawDescriptorWriteTests(device, recorder); + drawSnapshotTableSetTests(device, recorder); std::cout << "Draw descriptor writes and snapshot overrides passed\n"; return 0; } @@ -8256,6 +8362,7 @@ int main(int argc, char** argv) { misalignedRegionTests(device, recorder); drawSnapshotReuseTests(device, recorder); drawDescriptorWriteTests(device, recorder); + drawSnapshotTableSetTests(device, recorder); drawSnapshotCapTests(device, recorder); drawSnapshotEvictionTests(device); unwrittenSnapshotTests(device); From 1cfa32a86b22558eaebfdd2f31689d4dbbd10ad4 Mon Sep 17 00:00:00 2001 From: GalaxyGamingFR <153150716+GalaxyGamingFR@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:50:02 -0700 Subject: [PATCH 020/103] fix(check_compile): take changed units from the merge base changed() diffed the working tree against the tip of the base, so files changed only on upstream main since the branch point were reported and compiled. It now diffs against the merge base of the base and HEAD. A regression test covers a base-only change. Co-Authored-By: Claude Opus 5.5 --- tools/check_compile.py | 5 ++++- tools/tests/test_check_compile.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tools/check_compile.py b/tools/check_compile.py index 1ec01a433c..71bb8a836b 100644 --- a/tools/check_compile.py +++ b/tools/check_compile.py @@ -58,7 +58,10 @@ def index(build): def changed(root, base): - result = capture(["git", "-C", str(root), "diff", "--name-only", "--diff-filter=ACMR", "-z", base]) + fork = capture(["git", "-C", str(root), "merge-base", base, "HEAD"]) + if fork.returncode != 0: + raise ValueError(f"git merge-base {base}: {fork.stderr.strip()}") + result = capture(["git", "-C", str(root), "diff", "--name-only", "--diff-filter=ACMR", "-z", fork.stdout.strip()]) if result.returncode != 0: raise ValueError(f"git diff {base}: {result.stderr.strip()}") untracked = capture(["git", "-C", str(root), "ls-files", "--others", "--exclude-standard", "-z"]) diff --git a/tools/tests/test_check_compile.py b/tools/tests/test_check_compile.py index b691d60d45..90d85a9602 100644 --- a/tools/tests/test_check_compile.py +++ b/tools/tests/test_check_compile.py @@ -170,6 +170,28 @@ class CheckCompileTests(unittest.TestCase): self.assertEqual(result.returncode, 1) self.assertIn("changed.cpp:1:1: error: no", result.stderr) + def test_files_changed_only_on_the_base_since_the_branch_point_are_not_checked(self): + script = self.compiler("import sys\nsys.stderr.write(sys.argv[-1] + ': error: no\\n')\nsys.exit(1)\n") + self.manifest([self.entry(name, f'"{sys.executable}" "{script}" "{self.root / name}"') + for name in ("mine.cpp", "other.cpp")]) + git = ["git", "-C", str(self.root)] + self.source("other.cpp") + subprocess.run([*git, "add", "other.cpp"], check=True) + subprocess.run([*git, "commit", "-qm", "add"], check=True) + subprocess.run([*git, "branch", "-M", "main"], check=True) + subprocess.run([*git, "checkout", "-qb", "side"], check=True) + self.source("mine.cpp") + subprocess.run([*git, "add", "mine.cpp"], check=True) + subprocess.run([*git, "commit", "-qm", "mine"], check=True) + subprocess.run([*git, "checkout", "-q", "main"], check=True) + self.source("other.cpp", "int y;\n") + subprocess.run([*git, "commit", "-qam", "other"], check=True) + subprocess.run([*git, "checkout", "-q", "side"], check=True) + result = self.tool("--base", "main") + self.assertEqual(result.returncode, 1) + self.assertIn("mine.cpp: error: no", result.stderr) + self.assertNotIn("other.cpp", result.stderr) + def test_no_changed_translation_unit_is_not_a_failure(self): self.source("clean.cpp") subprocess.run(["git", "-C", str(self.root), "add", "clean.cpp"], check=True) From a70dea62e7facfee1146154767551f49a241f13b Mon Sep 17 00:00:00 2001 From: GalaxyGamingFR <153150716+GalaxyGamingFR@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:50:13 -0700 Subject: [PATCH 021/103] fix(check_main_sync): fetch an explicit --base before resolving it main() resolved the base before running --fetch, so an explicit base on a remote that was never fetched failed with "no base ref found" and never fetched. The fetch is now a local helper and runs first when --base is given; the default base still resolves first, then fetches. Co-Authored-By: Claude Opus 5.5 --- tools/check_main_sync.py | 11 ++++++++--- tools/tests/test_check_main_sync.py | 8 ++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/check_main_sync.py b/tools/check_main_sync.py index fa3265d310..12d5b78a32 100644 --- a/tools/check_main_sync.py +++ b/tools/check_main_sync.py @@ -97,12 +97,17 @@ if __name__ == "__main__": parser.add_argument("--strict", action="store_true", help="also fail when the branch is behind the base") args = parser.parse_args() try: - base = resolve_base(args.repo, args.base) - if args.fetch: - remote, _, branch = base.partition("/") + def fetch(ref): + remote, _, branch = ref.partition("/") if not branch: parser.error("--fetch needs a remote-tracking base such as upstream/main") git(args.repo, "fetch", "--quiet", remote, branch) + + if args.fetch and args.base: + fetch(args.base) + base = resolve_base(args.repo, args.base) + if args.fetch and not args.base: + fetch(base) data = report(args.repo, base, args.head) except RuntimeError as error: parser.error(str(error)) diff --git a/tools/tests/test_check_main_sync.py b/tools/tests/test_check_main_sync.py index 4a13a56e0a..d893e70b0e 100644 --- a/tools/tests/test_check_main_sync.py +++ b/tools/tests/test_check_main_sync.py @@ -177,6 +177,14 @@ class CheckMainSyncTests(unittest.TestCase): self.assertEqual(result.returncode, 2) self.assertIn("no base ref found", result.stderr) + def test_explicit_base_is_fetched_before_it_is_resolved(self): + with tempfile.TemporaryDirectory() as other: + subprocess.run(["git", "clone", "-q", self.directory.name, other], check=True, capture_output=True) + self.repo.git("remote", "add", "upstream", other) + result = subprocess.run([sys.executable, str(TOOL), "--repo", self.directory.name, "--base", "upstream/main", "--fetch"], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("merge: clean", result.stdout) + if __name__ == "__main__": unittest.main() From 74f6a23bbaa9e23267edbf86e0c64df41436f9b4 Mon Sep 17 00:00:00 2001 From: Ishmam Ahmed <89687266+IshmamDC217@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:24:05 +0100 Subject: [PATCH 022/103] fix(tools): find Python and CMake comments across multi-line strings The comment check looked at each added .py, .cmake and CMakeLists.txt line on its own and treated any # outside a string closed on that line as a comment. A # inside a Python triple-quoted string or a CMake quoted or bracket argument that spans lines was reported, so adding an #include to the C++ held in PatchSpirvTools.cmake's [=[ ]=] arguments failed the check. An escaped \# in CMake was reported too, while the inner lines of a multi-line #[[ ]] bracket comment were not. Find comments over the whole file instead: Python COMMENT tokens from tokenize, and a CMake scan that skips quoted and bracket arguments and escapes and covers every line of a bracket comment. A Python file that does not tokenize keeps the previous per-line check. --- tools/check_conventions.py | 25 +++++++++- .../tests/test_check_conventions_comments.py | 48 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/tools/check_conventions.py b/tools/check_conventions.py index 000a1de445..2f682e14de 100644 --- a/tools/check_conventions.py +++ b/tools/check_conventions.py @@ -1,10 +1,12 @@ import argparse +import io import json import os import posixpath import re import subprocess import sys +import tokenize from collections import defaultdict from pathlib import Path, PurePosixPath from urllib.parse import unquote @@ -43,6 +45,7 @@ DEBT_TARGET = re.compile(r"\]\((?:\.\./)+(core/[^)#\s]+)") CONVENTIONAL = re.compile(r"^(feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)(\([^)]+\))?!?: \S") STRINGS = re.compile(r'"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'') ALLOWED_COMMENT = re.compile(r"^\s*(#\s*endif\b|}\s*;?\s*//\s*(end\s+)?namespace\b)") +CMAKE_TOKEN = re.compile(r'\[(=*)\[.*?\]\1\]|"(?:\\.|[^"\\])*"|(?P#\[(?P=*)\[.*?\](?P=level)\]|#[^\n]*)|\\.', re.S) EXPORT = re.compile(r"\bAPS5_VABI\s+(\w+)\s*\(") STUB_BODY = re.compile(r"^(?:\(void\)\s*\w+\s*;|static_cast\(\s*\w+\s*\)\s*;)*" r"(?:return\s*\(?\s*(?:-?(?:0x[0-9a-fA-F]+|\d+)[uUlL]*|nullptr|true|false|\w*_OK)?\s*\)?\s*;)?$") @@ -159,9 +162,12 @@ class Check: self.report("silent-stub", path, number, match.group(1)) self.duplicates(exports) for path, lines in self.files(lambda p: p.suffix in (".py", ".cmake") or p.name == "CMakeLists.txt"): + source = self.show(path) + comments = python_comments(source) if path.endswith(".py") else cmake_comments(source) for number, text in lines: shebang = text.startswith("#!") or (number == 1 and text.startswith("\ufeff#!")) - if "#" in STRINGS.sub('""', text) and not shebang: + comment = number in comments if comments is not None else "#" in STRINGS.sub('""', text) + if comment and not shebang: self.report("comment", path, number) if any(is_debt(path) for path in self.status): self.findings = [finding for finding in self.findings if finding[0] != "comment"] @@ -399,6 +405,23 @@ def allowed_notes(path): return path in ("README.md", "CONTRIBUTING.md") or p.parts[0] == ".github" or p.parts[:2] in (("docs", "dev"), ("docs", "user")) +def python_comments(text): + try: + tokens = list(tokenize.generate_tokens(io.StringIO(text.removeprefix("\ufeff")).readline)) + except (tokenize.TokenError, SyntaxError): + return None + return {token.start[0] for token in tokens if token.type == tokenize.COMMENT} + + +def cmake_comments(text): + lines = set() + for match in CMAKE_TOKEN.finditer(text): + if match.group("comment") is not None: + first = text.count("\n", 0, match.start()) + 1 + lines.update(range(first, first + match.group().count("\n") + 1)) + return lines + + def cpp_text(text, preserve_comments=False): def mask(match): if match.group("code") is not None or preserve_comments and match.group("comment") is not None: diff --git a/tools/tests/test_check_conventions_comments.py b/tools/tests/test_check_conventions_comments.py index b7f46f54f8..9156226179 100644 --- a/tools/tests/test_check_conventions_comments.py +++ b/tools/tests/test_check_conventions_comments.py @@ -157,6 +157,54 @@ class RawLiteralCommentTests(unittest.TestCase): self.assertEqual(self.findings(comment, [1]), [("comment", self.path, 1, "")]) +class HashCommentTests(unittest.TestCase): + def findings(self, path, source, added): + check = Check.__new__(Check) + check.tree = set() + check.status = {} + check.findings = [] + lines = source.split("\n") + check.added = {path: [(number, lines[number - 1]) for number in added]} + with patch.object(check, "show", return_value=source), patch.object(check, "duplicates"): + check.code() + return check.findings + + def test_hash_inside_python_multiline_string_is_not_a_comment(self): + for quote in ('"""', "'''"): + with self.subTest(quote=quote): + source = f'TEMPLATE = {quote}\n#include \n{quote}\nprint("#ok")\n' + self.assertEqual(self.findings("tools/generate.py", source, [1, 2, 3, 4]), []) + + def test_python_comment_after_a_multiline_string_is_reported(self): + source = 'TEMPLATE = """\n#include \n""" # pending\nvalue = 1 # pending\n' + self.assertEqual(self.findings("tools/generate.py", source, [2, 3, 4]), + [("comment", "tools/generate.py", 3, ""), ("comment", "tools/generate.py", 4, "")]) + + def test_python_shebang_is_not_a_comment(self): + for source in ("#!/usr/bin/env python3\nprint(1)\n", "\ufeff#!/usr/bin/env python3\nprint(1)\n"): + with self.subTest(source=source): + self.assertEqual(self.findings("tools/generate.py", source, [1, 2]), []) + + def test_unparsable_python_falls_back_to_the_line_check(self): + source = 'TEMPLATE = """\n#include \n' + self.assertEqual(self.findings("tools/generate.py", source, [2]), [("comment", "tools/generate.py", 2, "")]) + + def test_hash_inside_cmake_quoted_and_bracket_arguments_is_not_a_comment(self): + for source in ('set(content "#ifndef GUARD\n#define GUARD\n#endif\n")\n', + 'set(patch [=[\n#include \n]=])\n', + 'set(patch [[\n#include \n]])\n', + 'message(STATUS "a\\"#b")\n', + 'string(REPLACE \\# x out "${in}")\n'): + with self.subTest(source=source): + added = list(range(1, source.count("\n") + 1)) + self.assertEqual(self.findings("cmake/Generate.cmake", source, added), []) + + def test_cmake_line_and_bracket_comments_are_reported(self): + source = 'set(value 1) # pending\n#[[\npending\n]]\nset(patch [=[\n#include \n]=]) # pending\n' + self.assertEqual(self.findings("CMakeLists.txt", source, [1, 2, 3, 4, 6, 7]), + [("comment", "CMakeLists.txt", number, "") for number in (1, 2, 3, 4, 7)]) + + class SilentStubTests(unittest.TestCase): def test_commented_stub_is_reported_when_debt_change_allows_comments(self): checker = Path(__file__).resolve().parents[1] / "check_conventions.py" From e54ea3ea298f988e9708a9388d1cacec317b23ce Mon Sep 17 00:00:00 2001 From: simonbronk Date: Fri, 16 Oct 2026 10:51:42 +0200 Subject: [PATCH 023/103] fix(stubgen): reject read-only outputs before replacement --- tools/stubgen.py | 5 +++++ tools/tests/test_stubgen.py | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tools/stubgen.py b/tools/stubgen.py index 2d451278cc..ae9f68955b 100644 --- a/tools/stubgen.py +++ b/tools/stubgen.py @@ -2,6 +2,7 @@ import argparse import os import re +import stat import sys import tempfile from pathlib import Path @@ -194,6 +195,10 @@ def write_outputs(outputs): temporary.chmod(path.stat().st_mode & 0o7777) else: temporary.chmod(0o666 & ~umask) + if os.name == "nt": + for _, path in staged: + if path.exists() and path.stat().st_file_attributes & stat.FILE_ATTRIBUTE_READONLY: + raise PermissionError(f"Output is read-only: {path}") for temporary, path in staged: temporary.replace(path) finally: diff --git a/tools/tests/test_stubgen.py b/tools/tests/test_stubgen.py index 3a8e9a8be8..bc4bf37488 100644 --- a/tools/tests/test_stubgen.py +++ b/tools/tests/test_stubgen.py @@ -41,6 +41,31 @@ class StubgenTests(unittest.TestCase): class StubGeneratorTests(unittest.TestCase): + @unittest.skipUnless(os.name == "nt", "Windows denies replacing read-only output files") + def test_readonly_second_output_preserves_both_outputs(self): + import subprocess + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + nids = root / "nids.txt" + code = root / "Export.cpp" + debt = root / "debt.md" + nids.write_text("AC1FtjqMCL0\n", encoding="utf-8") + code.write_bytes(b"original code\n") + debt.write_bytes(b"original debt\n") + debt.chmod(stat.S_IREAD) + try: + result = subprocess.run( + [sys.executable, str(Path(stubgen.__file__)), "--module", "libSceTest", + "--nids", str(nids), "--out", str(code), "--td-out", str(debt)], + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("Traceback", result.stderr) + self.assertEqual(code.read_bytes(), b"original code\n") + self.assertEqual(debt.read_bytes(), b"original debt\n") + self.assertEqual(sorted(path.name for path in root.iterdir()), ["Export.cpp", "debt.md", "nids.txt"]) + finally: + debt.chmod(stat.S_IREAD | stat.S_IWRITE) + def test_default_stdout_contains_only_cpp(self): with tempfile.TemporaryDirectory() as directory: nids = Path(directory) / "nids.txt" From ca174bc7e630ad11adda871fc7f3cc9cd73cf44f Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Fri, 16 Oct 2026 02:39:51 -0700 Subject: [PATCH 024/103] docs(agc): drop the disk cache's format version from ARCHITECTURE.md The line said FormatVersion 36 while the code is at 37, as main's says 33 against 36. Without the number it no longer goes stale on each bump or conflicts between PRs that bump it. --- docs/dev/ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 4a54b36e5e..8a1a764275 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -119,7 +119,7 @@ At every invocation, `TableSnapshotter` in [`ResourceMaterializer.cpp`](../../co `ResolveImageTables` turns the snapshot into the map buffer (`DescriptorRole::ImageTableMap`, layout in [`ImageTableAbi.hpp`](../../core/shader/recompiler/ImageTableAbi.hpp)): a header, eight words per table (map start, key count, heap base and size, the code for keys past the snapshot, a table fault) and one record per key (flags with the decode mode, the swizzle dword and the element for images; flags and element for samplers). The shader reads `map[key]` and switches over the image's prepared runtime modes. Each distinct T# becomes one element, up to `ImageElementBudget` and the device's capacity. An entry that does not decode, is multisampled or over budget, pairs a reducing sampler with a point-only mode, is an unnormalized or reducing S#, or whose surface has no mapped byte is poisoned: it reads zeros and reports a fault through the BDA fault buffer only when a lane selects it (`APS5_IMAGE_TABLE_STRICT=1` throws at the snapshot instead). The driver's `resolveImageTables` in [`ShaderResources.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) looks up or creates a texture per element and keeps the last four resolved tables per thread for reuse within one collect epoch while their memory is unchanged. Cached objects grow with their tables, so the resource cache keeps at most `ResourceCache::TableElementBudget()` table elements (65,536, `APS5_RESOURCE_CACHE_TABLE_ELEMENTS`) and evicts its least recently used table objects past that, and the dispatch cache evicts its least recently used entries while their captured words exceed `Driver::DispatchCacheBytes()` (256 MiB, `APS5_DISPATCH_CACHE_BYTES`). -The disk cache (`FormatVersion` 36) stores the table and table operation of each image and sampler, the image shapes of the table binding, the invocation's table poison list, read ranges and fault count, and the table column of each descriptor source in the front-end plan, with its key domains and its selection. +The disk cache stores the table and table operation of each image and sampler, the image shapes of the table binding, the invocation's table poison list, read ranges and fault count, and the table column of each descriptor source in the front-end plan, with its key domains and its selection. ### Untracked storage image comparisons From 7e383a894341dd26148b417da7fb5ff40e463e56 Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:02:22 +0400 Subject: [PATCH 025/103] fix: key oracle builds by the compiler selected through PATH --- tools/hw-oracle/hw_oracle.py | 4 +++- tools/hw-oracle/test_hw_oracle.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tools/hw-oracle/hw_oracle.py b/tools/hw-oracle/hw_oracle.py index 8424ab09bf..427f1ed006 100644 --- a/tools/hw-oracle/hw_oracle.py +++ b/tools/hw-oracle/hw_oracle.py @@ -44,7 +44,9 @@ def oracle(): root = rocm_root() if root is not None: root = root.resolve() - configuration = repr((compiler, str(root))).encode("utf-8") + executable = shutil.which(compiler[0]) + selected = str(Path(executable).resolve()) if executable is not None else None + configuration = repr((compiler, selected, str(root))).encode("utf-8") key = hashlib.sha256(source.read_bytes() + configuration).hexdigest() directory = CACHE / key binary = directory / "oracle" diff --git a/tools/hw-oracle/test_hw_oracle.py b/tools/hw-oracle/test_hw_oracle.py index 31fa5240e0..5cdc8c9e47 100644 --- a/tools/hw-oracle/test_hw_oracle.py +++ b/tools/hw-oracle/test_hw_oracle.py @@ -299,6 +299,30 @@ class TargetOverrideTests(unittest.TestCase): class OracleConfigurationTests(unittest.TestCase): + @unittest.skipIf(os.name == "nt", "POSIX executable compiler fixture") + def test_path_selecting_another_compiler_invalidates_the_cache(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in ("first", "second"): + compiler = root / name / "fixture-cc" + compiler.parent.mkdir() + compiler.write_text( + "#!" + sys.executable + "\nimport sys\nfrom pathlib import Path\n" + "Path(sys.argv[sys.argv.index('-o') + 1]).write_bytes(" + repr(name.encode()) + ")\n", + encoding="utf-8") + compiler.chmod(0o755) + with patch.object(hw_oracle, "CACHE", root / "cache"), \ + patch.object(hw_oracle, "rocm_root", return_value=None): + with patch.dict(os.environ, {"CC": "fixture-cc", "PATH": str(root / "first")}): + first = hw_oracle.oracle() + self.assertEqual(first.read_bytes(), b"first") + self.assertEqual(hw_oracle.oracle(), first) + with patch.dict(os.environ, {"CC": "fixture-cc", "PATH": str(root / "second")}): + second = hw_oracle.oracle() + self.assertEqual(second.read_bytes(), b"second") + self.assertNotEqual(first, second) + + @unittest.skipUnless(sys.platform.startswith("linux"), "Linux compiler and shared-library fixture") def test_cache_and_compiler_command_with_host_toolchain(self): compiler = shutil.which("cc") From d40347def7858c64924c878060e90d37e0197d3f Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:27:44 +0400 Subject: [PATCH 026/103] fix: preserve quoted Git filenames in overlap detection --- tools/pr_overlap.py | 21 ++++++++----- tools/tests/test_pr_overlap.py | 39 +++++++++++++++++++++++++ tools/tests/test_pr_overlap_encoding.py | 4 +-- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/tools/pr_overlap.py b/tools/pr_overlap.py index 6a558f828d..0351d6a685 100644 --- a/tools/pr_overlap.py +++ b/tools/pr_overlap.py @@ -17,7 +17,11 @@ FETCH_REFS = 100 def git(*args): - return subprocess.run(["git", "-c", "core.quotePath=false", *args], capture_output=True, text=True, encoding="utf-8", errors="replace", check=True).stdout + result = subprocess.run(["git", "-c", "core.quotePath=false", *args], capture_output=True) + result.stdout = result.stdout.decode("utf-8", errors="replace") + result.stderr = result.stderr.decode("utf-8", errors="replace") + result.check_returncode() + return result.stdout def own(prs, sha): @@ -79,11 +83,12 @@ def fetch(base, prs): def merge(*args): - result = subprocess.run(["git", "-c", "core.quotePath=false", "merge-tree", "--write-tree", "--name-only", "--no-messages", *args], - capture_output=True, text=True, encoding="utf-8", errors="replace") - lines = result.stdout.splitlines() - if result.returncode not in (0, 1) or not lines: - raise RuntimeError(f"git merge-tree {' '.join(args)} failed: {result.stderr.strip()}") + result = subprocess.run(["git", "-c", "core.quotePath=false", "merge-tree", "--write-tree", "--name-only", "--no-messages", "-z", *args], + capture_output=True) + lines = result.stdout.decode("utf-8", errors="replace").rstrip("\0").split("\0") + if result.returncode not in (0, 1) or not lines[0]: + error = result.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(f"git merge-tree {' '.join(args)} failed: {error}") return lines[0], set(lines[1:]) if result.returncode else set() @@ -115,8 +120,8 @@ def scan(pr): pr["tree"] = None if conflicted else commit(tree) old, new = (git("merge-base", "refs/pr/base", pr["ref"]).strip(), pr["ref"]) if conflicted else ("refs/pr/base", pr["tree"]) pr["files"], pr["added"] = set(), set() - for line in git("diff", "--name-status", "--no-renames", old, new).splitlines(): - status, path = line.split("\t", 1) + changes = git("diff", "--name-status", "--no-renames", "-z", old, new).split("\0") + for status, path in zip(changes[::2], changes[1::2]): pr["files"].add(path) if status == "A": pr["added"].add(path) diff --git a/tools/tests/test_pr_overlap.py b/tools/tests/test_pr_overlap.py index dad871a5e2..3127a1b00d 100644 --- a/tools/tests/test_pr_overlap.py +++ b/tools/tests/test_pr_overlap.py @@ -227,6 +227,45 @@ class ConflictTests(unittest.TestCase): class UnicodePathTests(unittest.TestCase): + @unittest.skipIf(os.name == "nt", "POSIX filenames containing control characters") + def test_control_characters_keep_exact_paths_in_scans_and_conflicts(self): + previous = os.getcwd() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.run( + ["git", "-c", "user.name=Overlap Test", "-c", "user.email=test@example.com", + "-c", "commit.gpgsign=false", *args], cwd=root, + capture_output=True, text=True, encoding="utf-8", check=True).stdout.strip() + filenames = {"core/libs/prx/libExample/return\r.cpp", "core/libs/prx/libExample/tab\t.cpp", "core/libs/prx/libExample/line\n.cpp", + "core/libs/prx/libExample/quote\".cpp"} + for filename in filenames: + source = root / filename + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text("int value = 0;\n", encoding="utf-8") + git("init", "-q") + git("add", ".") + git("commit", "-qm", "test: create base") + base = git("rev-parse", "HEAD") + git("update-ref", "refs/pr/base", base) + for branch, value in (("left", 1), ("right", 2)): + git("checkout", "-qb", branch, base) + for filename in filenames: + (root / filename).write_text(f"int value = {value};\n", encoding="utf-8") + git("add", ".") + git("commit", "-qm", "fix: change value") + try: + os.chdir(root) + left, right = {"ref": "left"}, {"ref": "right"} + pr_overlap.scan(left) + pr_overlap.scan(right) + self.assertEqual(left["files"], filenames) + self.assertEqual(right["files"], filenames) + self.assertEqual(pr_overlap.conflicts(left, right), {name: [(1, 1)] for name in filenames}) + finally: + os.chdir(previous) + + def test_scans_and_reports_conflicts_with_git_path_quoting_enabled(self): previous = os.getcwd() with tempfile.TemporaryDirectory() as directory: diff --git a/tools/tests/test_pr_overlap_encoding.py b/tools/tests/test_pr_overlap_encoding.py index c8c04aff4d..1dad26c746 100644 --- a/tools/tests/test_pr_overlap_encoding.py +++ b/tools/tests/test_pr_overlap_encoding.py @@ -81,11 +81,11 @@ class EncodingTests(unittest.TestCase): def test_merge_conflict_preserves_unicode_paths(self): path = UNICODE + ".txt" - with command_output(TREE + "\n" + path + "\n", returncode=1): + with command_output(TREE + "\0" + path + "\0", returncode=1): self.assertEqual(pr_overlap.merge("main", "topic"), (TREE, {path})) def test_merge_success(self): - with command_output(TREE + "\n"): + with command_output(TREE + "\0"): self.assertEqual(pr_overlap.merge("main", "topic"), (TREE, set())) def test_merge_error_preserves_unicode(self): From 7a09d070c5ad964e45529f4df603d2a3cde56f2a Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:29:00 +0400 Subject: [PATCH 027/103] fix: classify every explicit export alias in import audits --- tools/import_audit.py | 11 +++++------ tools/tests/test_import_audit.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tools/import_audit.py b/tools/import_audit.py index 037dba8fe3..100c912ce0 100644 --- a/tools/import_audit.py +++ b/tools/import_audit.py @@ -360,10 +360,10 @@ def read_modules(directories): return imports, {name: sorted(files) for name, files in exports.items()} -def nid_of(name, aliases): +def nids_of(name, aliases): if name in aliases: return aliases[name] - return nid_names.compute_nid(name.removesuffix(NID_POSTFIX)) + return {nid_names.compute_nid(name.removesuffix(NID_POSTFIX))} def source_stubs(source): @@ -375,14 +375,13 @@ def source_stubs(source): aliases = {} for file in library.rglob("*.cpp"): for nid, name in nid_names.source_aliases(file.read_text(errors="ignore")): - aliases[name] = nid + aliases.setdefault(name, set()).add(nid) group = progress.scan_library(library) if "shared_sources" in group: shared[library.name] = group["shared_sources"] - done = {nid_of(name, aliases) for name in group["done_names"]} + done = {nid for name in group["done_names"] for nid in nids_of(name, aliases)} for name in group["todo_names"]: - nid = nid_of(name, aliases) - if nid not in done: + for nid in nids_of(name, aliases) - done: stubs.setdefault((library.name + ".prx", nid), name) for library, owner in shared.items(): chain = [library] diff --git a/tools/tests/test_import_audit.py b/tools/tests/test_import_audit.py index 9a0359e654..ef860de913 100644 --- a/tools/tests/test_import_audit.py +++ b/tools/tests/test_import_audit.py @@ -907,6 +907,19 @@ class ImportAuditTests(unittest.TestCase): with self.assertRaisesRegex(import_audit.AuditError, "shared source cycle.*libA.*libB"): import_audit.source_stubs(source) + def test_multiple_aliases_of_one_function_keep_the_same_classification(self): + aliases = 'APS5_EXPORT("AAAAAAAAAAA", sceExample);\nAPS5_EXPORT("BBBBBBBBBBB", sceExample);\n' + for body, classification in ((STUB_SOURCE, "stub"), (DONE_SOURCE, "implemented")): + with self.subTest(classification=classification): + write_source(self.work.source, "libSceExample", aliases + body.format(name="sceExample")) + stubs = import_audit.source_stubs(self.work.source) + records, _ = import_audit.audit( + [(nid, "libSceExample.prx", False) for nid in ("AAAAAAAAAAA", "BBBBBBBBBBB")], + {nid: ["libSceExample.prx"] for nid in ("AAAAAAAAAAA", "BBBBBBBBBBB")}, stubs, + {"libSceExample.prx"}, set()) + self.assertEqual([record["class"] for record in records], [classification, classification]) + + def test_formatted_export_aliases_remain_stubs(self): work = self.work nid = "AAAAAAAAAAA" From ef3c83d6becbcd6c282d2d5e33d38385898212f5 Mon Sep 17 00:00:00 2001 From: ArjenS-LA Date: Fri, 16 Oct 2026 04:44:34 -0700 Subject: [PATCH 028/103] fix(tools): resolve named audit imports through NIDs --- docs/user/USAGE.md | 2 +- tools/import_audit.py | 10 +++++-- tools/tests/test_import_audit.py | 47 ++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index c38899b148..4be6d5398d 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -237,7 +237,7 @@ The report also lists needed libraries that have no file in `--libs` or `--modul | `--names ` | NID and name per line, as in `aerolib.csv` from [`tools/nid_names.py`](../../tools/nid_names.py); names the absent imports. Never downloaded; a name that does not hash to its NID is marked. | | `--json ` | Full result, stamped with the repository's commit and whether its tree was modified. | -Registry `nid` fields may contain either encoded NIDs or ordinary ELF symbol names such as `strcmp`. Ordinary names are matched against literal exports, without hashing them into NIDs. +Registry `nid` fields may contain either encoded NIDs or ordinary ELF symbol names such as `strcmp`. Imports are matched against literal exports first, then against their computed NIDs when the literal lookup misses. Reports retain the original import name. Export availability does not establish that every platform loader supports that lookup path. Exit codes: `0` nothing blocks loading, `1` there are `absent` imports or needed libraries without a file, `2` an input could not be read; the message names the file and the value. diff --git a/tools/import_audit.py b/tools/import_audit.py index 4059b91a3a..528e20e45e 100644 --- a/tools/import_audit.py +++ b/tools/import_audit.py @@ -459,11 +459,17 @@ def audit(imports, exports, stubs, libraries, modules, module_imports=None, modu providers = exports.get(nid, []) if library is None: providers = sorted(set(providers) | set((module_exports or {}).get(nid, []))) - kind = classify(nid, library, providers, stubs, modules) + resolved_nid = nid + if not providers: + resolved_nid = nid_names.compute_nid(nid) + providers = exports.get(resolved_nid, []) + if library is None: + providers = sorted(set(providers) | set((module_exports or {}).get(resolved_nid, []))) + kind = classify(resolved_nid, library, providers, stubs, modules) record["class"] = kind record["providers"] = providers record["library_mismatch"] = kind in ("implemented", "stub") and bool(library) and library not in providers and "libc.prx" not in providers - record["name"] = next((stubs[(provider, nid)] for provider in providers if (provider, nid) in stubs), None) if kind == "stub" else None + record["name"] = next((stubs[(provider, resolved_nid)] for provider in providers if (provider, resolved_nid) in stubs), None) if kind == "stub" else None ordered = sorted(records.values(), key=lambda record: (record["class"], record["library"] or "", record.get("module") or "", record["nid"])) check_conservation(len(imports) + sum(len(symbols) for symbols, _ in (module_imports or {}).values()), ordered) missing = defaultdict(int) diff --git a/tools/tests/test_import_audit.py b/tools/tests/test_import_audit.py index 993cf3a107..9ad3683144 100644 --- a/tools/tests/test_import_audit.py +++ b/tools/tests/test_import_audit.py @@ -195,6 +195,53 @@ class ImportAuditTests(unittest.TestCase): self.assertTrue(records["missing_function"]["weak"]) self.assertIsNone(data["stops"][0]["nid"]) + def test_named_imports_fall_back_to_nid_exports(self): + work = self.work + (work.libs / "libc.prx").write_bytes(exporting("j4ViWNHEgww")) + registry, report = work.root / "homebrew.registry.json", work.root / "audit.json" + write_registry(registry, [("strlen", ""), ("strlen", ""), + ("missing_function", "", True)]) + code, _, err = work.run(registry, "--json", str(report)) + self.assertEqual((code, err), (1, "")) + data = json.loads(report.read_text(encoding="utf-8")) + records = {record["nid"]: record for record in data["imports"]} + self.assertEqual(data["references"], 3) + self.assertEqual(records["strlen"]["references"], 2) + self.assertEqual(records["strlen"]["class"], "implemented") + self.assertEqual(records["strlen"]["providers"], ["libc.prx"]) + self.assertEqual(records["missing_function"]["class"], "absent") + self.assertTrue(records["missing_function"]["weak"]) + self.assertIsNone(data["stops"][0]["nid"]) + + def test_named_nid_fallback_preserves_stub_and_library_classification(self): + nid = nid_names.compute_nid(MESSAGE_NAME) + imports = [(MESSAGE_NAME, "libSceOther.prx", False)] + records, missing = import_audit.audit(imports, {nid: ["libSceMsgDialog.prx"]}, + {("libSceMsgDialog.prx", nid): MESSAGE_NAME}, + {"libSceMsgDialog.prx", "libSceOther.prx"}, set()) + self.assertEqual(missing, {}) + self.assertEqual(records[0]["class"], "stub") + self.assertEqual(records[0]["nid"], MESSAGE_NAME) + self.assertEqual(records[0]["name"], MESSAGE_NAME) + self.assertTrue(records[0]["library_mismatch"]) + + def test_literal_exports_take_precedence_over_nid_fallback(self): + imports = [("strlen", "", False)] + stubs = {("literal.prx", "strlen"): "strlen"} + exports = {"strlen": ["literal.prx"], "j4ViWNHEgww": ["libc.prx"]} + records, _ = import_audit.audit(imports, exports, stubs, set(exports), set()) + self.assertEqual(records[0]["class"], "stub") + self.assertEqual(records[0]["providers"], ["literal.prx"]) + module_imports = {"guest.so": (["strlen"], [])} + records, _ = import_audit.audit([], {"j4ViWNHEgww": ["libc.prx"]}, {}, + {"libc.prx"}, set(module_imports), module_imports, + {"strlen": ["other.so"]}) + self.assertEqual(records[0]["providers"], ["other.so"]) + records, _ = import_audit.audit([], {"j4ViWNHEgww": ["libc.prx"]}, {}, + {"libc.prx"}, set(module_imports), module_imports) + self.assertEqual(records[0]["class"], "implemented") + self.assertEqual(records[0]["providers"], ["libc.prx"]) + def test_empty_and_control_character_imports_remain_invalid(self): registry = self.work.root / "homebrew.registry.json" for name in ("", "#A#B", "bad\x00name", "bad\nname", "bad\x7fname", "bad name"): From 2132abc6988ab4500a58bc612c977a87038ab394 Mon Sep 17 00:00:00 2001 From: billybobpicklio Date: Fri, 16 Oct 2026 09:17:24 -0400 Subject: [PATCH 029/103] test(agc): register the draw descriptor recorder tests with CTest agc_driver_recorder_tests --draw-descriptors-only (drawDescriptorWriteTests and drawSnapshotTableSetTests) had no CTest entry, so CI never ran it. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 051af9c6e5..7c791571a1 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1166,8 +1166,10 @@ if(BUILD_TESTING) set_tests_properties(agc_shadowed_key_stores PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) add_test(NAME agc_sample_dumps COMMAND agc_driver_recorder_tests --sample-dumps-only) set_tests_properties(agc_sample_dumps PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) + add_test(NAME agc_draw_descriptors COMMAND agc_driver_recorder_tests --draw-descriptors-only) + set_tests_properties(agc_draw_descriptors PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60) if(WIN32) - set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps agc_draw_descriptors PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() From d707066f656c58a3fed7d0e82aca4f1c52eb00d9 Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:07:46 -0400 Subject: [PATCH 030/103] test(agc): make the null-image fixture module and layout agree The stencil null feedback fixture splices a storage image array into the fragment module by hand. spirv-val rejects the result with "Decorate is in an invalid layout section": the decorations were inserted in front of the first OpTypeVoid, which the recompiler emits after its Input and Output variables. The binding also used set 1 with the vertex-group number of a different heap kind (the 1D array one, not the 2D image the module declares), so the set layout the driver builds from the binding metadata (binding / 65 picks the set) put it in set 0 while the module declared set 1. Insert the decorations after the existing annotations, declare the image array type the recompiler emits for a float 2D storage heap, list the variable in the SPIR-V 1.4 entry point interface like the recompiler does, and number the binding as the recompiler does for the fragment stage, in the descriptor set the target uses. When built with ANYPS5_ENABLE_SPIRV_TOOLS the fixture validates its module against the Vulkan 1.1 environment. Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 4 + .../tests/execution/StencilNullFeedback.cpp | 119 ++++++++++++------ 2 files changed, 87 insertions(+), 36 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 6bdee1b0e9..0e16fc0c90 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1379,6 +1379,10 @@ foreach(agcExecutionTestName IN LISTS ANYPS5_AGC_EXECUTION_TESTS) message(FATAL_ERROR "ANYPS5_AGC_EXECUTION_TESTS names ${agcExecutionTestName}, which has no source in tests/execution") endif() endforeach() +if(ANYPS5_ENABLE_SPIRV_TOOLS) + target_compile_definitions(agc_driver_stencil_null_feedback_tests PRIVATE ANYPS5_ENABLE_SPIRV_TOOLS=1) + target_link_libraries(agc_driver_stencil_null_feedback_tests PRIVATE SPIRV-Tools-static) +endif() target_link_libraries(agc_driver_gpu_map_on_fiber_tests PRIVATE libSceFiber) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp index befa0444a8..389c56e3f4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StencilNullFeedback.cpp @@ -13,6 +13,9 @@ #include #include #include +#if ANYPS5_ENABLE_SPIRV_TOOLS +#include +#endif namespace AgcDriver { @@ -89,13 +92,71 @@ ShaderRecompiler::ShaderPixelStageInfo PixelInfo(std::uint32_t shaderControl) { return AgcDriver::Graphics::DecodePixelStageInfo(context, {0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u}); } -void AttachNullImages(ShaderRecompiler::RecompileResult& result, bool depthAlias = false) { +constexpr std::uint32_t ImageElements = 2u; +constexpr std::uint32_t StorageFloat2DHeap = ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding + 2u; + +void Emit(std::vector& output, spv::Op opcode, std::initializer_list operands) { + output.push_back((static_cast(operands.size() + 1u) << spv::WordCountShift) | static_cast(opcode)); + output.insert(output.end(), operands.begin(), operands.end()); +} + +bool StartsDeclarations(spv::Op opcode) { + return opcode == spv::OpUndef || opcode == spv::OpVariable || (opcode >= spv::OpTypeVoid && opcode <= spv::OpTypeForwardPointer) || (opcode >= spv::OpConstantTrue && opcode <= spv::OpSpecConstantOp); +} + +void DeclareImageHeap(std::vector& module, std::uint32_t set, std::uint32_t binding, std::uint32_t elements) { + Require(module.size() >= 5u && module[0] == spv::MagicNumber, "the null-image fixture module is not SPIR-V"); + std::uint32_t scalar = 0u, floating = 0u; + std::size_t entry = 0u, declarations = 0u, functions = 0u; + for (std::size_t offset = 5u; offset < module.size();) { + const auto count = module[offset] >> spv::WordCountShift; + const auto opcode = static_cast(module[offset] & spv::OpCodeMask); + Require(count != 0u && count <= module.size() - offset, "the null-image fixture module has a truncated instruction"); + if (opcode == spv::OpFunction) { + functions = offset; + break; + } + if (opcode == spv::OpEntryPoint) { + Require(entry == 0u, "the null-image fixture module has several entry points"); + entry = offset; + } + if (declarations == 0u && StartsDeclarations(opcode)) declarations = offset; + if (opcode == spv::OpTypeInt && module[offset + 2u] == 32u && module[offset + 3u] == 0u) scalar = module[offset + 1u]; + if (opcode == spv::OpTypeFloat && module[offset + 2u] == 32u) floating = module[offset + 1u]; + offset += count; + } + Require(entry != 0u && declarations != 0u && functions != 0u && floating != 0u, "the null-image fixture module has no entry point, declarations or scalar shader types"); + auto id = module[3]; + const bool declareScalar = scalar == 0u; + if (declareScalar) scalar = id++; + const auto length = id++, image = id++, array = id++, pointer = id++, variable = id++; + std::vector annotations; + Emit(annotations, spv::OpDecorate, {variable, spv::DecorationDescriptorSet, set}); + Emit(annotations, spv::OpDecorate, {variable, spv::DecorationBinding, binding}); + std::vector types; + if (declareScalar) Emit(types, spv::OpTypeInt, {scalar, 32u, 0u}); + Emit(types, spv::OpConstant, {scalar, length, elements}); + Emit(types, spv::OpTypeImage, {image, floating, spv::Dim2D, 0u, 0u, 0u, 2u, spv::ImageFormatUnknown}); + Emit(types, spv::OpTypeArray, {array, image, length}); + Emit(types, spv::OpTypePointer, {pointer, spv::StorageClassUniformConstant, array}); + Emit(types, spv::OpVariable, {pointer, variable, spv::StorageClassUniformConstant}); + module.insert(module.begin() + static_cast(functions), types.begin(), types.end()); + module.insert(module.begin() + static_cast(declarations), annotations.begin(), annotations.end()); + if (module[1] >= 0x00010400u) { + module.insert(module.begin() + static_cast(entry + (module[entry] >> spv::WordCountShift)), variable); + module[entry] += 1u << spv::WordCountShift; + } + module[3] = id; +} + +void AttachNullImages(ShaderRecompiler::RecompileResult& result, const ShaderRecompiler::SpirvTarget& target, bool depthAlias = false) { namespace Runtime = ShaderRecompiler::RuntimeAbi; const auto address = static_cast(reinterpret_cast(depthAlias ? DepthMemory.data() : NullBindingTexture.data())); const auto format = depthAlias ? 22u : 56u; const std::array words{static_cast(address >> 8u), static_cast(address >> 40u) | (format << 20u) | (((Width - 1u) & 3u) << 30u), ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), 0xfacu | (9u << 28u), 0u, 0u, 0u, 0u}; - constexpr auto binding = Runtime::FirstStorageImageBinding + 1u; - ShaderRecompiler::DescriptorBinding images{ShaderRecompiler::DescriptorKind::StorageImage, ShaderRecompiler::DescriptorRole::GuestImages, Runtime::FragmentDescriptorSet, binding, 2u, std::vector(16u)}; + const auto set = target.registrationModules ? Runtime::FragmentDescriptorSet : Runtime::DescriptorSet; + const auto binding = Runtime::BindingNumber(Runtime::Stage::Fragment, static_cast(StorageFloat2DHeap)); + ShaderRecompiler::DescriptorBinding images{ShaderRecompiler::DescriptorKind::StorageImage, ShaderRecompiler::DescriptorRole::GuestImages, set, binding, ImageElements, std::vector(ImageElements * words.size())}; std::copy(words.begin(), words.end(), images.guestDescriptor.begin()); images.imageShape = ShaderRecompiler::DescriptorImageShape::Image2D; images.imageWritten = {false, false}; @@ -103,41 +164,22 @@ void AttachNullImages(ShaderRecompiler::RecompileResult& result, bool depthAlias images.imageAtomic64 = {false, false}; result.bindings.push_back(std::move(images)); auto module = result.spirv.Copy(); - auto id = module[3]; - std::uint32_t scalar = 0u, floating = 0u; - std::size_t types = 0u, functions = 0u; - for (std::size_t offset = 5u; offset < module.size(); offset += module[offset] >> 16u) { - const auto opcode = module[offset] & 0xffffu; - if (types == 0u && opcode == spv::OpTypeVoid) types = offset; - if (opcode == spv::OpTypeInt && module[offset + 2u] == 32u && module[offset + 3u] == 0u) scalar = module[offset + 1u]; - if (opcode == spv::OpTypeFloat && module[offset + 2u] == 32u) floating = module[offset + 1u]; - if (opcode == spv::OpFunction) { functions = offset; break; } - } - Require(types != 0u && functions != 0u && floating != 0u, "the null-image fixture has no scalar shader types"); - const bool declareScalar = scalar == 0u; - if (declareScalar) scalar = id++; - const auto count = id++, image = id++, array = id++, pointer = id++, variable = id++; - const auto emit = [](std::vector& output, spv::Op opcode, std::initializer_list operands) { - output.push_back((static_cast(operands.size() + 1u) << 16u) | static_cast(opcode)); - output.insert(output.end(), operands.begin(), operands.end()); - }; - std::vector annotations; - emit(annotations, spv::OpDecorate, {variable, spv::DecorationDescriptorSet, Runtime::FragmentDescriptorSet}); - emit(annotations, spv::OpDecorate, {variable, spv::DecorationBinding, binding}); - emit(annotations, spv::OpDecorate, {variable, spv::DecorationNonWritable}); - std::vector declarations; - if (declareScalar) emit(declarations, spv::OpTypeInt, {scalar, 32u, 0u}); - emit(declarations, spv::OpConstant, {scalar, count, 2u}); - emit(declarations, spv::OpTypeImage, {image, floating, spv::Dim2D, 0u, 0u, 0u, 2u, depthAlias ? spv::ImageFormatR32f : spv::ImageFormatRgba8}); - emit(declarations, spv::OpTypeArray, {array, image, count}); - emit(declarations, spv::OpTypePointer, {pointer, spv::StorageClassUniformConstant, array}); - emit(declarations, spv::OpVariable, {pointer, variable, spv::StorageClassUniformConstant}); - module.insert(module.begin() + functions, declarations.begin(), declarations.end()); - module.insert(module.begin() + types, annotations.begin(), annotations.end()); - module[3] = id; + DeclareImageHeap(module, set, binding, ImageElements); result.spirv = std::move(module); } +#if ANYPS5_ENABLE_SPIRV_TOOLS +void RequireValidModule(const ShaderRecompiler::RecompileResult& result, const ShaderRecompiler::SpirvTarget& target) { + std::string diagnostics; + spvtools::SpirvTools validator(target.spirvVersion >= 0x00010400u ? SPV_ENV_VULKAN_1_1_SPIRV_1_4 : SPV_ENV_VULKAN_1_1); + validator.SetMessageConsumer([&](spv_message_level_t, const char*, const spv_position_t&, const char* message) { + if (message != nullptr) diagnostics += std::string(message) + '\n'; + }); + const auto& words = result.spirv.Words(); + const bool valid = validator.Validate(words.data(), words.size()); + Require(valid, "the null-image fixture emitted invalid SPIR-V:\n" + diagnostics); +} +#endif void Draw(AgcDriver::VulkanDevice& device, std::span pixelCode, std::uint32_t shaderControl, const VkStencilOpState& stencil, std::uint32_t vertices = 3u, std::uint32_t instances = 1u, bool depthFail = false, const std::function& alter = {}, const std::function& alterDraw = {}, std::span> vertexData = Vertices, std::span vertexCode = VertexCode, std::span pixelData = {}, std::span pixelSrt = {}, bool nullImages = false, bool depthAlias = false) { Pixels.fill(Kept); @@ -171,7 +213,12 @@ void Draw(AgcDriver::VulkanDevice& device, std::span pixelC }; fragment.useCache = false; auto pixelResult = ShaderRecompiler::Recompile(fragment); - if (nullImages) AttachNullImages(pixelResult, depthAlias); + if (nullImages) { + AttachNullImages(pixelResult, target, depthAlias); +#if ANYPS5_ENABLE_SPIRV_TOOLS + RequireValidModule(pixelResult, target); +#endif + } if (!pixelResult.stencilReferences.empty()) { Require(pixelResult.stencilReferenceDword * sizeof(std::uint32_t) + sizeof(std::uint32_t) == pixelResult.pushConstants.size(), "dynamic reference is not reserved after the stage payload"); for (std::size_t cursor = 5; cursor < pixelResult.spirv.size();) { From 6dbbf4d7d9bb27f6f5cbbed9e24934b5d8ec5452 Mon Sep 17 00:00:00 2001 From: Mouad-Scriptz-0 Date: Sun, 11 Oct 2026 21:41:54 +0000 Subject: [PATCH 031/103] perf(recompiler): select guest memory loads instead of branching around them LDS, scratch and direct storage-buffer loads load element 0 when the lane is inactive or the address is out of bounds and select 0 as the result, instead of opening a selection per guard. Stores, atomics, formatted and physical-address accesses keep their branches. A branch guard cost four blocks per lane half, and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. Co-Authored-By: Claude Fable 5.1 --- .../SpirvBackend/SpirvEmitterHelpers.hpp | 1 + .../src/SpirvMemory/SpirvBufferAccess.cpp | 4 ++ .../SpirvMemory/SpirvMemoryInstructions.cpp | 55 +++++++++---------- .../SpirvBackend/src/SpirvRuntimeBuffer.cpp | 43 +++++++-------- docs/dev/debt/shader.md | 1 + 5 files changed, 52 insertions(+), 52 deletions(-) diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index dcd4b95b44..cfd8205030 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -117,6 +117,7 @@ std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResou std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); bool InFunctionStorage(const SpirvEmitterState& state, ResourceKind kind); std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); +std::uint32_t EmitValueOrZero(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t value); std::uint32_t EmitStorageBufferElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index, std::uint32_t pointerType); std::uint32_t EmitTBufferBitcastU32ToI32(SpirvEmitterState& state, std::uint32_t value); bool IsSignedFormatComponent(SpirvFormatComponentType type); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp index 9312b24e49..c8716c472a 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp @@ -173,6 +173,10 @@ bool InFunctionStorage(const SpirvEmitterState& state, ResourceKind kind) { return kind == ResourceKind::Lds && !LdsInDeviceMemory(state) && ShaderWorkgroupInput(state) == nullptr; } +std::uint32_t EmitValueOrZero(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t value) { + return Select(state, TypeU32(state), condition, value, ConstantU32(state, 0u)); +} + std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index) { if (access.kind == ResourceKind::Lds && LdsInDeviceMemory(state)) { return EmitStorageBufferElementPointer(state, access, EmitAddU32(state, state.ldsBufferBase, index), TypeStorageBufferElementPointer(state)); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index e152d36092..f8bd930b18 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -134,36 +134,33 @@ std::uint32_t LoadSubwordInBounds(SpirvValueEmitContext& ctx, const MemoryResour return Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), left, ConstantU32(state, 32u - bits)); } -std::uint32_t LoadWordPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource) { +std::uint32_t LoadEnable(SpirvEmitterState& state, const MemoryResourceAccess& resource, std::uint32_t index, std::uint32_t active) { + const auto inBounds = EmitMemoryElementInBounds(state, resource, index); + return active == 0u ? inBounds : Binary(state, spv::OpLogicalAnd, TypeBool(state), active, inBounds); +} + +std::uint32_t LoadWordPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, std::uint32_t active) { auto& state = ctx.state; const auto index = EmitMemoryElementIndex(state, resource, DwordIndex(ctx, inst, mem)); - return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&]() { - return LoadWordInBounds(ctx, resource, index); - }); + const auto enable = LoadEnable(state, resource, index, active); + return EmitValueOrZero(state, enable, LoadWordInBounds(ctx, resource, EmitValueOrZero(state, enable, index))); } std::uint32_t LoadWord(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { - return EmitValueOrZeroIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - return LoadWordPrepared(ctx, inst, mem, resource); - }); + return LoadWordPrepared(ctx, inst, mem, PrepareMemoryResourceAccess(ctx.state, mem), ActiveArgument(ctx, inst)); } -std::uint32_t LoadSubwordPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, std::uint32_t bits, bool signExtend) { +std::uint32_t LoadSubwordPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, std::uint32_t bits, bool signExtend, std::uint32_t active) { auto& state = ctx.state; const auto address = ByteAddress(ctx, inst, mem); const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); const auto index = EmitMemoryElementIndex(state, resource, rawIndex); - return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&]() { - return LoadSubwordInBounds(ctx, resource, address, index, bits, signExtend); - }); + const auto enable = LoadEnable(state, resource, index, active); + return EmitValueOrZero(state, enable, LoadSubwordInBounds(ctx, resource, address, EmitValueOrZero(state, enable, index), bits, signExtend)); } std::uint32_t LoadSubword(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t bits) { - return EmitValueOrZeroIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - return LoadSubwordPrepared(ctx, inst, mem, resource, bits, false); - }); + return LoadSubwordPrepared(ctx, inst, mem, PrepareMemoryResourceAccess(ctx.state, mem), bits, false, ActiveArgument(ctx, inst)); } std::uint32_t ConstantDeviceAddress(SpirvEmitterState& state, std::uint64_t value) { @@ -368,20 +365,18 @@ std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t comp std::uint32_t LoadWideShared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { auto& state = ctx.state; - return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { - const auto resource = PrepareMemoryResourceAccess(state, mem); - const auto base = ByteAddress(ctx, inst, mem); - std::array values{}; - for (std::uint32_t component = 0; component < components; component++) { - const auto address = component == 0u ? base : Binary(state, spv::OpIAdd, TypeU32(state), base, ConstantU32(state, component * 4u)); - const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); - const auto index = EmitMemoryElementIndex(state, resource, rawIndex); - values.at(component) = EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&]() { - return LoadWordInBounds(ctx, resource, index); - }); - } - return ConstructU32Composite(state, components, values); - }); + const auto active = ActiveArgument(ctx, inst); + const auto resource = PrepareMemoryResourceAccess(state, mem); + const auto base = ByteAddress(ctx, inst, mem); + std::array values{}; + for (std::uint32_t component = 0; component < components; component++) { + const auto address = component == 0u ? base : Binary(state, spv::OpIAdd, TypeU32(state), base, ConstantU32(state, component * 4u)); + const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); + const auto index = EmitMemoryElementIndex(state, resource, rawIndex); + const auto enable = LoadEnable(state, resource, index, active); + values.at(component) = EmitValueOrZero(state, enable, LoadWordInBounds(ctx, resource, EmitValueOrZero(state, enable, index))); + } + return ConstructU32Composite(state, components, values); } void StoreWideShared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index ec559b5340..f31fa225a0 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -169,26 +169,24 @@ std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruct const auto unaligned = nonzero(state, misalignment); const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); const auto last = Select(state, TypeU32(state), unaligned, next, index); - return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { - const auto load = [&](std::uint32_t word) { - const auto value = state.module.AllocateId(); - const auto pointer = EmitMemoryElementPointer(state, access, word); - if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, access.memoryAccess); - else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); - return value; - }; - const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), misalignment, ConstantU32(state, 3u)); - const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(index), shift); - const auto high = EmitValueOrZeroIfCondition(state, unaligned, [&] { - return Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(next), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift)); - }); - auto value = EmitOrU32(state, low, high); - if (bits != 32u) { - const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, address, 3u), ConstantU32(state, 3u)); - value = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, subwordShift); - } - return EmitAndConstant(state, value, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); - }); + const auto inBounds = EmitMemoryElementInBounds(state, access, last); + const auto load = [&](std::uint32_t word) { + const auto value = state.module.AllocateId(); + const auto pointer = EmitMemoryElementPointer(state, access, word); + if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, access.memoryAccess); + else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; + }; + const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), misalignment, ConstantU32(state, 3u)); + const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(EmitValueOrZero(state, inBounds, index)), shift); + const auto highWord = load(EmitValueOrZero(state, both(state, inBounds, unaligned), next)); + const auto high = EmitValueOrZero(state, unaligned, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), highWord, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift))); + auto value = EmitOrU32(state, low, high); + if (bits != 32u) { + const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, address, 3u), ConstantU32(state, 3u)); + value = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, subwordShift); + } + return EmitValueOrZero(state, inBounds, EmitAndConstant(state, value, bits == 32u ? 0xffffffffu : (1u << bits) - 1u)); } std::uint32_t byteBuffers(SpirvEmitterState& state) { @@ -726,7 +724,8 @@ std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValu std::array values{}; for (std::uint32_t component = 0; component < components; ++component) { const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u); - values[component] = EmitValueOrZeroIfCondition(state, access.inBounds, [&] { return readBuffer(context, instruction, access.guest, memory.dataBits); }); + values[component] = memory.gpuDescriptor ? EmitValueOrZeroIfCondition(state, access.inBounds, [&] { return readBuffer(context, instruction, access.guest, memory.dataBits); }) + : EmitValueOrZero(state, access.inBounds, readBuffer(context, instruction, access.guest, memory.dataBits)); } return composite(state, components, values); }; @@ -776,7 +775,7 @@ std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const const auto byte = EmitAndConstant(state, address, ~3u); const auto offset = Unary(state, spv::OpUConvert, u64, byte); const auto inBounds = both(state, noCarry, both(state, resource.valid, Binary(state, spv::OpULessThanEqual, TypeBool(state), Binary(state, spv::OpIAdd, u64, offset, BdaConstant(state, 4u)), size))); - if (!context.Memory(instruction).gpuDescriptor) return EmitValueOrZeroIfCondition(state, inBounds, [&] { return readBuffer(context, instruction, byte, 32u); }); + if (!context.Memory(instruction).gpuDescriptor) return EmitValueOrZero(state, inBounds, readBuffer(context, instruction, byte, 32u)); return EmitValueOrZeroIfCondition(state, inBounds, [&] { return readBuffer(context, instruction, AddBdaAddress(context, instruction, resource.base, offset, false), 32u); }); } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index ec051f71ed..427301445d 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -147,3 +147,4 @@ - [Flag structurization](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a graph that every structurizer attempt refuses, irreducible and non-terminating graphs included, is rebuilt with one reach flag per block and one again flag per loop (`structurizeWithFlags`): each block keeps one copy and runs behind a guard on its flag, and each strongly connected component becomes a loop whose single latch repeats it while a back edge set its flag. The output has about 8 times the input's blocks and one route variable per block, and ends in an empty Return that never runs. Only blocks that start the top-level order unconditionally run unguarded, so resource planning also sees the static paths that skip a guarded block and can plan extra descriptor candidates for them. The attempts still run first. Shared merge splitting refuses an attempt once the graph has grown past 16 times its block count at the start of splitting (at least 64 blocks); succeeding attempts grew at most 10.6 times on random graphs of up to 128 blocks, and an attempt that would succeed only beyond the budget leaves the graph to a later attempt or to this construction. Splitting that adds one block per round stays within that budget until its 4 rounds per block run out, so the attempts can still take more than 10 s (9 of 2,000 random graphs of up to 128 blocks, 63 s at most). `APS5_FORCE_STRUCTURIZER_FALLBACK=1` builds every graph this way, for tests - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) use only the base address, stride and record count for resources accessed exclusively by scalar loads; their fourth word is normalized to zero, following AMD RDNA2 ISA section 7.2.2. Vector and mixed accesses retain descriptor validation. Native Vulkan readbacks cover ignored fourth-word fields on Windows/NVIDIA; their console behavior has not been measured. - [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - `APS5_SPANNED_WAVE=1`, off by default, runs a single-wave wave64 compute program at one lane per invocation on a 32-lane host subgroup and exchanges ballots, shuffles and reductions between the two host subgroups through workgroup memory and barriers. The driver turns it on only where compute subgroups can be pinned at 32 full lanes (`subgroupSizeControl` and `computeFullSubgroups`), passes it as `SpirvTarget::spannedWaves` (part of the cache key and the serialized request), and creates the pipelines of spanned programs with `REQUIRE_FULL_SUBGROUPS`, since the lane number `SubgroupId * 32 + SubgroupLocalInvocationId` needs two full subgroups. Those barriers are correct only where every invocation of the workgroup reaches them, which holds for the guest's wave-uniform control flow. The emulated 64-bit LDS atomics ([LockedLdsUpdate](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp)) run only on the lanes the exec mask keeps, so their lock loop ballots within its own host subgroup, as its elect already does. Any other cross-lane exchange emitted inside per-lane control flow would hang a spanned wave in the same way, and the emitter does not reject one: there is no uniformity check. The only check so far is manual: disassemble the module (`spirv-dis`), list the selection and loop constructs that enclose each `OpControlBarrier`, and read the condition of each enclosing selection. In Ghost of Yōtei's program 0x80003af600 (PPSA26344), 108 of 111 barriers sat only in loops on wave-uniform conditions and 3 inside per-invocation conditionals, all in the lock loop. Other programs and titles have not been scanned. An earlier, cheaper ballot (one barrier instead of three) also lost the device for a reason that is not understood, which may be another barrier placement problem. Measured on an RTX 4090 (Windows): an offline replay of that program and two Yōtei runs that previously lost the device (NVIDIA TDR) now run past it, and [LdsMaskWaterfall.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp) runs both modes and covers a half-inactive wave. Stray (PPSA02101) on Intel ANV and a Demon's Souls (PPSA01342) request replay showed no fault, before this fix +- [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - LDS, scratch and direct storage-buffer loads (`LoadWord`, `LoadSubword`, `LoadWideShared`, `readBuffer` and `EmitRuntimeScalarBufferLoad` in [SpirvRuntimeBuffer.cpp](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp)) load element 0 when the lane is inactive or the address is out of bounds and select 0 as the result, instead of branching around the load; stores, atomics, formatted and physical-address (BDA) accesses still branch. A branch guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. The unconditional load depends on element 0 being addressable: SPIR-V function and workgroup arrays have at least one element, and a storage buffer with no elements reads out of bounds, which is defined only with `robustBufferAccess` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it). From ccaeeed416823445dba15212eee397662ac1e304 Mon Sep 17 00:00:00 2001 From: Mouad-Scriptz-0 Date: Sun, 11 Oct 2026 23:11:57 +0000 Subject: [PATCH 032/103] fix(recompiler): point selected guest loads at locations nothing writes A load selected instead of branched still accesses memory, so its fallback element must be one no write can race with: scratch keeps element 0, LDS in workgroup or function memory reads a sentinel dword appended after the program's LDS and lock, and storage buffers read element 0xfffffffe, which robustBufferAccess2 turns into a zero load without a memory access. The device enables robustBufferAccess2 when it has it and the target carries the flag; without it, and for LDS in device memory, buffer loads keep their selections. The aligned high word of a buffer read re-reads the low word's element instead of word 0. agc_guarded_memory_loads traces the emitted loads in both device modes. Co-Authored-By: Claude Fable 5.1 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 14 + .../Execution/src/VulkanDevice.cpp | 9 +- .../tests/GuardedMemoryLoads.cpp | 375 ++++++++++++++++++ .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 1 + .../libSceAgcDriver/tools/ShaderReplay.cpp | 8 +- core/shader/recompiler/CacheKey.hpp | 1 + core/shader/recompiler/Recompiler.cpp | 1 + core/shader/recompiler/Recompiler.hpp | 1 + .../include/SpirvBackend/SpirvEmitter.hpp | 1 + .../SpirvBackend/SpirvEmitterHelpers.hpp | 3 + .../SpirvBackend/SpirvEmitterState.hpp | 1 + .../SpirvBackend/src/SpirvEmitter.cpp | 1 + .../src/SpirvMemory/SpirvBufferAccess.cpp | 17 +- .../SpirvMemory/SpirvMemoryInstructions.cpp | 9 +- .../SpirvBackend/src/SpirvModuleEmitter.cpp | 2 +- .../SpirvBackend/src/SpirvRuntimeBuffer.cpp | 37 +- docs/dev/debt/shader.md | 1 + 17 files changed, 462 insertions(+), 20 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index ada29c686f..3eed40db9f 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -498,6 +498,20 @@ target_link_libraries(agc_vertex_input_tests PRIVATE shader_recompiler_agc_drive set_target_properties(agc_vertex_input_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_vertex_input_tests) +add_test_executable(agc_guarded_memory_loads_tests EXCLUDE_FROM_ALL tests/GuardedMemoryLoads.cpp) +target_link_libraries(agc_guarded_memory_loads_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_guarded_memory_loads_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_guarded_memory_loads_tests) +if(BUILD_TESTING) + set_target_properties(agc_guarded_memory_loads_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_guarded_memory_loads COMMAND agc_guarded_memory_loads_tests) + set_tests_properties(agc_guarded_memory_loads PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_guarded_memory_loads PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_shared_write_tests EXCLUDE_FROM_ALL tests/SharedWrites.cpp) target_include_directories(agc_shared_write_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel libc) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 44c1dce86b..95dbfcfced 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -301,6 +301,7 @@ struct VulkanDevice::State { bool float16DenormPreserve = false; bool demoteToHelperInvocation = false; bool earlyFragmentSampleMaskTestBeforeSampleCounting = false; + bool robustBufferAccess2 = false; // VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in // graphics stages, and compute workgroups wider than a wave). bool descriptorIndexing = false; @@ -1832,9 +1833,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : VkPhysicalDeviceFeatures2 queried{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness2}; state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &queried); } - if (robustness2.nullDescriptor == VK_TRUE) { + state->robustBufferAccess2 = robustness2.robustBufferAccess2 == VK_TRUE && enabled.robustBufferAccess == VK_TRUE; + if (robustness2.nullDescriptor == VK_TRUE || state->robustBufferAccess2) { + const bool nullDescriptor = robustness2.nullDescriptor == VK_TRUE; robustness2 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT}; - robustness2.nullDescriptor = VK_TRUE; + robustness2.nullDescriptor = nullDescriptor ? VK_TRUE : VK_FALSE; + robustness2.robustBufferAccess2 = state->robustBufferAccess2 ? VK_TRUE : VK_FALSE; robustness2.pNext = bdaFeatures.pNext; bdaFeatures.pNext = &robustness2; deviceExtensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); @@ -3489,6 +3493,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const { target.spannedWaves = state->spannedWaves; target.fixedPushSlots = state->graphicsPipelineLibrary; target.registrationModules = state->graphicsPipelineLibrary; + target.robustBufferAccess2 = state->robustBufferAccess2; target.srgbDecodeFormats = state->srgbDecodeFormats; target.vertexSubgroups = (state->subgroup.supportedStages & VK_SHADER_STAGE_VERTEX_BIT) != 0; if (state->meshShader) { diff --git a/core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp b/core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp new file mode 100644 index 0000000000..241430786c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp @@ -0,0 +1,375 @@ +#define SPV_ENABLE_UTILITY_CODE +#include +#undef SPV_ENABLE_UTILITY_CODE +#include "IntermediateRepresentation/IrProgram.hpp" +#include "Optimization/ShaderStageInputInfo.hpp" +#include "SpirvBackend/SpirvEmitter.hpp" +#include "PipelineSpecialization.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +struct LoadCase { + ResourceKind kind; + IrOpcode opcode; + IrType type; + std::uint32_t bits = 32; + std::uint32_t components = 1; +}; + +std::vector emitLoad(const LoadCase& test, bool active, std::uint32_t offset, std::uint32_t records, bool robust) { + IrProgram program; + program.SetWaveSize(32); + program.Resources().stage = IrShaderStage::Compute; + program.Resources().srtPlanComplete = true; + program.Resources().resourceTrackingComplete = true; + program.Metadata().shaderInfoComplete = true; + program.Metadata().bindingLayoutComplete = true; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder().push_back(&block); + program.Metadata().blockInfo.emplace_back(); + const auto constant = [&](std::uint32_t number) -> IrValue& { + auto& value = program.CreateValue(IrOpcode::Void, IrType::U32); + value.SetImmediateU32(number); + return value; + }; + const auto emit = [&](IrOpcode opcode, IrType type, std::initializer_list arguments) -> IrValue& { + auto& value = program.CreateValue(opcode, type); + for (auto* argument : arguments) value.AddArgument(argument); + block.AppendInstruction(&value); + return value; + }; + auto& enabled = program.CreateValue(IrOpcode::Void, IrType::Bool); + enabled.SetImmediateBool(active); + auto& zero = constant(0); + auto& address = constant(offset); + MemoryInfo memory; + memory.kind = test.kind; + memory.dataBits = test.bits; + memory.dataDwords = test.components; + memory.componentCount = test.components; + program.Resources().memoryInfo.push_back(memory); + if (test.kind == ResourceKind::Scratch) { + program.Info().scratchDwords = 16; + auto& handle = emit(IrOpcode::GetScratchResource, IrType::AddressResource, {}); + emit(test.opcode, test.type, {&handle, &address, &zero, &enabled}); + } else if (test.kind == ResourceKind::Buffer) { + program.Resources().memoryInfo.front().coherent = true; + program.Info().buffers.emplace_back().read = true; + program.Metadata().bindings.descriptors.push_back({DescriptorBindingKind::Buffers, {0}}); + program.Metadata().bindings.pushDataStartDword = 0; + program.Metadata().bindings.memoryOffsetCount = 1; + auto& size = constant(records); + auto& handle = emit(IrOpcode::GetBufferResource, IrType::BufferResource, {&zero, &zero, &size, &zero}); + emit(test.opcode, test.type, {&handle, &zero, &address, &zero, &enabled}); + } else { + if (test.kind == ResourceKind::Gds) program.Metadata().bindings.descriptors.push_back({DescriptorBindingKind::Gds, {0}}); + emit(test.opcode, test.type, {&address, &enabled}); + } + ShaderComputeInputInfo compute; + compute.threadsNum[0] = 32; + compute.threadsNum[1] = 1; + compute.threadsNum[2] = 1; + compute.ldsSizeDwords = 16; + compute.scratchSizeDwords = 16; + compute.hostSubgroupSize = 32; + compute.waveSize = 32; + ShaderStageInputInfo inputs; + inputs.compute = &compute; + CompiledBindingLayout bindings; + bindings.layout = program.Metadata().bindings; + SpirvTargetOptions target{}; + target.vulkanVersion = 0x00401000; + target.spirvVersion = 0x00010300; + target.subgroupSize = 32; + target.robustBufferAccess2 = robust; + return SpirvEmitter().Emit(program, inputs, bindings, target); +} + +class AccessTrace { +public: + AccessTrace(const std::vector& module, std::uint32_t misalignment = 0, bool valid = true) + : words(module), misalignment(misalignment), valid(valid) { + require(words.size() >= 5, "SPIR-V module header is missing"); + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> 16; + require(count != 0 && count <= words.size() - offset, "SPIR-V instruction is truncated"); + const auto instruction = std::span(words).subspan(offset, count); + const auto opcode = static_cast(instruction[0] & 0xffff); + bool result = false; + bool type = false; + spv::HasResultAndType(opcode, &result, &type); + if (result) definitions.emplace(instruction[type ? 2 : 1], instruction); + if (opcode == spv::OpName) names.emplace(instruction[1], std::string(reinterpret_cast(instruction.data() + 2))); + if (opcode == spv::OpDecorate && instruction[2] == spv::DecorationSpecId) specializations.emplace(instruction[1], instruction[3]); + if (opcode == spv::OpLabel) labels.emplace(instruction[1], offset); + if (opcode == spv::OpEntryPoint) entryFunction = instruction[2]; + offset += count; + } + } + + std::vector Run() { + std::size_t cursor = 0; + bool entry = false; + for (std::size_t offset = 5; offset < words.size(); offset += words[offset] >> 16) { + const auto opcode = static_cast(words[offset] & 0xffff); + if (opcode == spv::OpFunction) entry = words[offset + 2] == entryFunction; + if (entry && opcode == spv::OpLabel) { cursor = offset; break; } + } + require(cursor != 0, "SPIR-V entry block is missing"); + std::vector reads; + std::uint32_t previous = 0; + std::uint32_t current = 0; + for (std::size_t steps = 0; steps < 4096; ++steps) { + const auto instruction = std::span(words).subspan(cursor, words[cursor] >> 16); + const auto opcode = static_cast(instruction[0] & 0xffff); + if (opcode == spv::OpLabel) current = instruction[1]; + if (opcode == spv::OpPhi) { + bool found = false; + for (std::size_t index = 3; index + 1 < instruction.size(); index += 2) { + if (instruction[index + 1] == previous) { phiValues[instruction[2]] = instruction[index]; found = true; } + } + require(found, "SPIR-V phi has no incoming edge for the executed path"); + } + if (opcode == spv::OpLoad && isGuestPointer(instruction[3])) { + const auto pointer = definition(instruction[3]); + require((pointer[0] & 0xffff) == spv::OpAccessChain, "guest read is not an indexed access"); + reads.push_back(value(pointer.back())); + if (rootName(instruction[3]) == "scratch_dwords") require(pointerStorage(instruction[3]) == spv::StorageClassFunction, "scratch selection accessed non-private storage"); + if (rootName(instruction[3]).starts_with("buffers")) require(instruction.size() == 5 && (instruction[4] & spv::MemoryAccessVolatileMask), "coherent buffer read lost its volatile access"); + } + if (opcode == spv::OpBranch || opcode == spv::OpBranchConditional) { + const auto destination = opcode == spv::OpBranch ? instruction[1] : instruction[value(instruction[1]) ? 2 : 3]; + previous = current; + cursor = labels.at(destination); + continue; + } + if (opcode == spv::OpReturn) return reads; + require(opcode != spv::OpFunctionCall && opcode != spv::OpSwitch && opcode != spv::OpLoopMerge, "access fixture unexpectedly requires a call, switch or loop"); + cursor += instruction.size(); + require(cursor < words.size(), "SPIR-V execution fell outside the function"); + } + throw std::runtime_error("SPIR-V access trace exceeded its bounded instruction budget"); + } + +private: + std::span definition(std::uint32_t id) const { + const auto found = definitions.find(id); + require(found != definitions.end(), "SPIR-V access condition refers to an undefined value"); + return found->second; + } + + std::string rootName(std::uint32_t id) const { + const auto instruction = definition(id); + if ((instruction[0] & 0xffff) == spv::OpAccessChain) return rootName(instruction[3]); + const auto found = names.find(id); + return found == names.end() ? std::string() : found->second; + } + + std::uint32_t pointerStorage(std::uint32_t id) const { + const auto instruction = definition(id); + if ((instruction[0] & 0xffff) == spv::OpAccessChain) return pointerStorage(instruction[3]); + require((instruction[0] & 0xffff) == spv::OpVariable, "access pointer has no variable root"); + return instruction[3]; + } + + bool isGuestPointer(std::uint32_t id) const { + const auto name = rootName(id); + return name == "lds_dwords" || name == "gds" || name == "scratch_dwords" || name.starts_with("buffers"); + } + + std::uint32_t value(std::uint32_t id) const { + const auto instruction = definition(id); + const auto opcode = static_cast(instruction[0] & 0xffff); + const auto argument = [&](std::size_t index) { return value(instruction[index]); }; + switch (opcode) { + case spv::OpConstantTrue: return 1; + case spv::OpConstantFalse: case spv::OpConstantNull: return 0; + case spv::OpConstant: return instruction[3]; + case spv::OpSpecConstant: { + const auto spec = specializations.at(id); + if (spec == PipelineSpecialization::BufferBase) return 0; + if (spec == PipelineSpecialization::BufferBase + 1) return 0x30001000; + if (spec == PipelineSpecialization::BufferBase + 2) return valid; + if (spec == PipelineSpecialization::BufferBase + 3) return misalignment; + if (spec == PipelineSpecialization::PushDataOffset) return 0; + throw std::runtime_error("unexpected specialization in access fixture"); + } + case spv::OpSelect: return argument(3) ? argument(4) : argument(5); + case spv::OpPhi: return value(phiValues.at(id)); + case spv::OpIAdd: return argument(3) + argument(4); + case spv::OpISub: return argument(3) - argument(4); + case spv::OpIMul: return argument(3) * argument(4); + case spv::OpShiftLeftLogical: + require(argument(4) < 32, "undefined shift in access condition"); + return argument(3) << argument(4); + case spv::OpShiftRightLogical: + require(argument(4) < 32, "undefined shift in access condition"); + return argument(3) >> argument(4); + case spv::OpBitwiseAnd: return argument(3) & argument(4); + case spv::OpBitwiseOr: return argument(3) | argument(4); + case spv::OpBitFieldUExtract: { + const auto count = argument(5); + require(count <= 32 && argument(4) <= 32 - count, "invalid bitfield in access condition"); + return (argument(3) >> argument(4)) & (count == 32 ? 0xffffffffu : (1u << count) - 1); + } + case spv::OpIEqual: return argument(3) == argument(4); + case spv::OpINotEqual: return argument(3) != argument(4); + case spv::OpULessThan: return argument(3) < argument(4); + case spv::OpULessThanEqual: return argument(3) <= argument(4); + case spv::OpUGreaterThanEqual: return argument(3) >= argument(4); + case spv::OpLogicalAnd: return argument(3) && argument(4); + case spv::OpLogicalOr: return argument(3) || argument(4); + case spv::OpLogicalNot: return !argument(3); + case spv::OpArrayLength: + require(isGuestPointer(instruction[3]), "unknown array length in access fixture"); + return 16; + case spv::OpLoad: + require(!isGuestPointer(instruction[3]), "guest data unexpectedly controls a fixture access"); + require(rootName(instruction[3]) == "vsharp" || rootName(instruction[3]).find("subgroup") != std::string::npos, "unknown input to access condition"); + return 0; + default: throw std::runtime_error("unsupported opcode in access condition: " + std::to_string(opcode)); + } + } + + const std::vector& words; + std::uint32_t misalignment; + bool valid; + std::uint32_t entryFunction = 0; + std::map> definitions; + std::map names; + std::map specializations; + std::map labels; + std::map phiValues; +}; + +constexpr std::uint32_t Unreachable = 0xfffffffeu; + +std::uint32_t ldsLength(const std::vector& words) { + std::map> definitions; + std::uint32_t variable = 0; + for (std::size_t offset = 5; offset < words.size(); offset += words[offset] >> 16) { + const auto instruction = std::span(words).subspan(offset, words[offset] >> 16); + const auto opcode = static_cast(instruction[0] & 0xffff); + if (opcode == spv::OpName && std::string(reinterpret_cast(instruction.data() + 2)) == "lds_dwords") variable = instruction[1]; + bool result = false; + bool type = false; + spv::HasResultAndType(opcode, &result, &type); + if (result) definitions.emplace(instruction[type ? 2 : 1], instruction); + } + require(variable != 0, "LDS array is missing"); + const auto pointer = definitions.at(definitions.at(variable)[1]); + const auto array = definitions.at(pointer[3]); + require((array[0] & 0xffff) == spv::OpTypeArray, "LDS variable is not an array"); + return definitions.at(array[3])[3]; +} + +void expectReads(const LoadCase& test, bool robust, bool active, std::uint32_t offset, std::vector expected, std::uint32_t records = 64, std::uint32_t misalignment = 0, bool valid = true) { + const auto module = emitLoad(test, active, offset, records, robust); + const auto actual = AccessTrace(module, misalignment, valid).Run(); + if (actual != expected) { + std::string detail = std::string(robust ? "robust" : "plain") + " guest reads:"; + for (const auto index : actual) detail += " " + std::to_string(index); + detail += "; expected:"; + for (const auto index : expected) detail += " " + std::to_string(index); + throw std::runtime_error(detail); + } +} + +void ldsLoads(IrOpcode opcode, IrType type, std::uint32_t bits, std::uint32_t components) { + const LoadCase test{ResourceKind::Lds, opcode, type, bits, components}; + const auto length = ldsLength(emitLoad(test, true, 8, 64, false)); + const auto sentinel = length - 1u; + require(sentinel >= 16u, "LDS fixture is smaller than its probes"); + for (const bool robust : {false, true}) { + std::vector sentinels(components, sentinel); + expectReads(test, robust, false, 8, sentinels); + expectReads(test, robust, true, (sentinel + 5u) * 4u, sentinels); + std::vector indices; + for (std::uint32_t index = 0; index < components; ++index) indices.push_back(2 + index); + expectReads(test, robust, true, 8, indices); + std::vector edge{15}; + edge.insert(edge.end(), components - 1u, sentinel); + expectReads(test, robust, true, 60, edge); + } +} + +} + +int main() { + unsigned failures = 0; + const auto run = [&](std::string_view name, const auto& test) { + try { + test(); + std::cout << "guarded memory loads: " << name << " passed\n"; + } catch (const std::exception& error) { + ++failures; + std::cout << "guarded memory loads: " << name << ": " << error.what() << '\n'; + } + }; + run("LDS word reads its sentinel", [] { ldsLoads(IrOpcode::LoadSharedU32, IrType::U32, 32, 1); }); + run("LDS subword reads its sentinel", [] { + ldsLoads(IrOpcode::LoadSharedU8, IrType::U8, 8, 1); + ldsLoads(IrOpcode::LoadSharedU16, IrType::U16, 16, 1); + }); + run("LDS wide reads its sentinel", [] { ldsLoads(IrOpcode::LoadSharedU32x4, IrType::U32x4, 32, 4); }); + run("GDS word", [] { + const LoadCase test{ResourceKind::Gds, IrOpcode::LoadSharedU32, IrType::U32, 32, 1}; + expectReads(test, false, false, 8, {}); + expectReads(test, false, true, 64, {}); + expectReads(test, false, true, 8, {2}); + expectReads(test, true, false, 8, {Unreachable}); + expectReads(test, true, true, 64, {Unreachable}); + expectReads(test, true, true, 8, {2}); + expectReads(test, true, true, 60, {15}); + }); + run("coherent buffer bounds", [] { + const LoadCase test{ResourceKind::Buffer, IrOpcode::LoadBufferU32, IrType::U32}; + for (const bool robust : {false, true}) expectReads(test, robust, false, 8, {}); + expectReads(test, false, true, 64, {}, 128); + expectReads(test, false, true, 60, {}, 32); + expectReads(test, false, true, 8, {}, 64, 0, false); + expectReads(test, true, true, 64, {Unreachable, Unreachable}, 128); + expectReads(test, true, true, 60, {Unreachable, Unreachable}, 32); + expectReads(test, true, true, 8, {Unreachable, Unreachable}, 64, 0, false); + }); + run("coherent buffer high word", [] { + const LoadCase test{ResourceKind::Buffer, IrOpcode::LoadBufferU32, IrType::U32}; + expectReads(test, false, true, 8, {2}); + expectReads(test, false, true, 8, {2, 3}, 64, 1); + expectReads(test, false, true, 60, {}, 64, 1); + expectReads(test, true, true, 8, {2, 2}); + expectReads(test, true, true, 8, {2, 3}, 64, 1); + expectReads(test, true, true, 60, {Unreachable, Unreachable}, 64, 1); + }); + run("private scratch selection", [] { + const LoadCase test{ResourceKind::Scratch, IrOpcode::LoadAddressU32, IrType::U32}; + for (const bool robust : {false, true}) { + expectReads(test, robust, false, 8, {0}); + expectReads(test, robust, true, 64, {0}); + expectReads(test, robust, true, 8, {2}); + } + }); + if (failures != 0) { + std::cerr << "guarded memory loads: a guest load reached a location the guarded program could not\n"; + return 1; + } + std::cout << "Guarded memory load tests passed\n"; + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 2ac7728dfd..e1035df27d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -622,6 +622,7 @@ void verifyKeySensitivity() { changes("f64 denormal preservation", [](SampleRequest& sample) { sample.request.target.float64DenormPreserve = true; }); changes("f16 denormal preservation", [](SampleRequest& sample) { sample.request.target.float16DenormPreserve = true; }); changes("spanned waves", [](SampleRequest& sample) { sample.request.target.spannedWaves = true; }); + changes("robust buffer access 2", [](SampleRequest& sample) { sample.request.target.robustBufferAccess2 = true; }); changes("the workgroup size limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSize[2] = 128; }); changes("the invocation limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupInvocations = 512; }); changes("the shared memory limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSharedMemoryBytes = 32768; }); diff --git a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp index 2383684946..2e7d97c327 100644 --- a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp @@ -47,6 +47,7 @@ bool g_prepare = false; bool g_materialize = false; bool g_graph = false; bool g_maintenance8 = false; +bool g_robust2 = false; bool g_code = false; bool Replay(const char* path) { @@ -145,6 +146,7 @@ bool Replay(const char* path) { } try { if (g_maintenance8) request.request.target.nonConstantImageOffsets = true; + if (g_robust2) request.request.target.robustBufferAccess2 = true; const auto result = ShaderRecompiler::Recompile(request.request); std::printf(" recompiled: %zu SPIR-V words\n", result.spirv.size()); if (g_spirv) { @@ -182,7 +184,7 @@ bool Replay(const char* path) { int main(int argc, char** argv) { if (argc < 2) { - std::fprintf(stderr, "usage: agc_shader_replay [--dis] [--asm] [--cfg] [--mem] [--spv] [--prepare] [--materialize] [--code] [--maintenance8] ...\n the driver writes shader_
.req files when APS5_DUMP_SHADERS is set\n"); + std::fprintf(stderr, "usage: agc_shader_replay [--dis] [--asm] [--cfg] [--mem] [--spv] [--prepare] [--materialize] [--code] [--maintenance8] [--robust2] ...\n the driver writes shader_
.req files when APS5_DUMP_SHADERS is set\n"); return 2; } int failures = 0; @@ -203,6 +205,10 @@ int main(int argc, char** argv) { g_maintenance8 = true; continue; } + if (std::string(argv[i]) == "--robust2") { + g_robust2 = true; + continue; + } if (std::string(argv[i]) == "--mem") { g_memory = true; continue; diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 753a7afbdc..f501252286 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -262,6 +262,7 @@ private: append(key, value.vertexSubgroups); append(key, value.float16DenormPreserve); append(key, value.spannedWaves); + append(key, value.robustBufferAccess2); } }; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 9bf473cf58..d08bc8a148 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -531,6 +531,7 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) targetOptions.fragmentDescriptorSet = request.target.registrationModules; targetOptions.float16DenormPreserve = request.target.float16DenormPreserve; targetOptions.spannedWaves = request.target.spannedWaves; + targetOptions.robustBufferAccess2 = request.target.robustBufferAccess2; constexpr SpirvEmitter spirvEmitter; CompiledShaderArtifact result; diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 123c488f40..79cf3c339f 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -268,6 +268,7 @@ struct SpirvTarget { bool vertexSubgroups = true; bool float16DenormPreserve = false; bool spannedWaves = false; + bool robustBufferAccess2 = false; }; struct BindingLayout { diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp index e4a861594f..7f7ccbdec5 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp @@ -26,6 +26,7 @@ struct SpirvTargetOptions { bool fragmentDescriptorSet = false; bool float16DenormPreserve = false; bool spannedWaves = false; + bool robustBufferAccess2 = false; }; struct SpirvEmitInfo { diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index cfd8205030..d996faf30e 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -118,6 +118,9 @@ std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryRe bool InFunctionStorage(const SpirvEmitterState& state, ResourceKind kind); std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); std::uint32_t EmitValueOrZero(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t value); +std::uint32_t LdsSentinelIndex(const SpirvEmitterState& state); +bool LoadsSelectable(const SpirvEmitterState& state, const MemoryResourceAccess& access); +std::uint32_t GuardedElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t condition, std::uint32_t index); std::uint32_t EmitStorageBufferElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index, std::uint32_t pointerType); std::uint32_t EmitTBufferBitcastU32ToI32(SpirvEmitterState& state, std::uint32_t value); bool IsSignedFormatComponent(SpirvFormatComponentType type); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 5c89545900..4c54b41678 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -107,6 +107,7 @@ struct SpirvEmitterState { bool registrationModules = false; bool fragmentDescriptorSet = false; bool float16DenormPreserve = false; + bool robustBufferAccess2 = false; std::unordered_set sharedLaneValues; std::uint32_t laneHalf = 0; std::uint32_t spirvVersion = 0x00010300u; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 0d629adaca..f30656653d 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -260,6 +260,7 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.registrationModules = target.registrationModules; state.fragmentDescriptorSet = target.fragmentDescriptorSet; state.float16DenormPreserve = target.float16DenormPreserve; + state.robustBufferAccess2 = target.robustBufferAccess2; state.hostSubgroupSize = target.subgroupSize; if (HostSubgroupSingleLane(state)) { state.requirements.subgroupBallot = false; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp index c8716c472a..5fa98f5610 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp @@ -20,7 +20,7 @@ namespace { if (ShaderWorkgroupInput(state) == nullptr) { FailEmit("function LDS was not prepared before function emission"); } - state.ldsVariable = state.module.DefineGlobalVariable(TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsDwordCount(state) + (state.requirements.ldsLock ? 1u : 0u)), spv::StorageClassWorkgroup); + state.ldsVariable = state.module.DefineGlobalVariable(TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsSentinelIndex(state) + 1u), spv::StorageClassWorkgroup); state.module.AddName(state.ldsVariable, "lds_dwords"); } @@ -177,6 +177,21 @@ std::uint32_t EmitValueOrZero(SpirvEmitterState& state, std::uint32_t condition, return Select(state, TypeU32(state), condition, value, ConstantU32(state, 0u)); } +std::uint32_t LdsSentinelIndex(const SpirvEmitterState& state) { + return LdsDwordCount(state) + (state.requirements.ldsLock ? 1u : 0u); +} + +bool LoadsSelectable(const SpirvEmitterState& state, const MemoryResourceAccess& access) { + if (access.kind == ResourceKind::Scratch) return true; + if (access.kind == ResourceKind::Lds) return !LdsInDeviceMemory(state); + return state.robustBufferAccess2; +} + +std::uint32_t GuardedElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t condition, std::uint32_t index) { + const auto fallback = access.kind == ResourceKind::Scratch ? 0u : access.kind == ResourceKind::Lds ? LdsSentinelIndex(state) : 0xfffffffeu; + return Select(state, TypeU32(state), condition, index, ConstantU32(state, fallback)); +} + std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index) { if (access.kind == ResourceKind::Lds && LdsInDeviceMemory(state)) { return EmitStorageBufferElementPointer(state, access, EmitAddU32(state, state.ldsBufferBase, index), TypeStorageBufferElementPointer(state)); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index f8bd930b18..d561dea75c 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -143,7 +143,8 @@ std::uint32_t LoadWordPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, auto& state = ctx.state; const auto index = EmitMemoryElementIndex(state, resource, DwordIndex(ctx, inst, mem)); const auto enable = LoadEnable(state, resource, index, active); - return EmitValueOrZero(state, enable, LoadWordInBounds(ctx, resource, EmitValueOrZero(state, enable, index))); + if (!LoadsSelectable(state, resource)) return EmitValueOrZeroIfCondition(state, enable, [&] { return LoadWordInBounds(ctx, resource, index); }); + return EmitValueOrZero(state, enable, LoadWordInBounds(ctx, resource, GuardedElementIndex(state, resource, enable, index))); } std::uint32_t LoadWord(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { @@ -156,7 +157,8 @@ std::uint32_t LoadSubwordPrepared(SpirvValueEmitContext& ctx, const IrValue& ins const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); const auto index = EmitMemoryElementIndex(state, resource, rawIndex); const auto enable = LoadEnable(state, resource, index, active); - return EmitValueOrZero(state, enable, LoadSubwordInBounds(ctx, resource, address, EmitValueOrZero(state, enable, index), bits, signExtend)); + if (!LoadsSelectable(state, resource)) return EmitValueOrZeroIfCondition(state, enable, [&] { return LoadSubwordInBounds(ctx, resource, address, index, bits, signExtend); }); + return EmitValueOrZero(state, enable, LoadSubwordInBounds(ctx, resource, address, GuardedElementIndex(state, resource, enable, index), bits, signExtend)); } std::uint32_t LoadSubword(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t bits) { @@ -374,7 +376,8 @@ std::uint32_t LoadWideShared(SpirvValueEmitContext& ctx, const IrValue& inst, co const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); const auto index = EmitMemoryElementIndex(state, resource, rawIndex); const auto enable = LoadEnable(state, resource, index, active); - values.at(component) = EmitValueOrZero(state, enable, LoadWordInBounds(ctx, resource, EmitValueOrZero(state, enable, index))); + values.at(component) = LoadsSelectable(state, resource) ? EmitValueOrZero(state, enable, LoadWordInBounds(ctx, resource, GuardedElementIndex(state, resource, enable, index))) + : EmitValueOrZeroIfCondition(state, enable, [&] { return LoadWordInBounds(ctx, resource, index); }); } return ConstructU32Composite(state, components, values); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 1f76ed7314..cd1f8c8ddc 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -1146,7 +1146,7 @@ void EmitProgram(SpirvEmitterState& state) { state.module.AddFunction(spv::OpFunction, TypeVoid(state), state.meshGuestFunc != 0u ? state.meshGuestFunc : state.mainFunc, spv::FunctionControlMaskNone, TypeFunction(state)); EmitLabel(state, state.entryLabel); if (state.requirements.functionLds) { - state.module.AddFunction(spv::OpVariable, TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsDwordCount(state)), state.ldsVariable, spv::StorageClassFunction); + state.module.AddFunction(spv::OpVariable, TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsSentinelIndex(state) + 1u), state.ldsVariable, spv::StorageClassFunction); } if (state.requirements.functionScratch) { for (std::uint32_t half = 0; half < state.laneCount; half++) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index f31fa225a0..7820a803c3 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -160,7 +160,7 @@ std::uint32_t directWordIndex(SpirvEmitterState& state, const MemoryResourceAcce return EmitAddU32(state, relative, base); } -std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t bits) { +std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t bits, std::uint32_t enable = 0u) { if (context.Memory(instruction).gpuDescriptor) return EmitBdaRead(context, instruction, address, bits); auto& state = context.state; const auto access = directAccess(context, instruction, false, !context.Memory(instruction).coherent); @@ -169,7 +169,8 @@ std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruct const auto unaligned = nonzero(state, misalignment); const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); const auto last = Select(state, TypeU32(state), unaligned, next, index); - const auto inBounds = EmitMemoryElementInBounds(state, access, last); + const auto bounds = EmitMemoryElementInBounds(state, access, last); + const auto inBounds = enable != 0u ? both(state, enable, bounds) : bounds; const auto load = [&](std::uint32_t word) { const auto value = state.module.AllocateId(); const auto pointer = EmitMemoryElementPointer(state, access, word); @@ -178,15 +179,27 @@ std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruct return value; }; const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), misalignment, ConstantU32(state, 3u)); - const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(EmitValueOrZero(state, inBounds, index)), shift); - const auto highWord = load(EmitValueOrZero(state, both(state, inBounds, unaligned), next)); - const auto high = EmitValueOrZero(state, unaligned, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), highWord, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift))); - auto value = EmitOrU32(state, low, high); - if (bits != 32u) { - const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, address, 3u), ConstantU32(state, 3u)); - value = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, subwordShift); + const auto combine = [&](std::uint32_t low, std::uint32_t high) { + auto value = EmitOrU32(state, low, high); + if (bits != 32u) { + const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, address, 3u), ConstantU32(state, 3u)); + value = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, subwordShift); + } + return EmitAndConstant(state, value, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); + }; + const auto highShift = Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift); + if (!LoadsSelectable(state, access)) { + return EmitValueOrZeroIfCondition(state, inBounds, [&] { + const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(index), shift); + const auto high = EmitValueOrZeroIfCondition(state, unaligned, [&] { return Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(next), highShift); }); + return combine(low, high); + }); } - return EmitValueOrZero(state, inBounds, EmitAndConstant(state, value, bits == 32u ? 0xffffffffu : (1u << bits) - 1u)); + const auto lowIndex = GuardedElementIndex(state, access, inBounds, index); + const auto highIndex = Select(state, TypeU32(state), both(state, inBounds, unaligned), next, lowIndex); + const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(lowIndex), shift); + const auto high = EmitValueOrZero(state, unaligned, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(highIndex), highShift)); + return EmitValueOrZero(state, inBounds, combine(low, high)); } std::uint32_t byteBuffers(SpirvEmitterState& state) { @@ -725,7 +738,7 @@ std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValu for (std::uint32_t component = 0; component < components; ++component) { const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u); values[component] = memory.gpuDescriptor ? EmitValueOrZeroIfCondition(state, access.inBounds, [&] { return readBuffer(context, instruction, access.guest, memory.dataBits); }) - : EmitValueOrZero(state, access.inBounds, readBuffer(context, instruction, access.guest, memory.dataBits)); + : readBuffer(context, instruction, access.guest, memory.dataBits, access.inBounds); } return composite(state, components, values); }; @@ -775,7 +788,7 @@ std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const const auto byte = EmitAndConstant(state, address, ~3u); const auto offset = Unary(state, spv::OpUConvert, u64, byte); const auto inBounds = both(state, noCarry, both(state, resource.valid, Binary(state, spv::OpULessThanEqual, TypeBool(state), Binary(state, spv::OpIAdd, u64, offset, BdaConstant(state, 4u)), size))); - if (!context.Memory(instruction).gpuDescriptor) return EmitValueOrZero(state, inBounds, readBuffer(context, instruction, byte, 32u)); + if (!context.Memory(instruction).gpuDescriptor) return readBuffer(context, instruction, byte, 32u, inBounds); return EmitValueOrZeroIfCondition(state, inBounds, [&] { return readBuffer(context, instruction, AddBdaAddress(context, instruction, resource.base, offset, false), 32u); }); } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 427301445d..7b7e78f8b8 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -148,3 +148,4 @@ - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) use only the base address, stride and record count for resources accessed exclusively by scalar loads; their fourth word is normalized to zero, following AMD RDNA2 ISA section 7.2.2. Vector and mixed accesses retain descriptor validation. Native Vulkan readbacks cover ignored fourth-word fields on Windows/NVIDIA; their console behavior has not been measured. - [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - `APS5_SPANNED_WAVE=1`, off by default, runs a single-wave wave64 compute program at one lane per invocation on a 32-lane host subgroup and exchanges ballots, shuffles and reductions between the two host subgroups through workgroup memory and barriers. The driver turns it on only where compute subgroups can be pinned at 32 full lanes (`subgroupSizeControl` and `computeFullSubgroups`), passes it as `SpirvTarget::spannedWaves` (part of the cache key and the serialized request), and creates the pipelines of spanned programs with `REQUIRE_FULL_SUBGROUPS`, since the lane number `SubgroupId * 32 + SubgroupLocalInvocationId` needs two full subgroups. Those barriers are correct only where every invocation of the workgroup reaches them, which holds for the guest's wave-uniform control flow. The emulated 64-bit LDS atomics ([LockedLdsUpdate](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp)) run only on the lanes the exec mask keeps, so their lock loop ballots within its own host subgroup, as its elect already does. Any other cross-lane exchange emitted inside per-lane control flow would hang a spanned wave in the same way, and the emitter does not reject one: there is no uniformity check. The only check so far is manual: disassemble the module (`spirv-dis`), list the selection and loop constructs that enclose each `OpControlBarrier`, and read the condition of each enclosing selection. In Ghost of Yōtei's program 0x80003af600 (PPSA26344), 108 of 111 barriers sat only in loops on wave-uniform conditions and 3 inside per-invocation conditionals, all in the lock loop. Other programs and titles have not been scanned. An earlier, cheaper ballot (one barrier instead of three) also lost the device for a reason that is not understood, which may be another barrier placement problem. Measured on an RTX 4090 (Windows): an offline replay of that program and two Yōtei runs that previously lost the device (NVIDIA TDR) now run past it, and [LdsMaskWaterfall.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp) runs both modes and covers a half-inactive wave. Stray (PPSA02101) on Intel ANV and a Demon's Souls (PPSA01342) request replay showed no fault, before this fix - [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - LDS, scratch and direct storage-buffer loads (`LoadWord`, `LoadSubword`, `LoadWideShared`, `readBuffer` and `EmitRuntimeScalarBufferLoad` in [SpirvRuntimeBuffer.cpp](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp)) load element 0 when the lane is inactive or the address is out of bounds and select 0 as the result, instead of branching around the load; stores, atomics, formatted and physical-address (BDA) accesses still branch. A branch guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. The unconditional load depends on element 0 being addressable: SPIR-V function and workgroup arrays have at least one element, and a storage buffer with no elements reads out of bounds, which is defined only with `robustBufferAccess` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it). +- [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; LDS in workgroup or function memory reads a sentinel dword appended after the program's LDS (and after the lock dword), which no bounds-checked write reaches; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it), and for LDS kept in device memory, those loads keep their selections; stores, atomics, formatted and physical-address (BDA) accesses always do. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. From d914130cc3d1097a6cb88fd161b5bf15421d2912 Mon Sep 17 00:00:00 2001 From: jweidenh <38249793+jweidenh@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:22:14 -0400 Subject: [PATCH 033/103] test(recompiler): cover LDS sentinel budget boundaries --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 14 +++ .../tests/LdsSentinelBudget.cpp | 115 ++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 core/libs/prx/libSceAgcDriver/tests/LdsSentinelBudget.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 3eed40db9f..d1545095c4 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -512,6 +512,20 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_lds_sentinel_budget_tests EXCLUDE_FROM_ALL tests/LdsSentinelBudget.cpp) +target_link_libraries(agc_lds_sentinel_budget_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_lds_sentinel_budget_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_lds_sentinel_budget_tests) +if(BUILD_TESTING) + set_target_properties(agc_lds_sentinel_budget_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_lds_sentinel_budget COMMAND agc_lds_sentinel_budget_tests) + set_tests_properties(agc_lds_sentinel_budget PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_lds_sentinel_budget PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_shared_write_tests EXCLUDE_FROM_ALL tests/SharedWrites.cpp) target_include_directories(agc_shared_write_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel libc) diff --git a/core/libs/prx/libSceAgcDriver/tests/LdsSentinelBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/LdsSentinelBudget.cpp new file mode 100644 index 0000000000..0e54ad04ce --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/LdsSentinelBudget.cpp @@ -0,0 +1,115 @@ +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +std::uint64_t workgroupBytes(std::span words) { + std::map> types; + std::map constants; + std::vector pointers; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> 16u; + require(count != 0u && count <= words.size() - offset, "malformed SPIR-V instruction"); + const auto instruction = words.subspan(offset, count); + const auto opcode = static_cast(instruction[0] & 0xffffu); + if (opcode == spv::OpTypeInt || opcode == spv::OpTypeFloat || opcode == spv::OpTypeVector || opcode == spv::OpTypeArray || opcode == spv::OpTypeStruct || opcode == spv::OpTypePointer) types.emplace(instruction[1], instruction); + if (opcode == spv::OpConstant) constants.emplace(instruction[2], instruction[3]); + if (opcode == spv::OpVariable && instruction[3] == spv::StorageClassWorkgroup) pointers.push_back(instruction[1]); + offset += count; + } + const std::function size = [&](std::uint32_t id) -> std::uint64_t { + const auto type = types.at(id); + switch (static_cast(type[0] & 0xffffu)) { + case spv::OpTypeInt: case spv::OpTypeFloat: return type[2] / 8u; + case spv::OpTypeVector: return size(type[2]) * type[3]; + case spv::OpTypeArray: return size(type[2]) * constants.at(type[3]); + case spv::OpTypeStruct: { + std::uint64_t bytes = 0; + for (const auto member : type.subspan(2)) bytes += size(member); + return bytes; + } + default: throw std::runtime_error("unexpected Workgroup variable type"); + } + }; + std::uint64_t bytes = 0; + for (const auto pointer : pointers) { + const auto type = types.at(pointer); + require((type[0] & 0xffffu) == spv::OpTypePointer, "Workgroup variable is not a pointer"); + bytes += size(type[3]); + } + return bytes; +} + +RecompileResult compile(std::uint32_t waveSize, std::uint32_t limit) { + static constexpr std::array code{ + 0x7e020280u, 0x7e0402c1u, 0x7e060280u, 0x7e080281u, + 0xd9000000u, 0x00000201u, 0xbf8cc07fu, 0xbf810000u + }; + static constexpr std::array capabilities{spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + static constexpr std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + static constexpr std::array userData{}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}}; + request.context.waveSize = waveSize; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 16u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00403000u; + request.target.spirvVersion = 0x00010600u; + request.target.subgroupSize = 32; + request.target.bdaAbiVersion = 1; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.maxWorkgroupSize = {1024u, 1024u, 64u}; + request.target.maxWorkgroupInvocations = 1024; + request.target.maxWorkgroupSharedMemoryBytes = limit; + request.layout = {0, 0, 0, 128}; + request.useCache = false; + return Recompile(request); +} + +void verify(std::uint32_t waveSize, std::uint32_t limit) { + const auto result = compile(waveSize, limit); + require(!result.spirv.empty(), "LDS atomic program did not compile"); + const auto declared = workgroupBytes(result.spirv.Words()); + require(declared <= limit, "declared " + std::to_string(declared) + " Workgroup bytes exceeds limit " + std::to_string(limit)); + require(result.workgroupMemoryDwords == (limit < 72u ? 17u : 0u), "incorrect device-memory LDS stride at limit " + std::to_string(limit)); + if (limit >= 72u) require(declared != 0u, "in-budget LDS unexpectedly lost its Workgroup allocation"); +} + +} + +int main() { + unsigned failures = 0; + for (const auto waveSize : {32u, 64u}) { + for (const auto limit : {64u, 68u, 72u}) { + const auto name = "wave" + std::to_string(waveSize) + " limit" + std::to_string(limit); + try { + verify(waveSize, limit); + std::cout << "LDS sentinel budget: " << name << " passed\n"; + } catch (const std::exception& error) { + ++failures; + const std::string message(error.what()); + std::cerr << "LDS sentinel budget: " << name << ": " << message.substr(0, message.find("RecompileRequest:")) << '\n'; + } + } + } + if (failures != 0) return 1; + std::cout << "LDS sentinel budget tests passed\n"; + return 0; +} From ae56b9558cf17cfcad944c0c7fa62d051f432101 Mon Sep 17 00:00:00 2001 From: jweidenh <38249793+jweidenh@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:22:46 -0400 Subject: [PATCH 034/103] test(agc): exercise LDS sentinel boundaries on native devices --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 7 +++ .../tests/execution/LdsMixedWidthAtomics.cpp | 63 ++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index d1545095c4..8289252e66 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1611,6 +1611,13 @@ if(BUILD_TESTING) if(WIN32) set_tests_properties(agc_sparse_buffer_holes PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_driver_lds_sentinel_budget COMMAND agc_driver_lds_mixed_width_atomics_tests --sentinel-budget) + set_tests_properties(agc_driver_lds_sentinel_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_driver_lds_sentinel_budget PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test(NAME agc_driver_emulated_color_compare_srgb_decode COMMAND agc_driver_emulated_color_compare_tests) set_tests_properties(agc_driver_emulated_color_compare_srgb_decode PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SRGB_SHADER_DECODE=1" diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp index 6528e8af9e..3db59e8190 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp @@ -6,9 +6,13 @@ #include #include #include +#include #include +#include #include #include +#include +#include #include namespace { @@ -182,14 +186,41 @@ void Check(const char* name, std::uint64_t operations, std::uint32_t run) { } } -void Run(AgcDriver::VulkanDevice& device, std::span code, const char* name, std::uint64_t operations, std::uint32_t repeats) { +std::uint64_t WorkgroupBytes(std::span words) { + std::unordered_map> types; + std::unordered_map constants; + std::vector pointers; + for (std::size_t at = 5; at < words.size();) { + const auto count = words[at] >> 16u; + Require(count != 0u && count <= words.size() - at, "lds sentinel budget: malformed SPIR-V"); + const auto instruction = words.subspan(at, count); + const auto opcode = instruction[0] & 0xffffu; + if (opcode == spv::OpTypeInt || opcode == spv::OpTypeArray || opcode == spv::OpTypePointer) types[instruction[1]] = instruction; + if (opcode == spv::OpConstant) constants[instruction[2]] = instruction[3]; + if (opcode == spv::OpVariable && instruction[3] == spv::StorageClassWorkgroup) pointers.push_back(instruction[1]); + at += count; + } + std::uint64_t total = 0; + for (const auto pointer : pointers) { + const auto definition = types.at(pointer); + Require((definition[0] & 0xffffu) == spv::OpTypePointer, "lds sentinel budget: Workgroup storage has no pointer type"); + const auto array = types.at(definition[3]); + Require((array[0] & 0xffffu) == spv::OpTypeArray, "lds sentinel budget: unexpected Workgroup allocation shape"); + const auto element = types.at(array[2]); + Require((element[0] & 0xffffu) == spv::OpTypeInt && element[2] == 32u, "lds sentinel budget: Workgroup array is not dwords"); + total += static_cast(constants.at(array[3])) * sizeof(std::uint32_t); + } + return total; +} + +void Run(AgcDriver::VulkanDevice& device, std::span code, const char* name, std::uint64_t operations, std::uint32_t repeats, std::uint32_t ldsDwords = LdsDwords, std::optional expectedDeviceDwords = std::nullopt) { std::vector userData(8, 0u); const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); std::copy(input.begin(), input.end(), userData.begin()); std::copy(output.begin(), output.end(), userData.begin() + 4); const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; - const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, LdsDwords, {false, false, false}, false, 1}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, ldsDwords, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, @@ -198,20 +229,46 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, c }; request.useCache = false; const auto result = ShaderRecompiler::Recompile(request); + if (expectedDeviceDwords) { + const auto bytes = WorkgroupBytes(result.spirv.Words()); + const auto limit = request.target.maxWorkgroupSharedMemoryBytes; + Require(bytes <= limit, "lds sentinel budget: guest LDS, lock and sentinel exceed the device limit before dispatch"); + Require(result.workgroupMemoryDwords == *expectedDeviceDwords, "lds sentinel budget: unexpected device-backed LDS allocation before dispatch"); + if (*expectedDeviceDwords == 0u) Require(bytes == limit, "lds sentinel budget: exact-fit case did not retain its lock and sentinel"); + std::printf("lds sentinel budget: %s declarations verified; guest=%u; workgroup=%llu; device=%u\n", name, ldsDwords * 4u, static_cast(bytes), result.workgroupMemoryDwords); + } for (std::uint32_t run = 0; run < repeats; ++run) { Output.fill(0xdeadbeefu); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); Check(name, operations, run); } + if (expectedDeviceDwords) std::printf("lds sentinel budget: %s GPU readback verified across 256 lanes\n", name); +} + +void SentinelBudget(AgcDriver::VulkanDevice& device) { + const auto target = device.Target(); + const auto limit = target.maxWorkgroupSharedMemoryBytes; + std::printf("lds sentinel budget: robustBufferAccess2=%u; maxWorkgroupSharedMemoryBytes=%u\n", static_cast(target.robustBufferAccess2), limit); + if (std::getenv("ANYPS5_REQUIRE_ROBUST2") != nullptr) Require(target.robustBufferAccess2, "lds sentinel budget: robustBufferAccess2 was not enabled"); + Require(limit >= 32u && limit % 4u == 0u, "lds sentinel budget: unexpected device shared-memory limit"); + const auto operations = static_cast(Threads) * Iterations; + Run(device, UniformCode, "limit-4", operations, 1, (limit - 4u) / 4u, (limit - 4u) / 4u + 1u); + Run(device, UniformCode, "limit-8", operations, 1, (limit - 8u) / 4u, 0u); + std::puts("lds sentinel budget tests passed"); } } -int main() { +int main(int argc, char** argv) { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (argc == 2 && std::string_view(argv[1]) == "--sentinel-budget") { + SentinelBudget(*device); + return 0; + } + Require(argc == 1, "lds mixed width atomics: unexpected argument"); Run(*device, UniformCode, "uniform", static_cast(Threads) * Iterations, 1); Run(*device, DivergentCode, "divergent", static_cast(Threads) / 2u * Iterations, 1); Run(*device, ParityCode, "split lanes", Threads / 2u, Repeats); From 21a3bd5606a627cad2fb9e7883c87fc558ec7829 Mon Sep 17 00:00:00 2001 From: jweidenh <38249793+jweidenh@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:23:44 -0400 Subject: [PATCH 035/103] fix(recompiler): reserve LDS sentinel and lock headroom --- core/shader/recompiler/Deserialize.cpp | 2 +- core/shader/recompiler/Recompiler.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/shader/recompiler/Deserialize.cpp b/core/shader/recompiler/Deserialize.cpp index 6f1b3a0ca9..61ccd6d5f9 100644 --- a/core/shader/recompiler/Deserialize.cpp +++ b/core/shader/recompiler/Deserialize.cpp @@ -167,7 +167,7 @@ int main(int argc, char** argv) { translateOptions.userDataBaseRegister = request.context.userDataBaseRegister; translateOptions.userDataCount = static_cast(request.context.userData.size()); translateOptions.scratchDwords = request.context.compute.has_value() ? request.context.compute->scratchDwords : 0u; - translateOptions.sharedMemoryBytes = request.context.compute.has_value() && static_cast(request.context.compute->ldsSizeDwords) * 4u + 4u > request.target.maxWorkgroupSharedMemoryBytes ? request.context.compute->ldsSizeDwords * 4u : 0u; + translateOptions.sharedMemoryBytes = request.context.compute.has_value() && static_cast(request.context.compute->ldsSizeDwords) * 4u + 8u > request.target.maxWorkgroupSharedMemoryBytes ? request.context.compute->ldsSizeDwords * 4u : 0u; translateOptions.inputInfo = inputInfo; InstructionTranslator translator; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index d08bc8a148..2481fb8554 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -123,7 +123,7 @@ ShaderStageInputInfo RequestInputInfo(const RecompileRequest& request) { std::uint32_t DeviceMemoryLdsBytes(const RecompileRequest& request) { if (!request.context.compute.has_value()) return 0u; const auto bytes = static_cast(request.context.compute->ldsSizeDwords) * 4u; - return bytes + 4u > request.target.maxWorkgroupSharedMemoryBytes ? static_cast(bytes) : 0u; + return bytes + 8u > request.target.maxWorkgroupSharedMemoryBytes ? static_cast(bytes) : 0u; } std::atomic frontEndBuilds{0}; From 0a9ee3698aeb7ae3095e624c748f7ff6a34b5bb1 Mon Sep 17 00:00:00 2001 From: Mouad-Scriptz-0 Date: Mon, 12 Oct 2026 08:51:36 +0000 Subject: [PATCH 036/103] perf(recompiler): keep LDS loads branched and drop the LDS sentinel NVIDIA's compiler predicates a branched LDS load at no cost, while the selected form costs about 2.5 instructions per dword and splits a ds_read_b128 into four loads (RTX 4060 measurements by @karanvora2599 in loads therefore keep the branch of main, in workgroup, function and device memory alike, and the sentinel dword that only they read goes away: the LDS array is the program's dwords plus the lock again, and the device-memory decision reserves the lock alone. The sentinel budget tests become lock budget tests at that boundary, and the guarded-load test pins the branch. Co-Authored-By: Claude Fable 5.1 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 22 ++++++------ .../tests/GuardedMemoryLoads.cpp | 19 ++++------ ...dsSentinelBudget.cpp => LdsLockBudget.cpp} | 10 +++--- .../tests/execution/LdsMixedWidthAtomics.cpp | 36 +++++++++---------- core/shader/recompiler/Deserialize.cpp | 2 +- core/shader/recompiler/Recompiler.cpp | 2 +- .../SpirvBackend/SpirvEmitterHelpers.hpp | 2 +- .../src/SpirvMemory/SpirvBufferAccess.cpp | 9 ++--- .../SpirvBackend/src/SpirvModuleEmitter.cpp | 2 +- docs/dev/debt/shader.md | 3 +- 10 files changed, 51 insertions(+), 56 deletions(-) rename core/libs/prx/libSceAgcDriver/tests/{LdsSentinelBudget.cpp => LdsLockBudget.cpp} (92%) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 8289252e66..3702492677 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -512,16 +512,16 @@ if(BUILD_TESTING) endif() endif() -add_test_executable(agc_lds_sentinel_budget_tests EXCLUDE_FROM_ALL tests/LdsSentinelBudget.cpp) -target_link_libraries(agc_lds_sentinel_budget_tests PRIVATE shader_recompiler_agc_driver libc) -set_target_properties(agc_lds_sentinel_budget_tests PROPERTIES CXX_EXTENSIONS OFF) -configure_windows_unwind(agc_lds_sentinel_budget_tests) +add_test_executable(agc_lds_lock_budget_tests EXCLUDE_FROM_ALL tests/LdsLockBudget.cpp) +target_link_libraries(agc_lds_lock_budget_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_lds_lock_budget_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_lds_lock_budget_tests) if(BUILD_TESTING) - set_target_properties(agc_lds_sentinel_budget_tests PROPERTIES EXCLUDE_FROM_ALL OFF) - add_test(NAME agc_lds_sentinel_budget COMMAND agc_lds_sentinel_budget_tests) - set_tests_properties(agc_lds_sentinel_budget PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + set_target_properties(agc_lds_lock_budget_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_lds_lock_budget COMMAND agc_lds_lock_budget_tests) + set_tests_properties(agc_lds_lock_budget PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) - set_tests_properties(agc_lds_sentinel_budget PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_lds_lock_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() @@ -1611,10 +1611,10 @@ if(BUILD_TESTING) if(WIN32) set_tests_properties(agc_sparse_buffer_holes PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - add_test(NAME agc_driver_lds_sentinel_budget COMMAND agc_driver_lds_mixed_width_atomics_tests --sentinel-budget) - set_tests_properties(agc_driver_lds_sentinel_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30) + add_test(NAME agc_driver_lds_lock_budget COMMAND agc_driver_lds_mixed_width_atomics_tests --lock-budget) + set_tests_properties(agc_driver_lds_lock_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30) if(WIN32) - set_tests_properties(agc_driver_lds_sentinel_budget PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_driver_lds_lock_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp b/core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp index 241430786c..e20b23d8d5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp @@ -294,19 +294,14 @@ void expectReads(const LoadCase& test, bool robust, bool active, std::uint32_t o void ldsLoads(IrOpcode opcode, IrType type, std::uint32_t bits, std::uint32_t components) { const LoadCase test{ResourceKind::Lds, opcode, type, bits, components}; - const auto length = ldsLength(emitLoad(test, true, 8, 64, false)); - const auto sentinel = length - 1u; - require(sentinel >= 16u, "LDS fixture is smaller than its probes"); + require(ldsLength(emitLoad(test, true, 8, 64, false)) == 16u, "LDS array declares more than the program's dwords"); for (const bool robust : {false, true}) { - std::vector sentinels(components, sentinel); - expectReads(test, robust, false, 8, sentinels); - expectReads(test, robust, true, (sentinel + 5u) * 4u, sentinels); + expectReads(test, robust, false, 8, {}); + expectReads(test, robust, true, 84, {}); std::vector indices; for (std::uint32_t index = 0; index < components; ++index) indices.push_back(2 + index); expectReads(test, robust, true, 8, indices); - std::vector edge{15}; - edge.insert(edge.end(), components - 1u, sentinel); - expectReads(test, robust, true, 60, edge); + expectReads(test, robust, true, 60, {15}); } } @@ -323,12 +318,12 @@ int main() { std::cout << "guarded memory loads: " << name << ": " << error.what() << '\n'; } }; - run("LDS word reads its sentinel", [] { ldsLoads(IrOpcode::LoadSharedU32, IrType::U32, 32, 1); }); - run("LDS subword reads its sentinel", [] { + run("LDS word keeps its branch", [] { ldsLoads(IrOpcode::LoadSharedU32, IrType::U32, 32, 1); }); + run("LDS subword keeps its branch", [] { ldsLoads(IrOpcode::LoadSharedU8, IrType::U8, 8, 1); ldsLoads(IrOpcode::LoadSharedU16, IrType::U16, 16, 1); }); - run("LDS wide reads its sentinel", [] { ldsLoads(IrOpcode::LoadSharedU32x4, IrType::U32x4, 32, 4); }); + run("LDS wide keeps its branch", [] { ldsLoads(IrOpcode::LoadSharedU32x4, IrType::U32x4, 32, 4); }); run("GDS word", [] { const LoadCase test{ResourceKind::Gds, IrOpcode::LoadSharedU32, IrType::U32, 32, 1}; expectReads(test, false, false, 8, {}); diff --git a/core/libs/prx/libSceAgcDriver/tests/LdsSentinelBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/LdsLockBudget.cpp similarity index 92% rename from core/libs/prx/libSceAgcDriver/tests/LdsSentinelBudget.cpp rename to core/libs/prx/libSceAgcDriver/tests/LdsLockBudget.cpp index 0e54ad04ce..8901e04a35 100644 --- a/core/libs/prx/libSceAgcDriver/tests/LdsSentinelBudget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/LdsLockBudget.cpp @@ -88,8 +88,8 @@ void verify(std::uint32_t waveSize, std::uint32_t limit) { require(!result.spirv.empty(), "LDS atomic program did not compile"); const auto declared = workgroupBytes(result.spirv.Words()); require(declared <= limit, "declared " + std::to_string(declared) + " Workgroup bytes exceeds limit " + std::to_string(limit)); - require(result.workgroupMemoryDwords == (limit < 72u ? 17u : 0u), "incorrect device-memory LDS stride at limit " + std::to_string(limit)); - if (limit >= 72u) require(declared != 0u, "in-budget LDS unexpectedly lost its Workgroup allocation"); + require(result.workgroupMemoryDwords == (limit < 68u ? 17u : 0u), "incorrect device-memory LDS stride at limit " + std::to_string(limit)); + if (limit >= 68u) require(declared != 0u, "in-budget LDS unexpectedly lost its Workgroup allocation"); } } @@ -101,15 +101,15 @@ int main() { const auto name = "wave" + std::to_string(waveSize) + " limit" + std::to_string(limit); try { verify(waveSize, limit); - std::cout << "LDS sentinel budget: " << name << " passed\n"; + std::cout << "LDS lock budget: " << name << " passed\n"; } catch (const std::exception& error) { ++failures; const std::string message(error.what()); - std::cerr << "LDS sentinel budget: " << name << ": " << message.substr(0, message.find("RecompileRequest:")) << '\n'; + std::cerr << "LDS lock budget: " << name << ": " << message.substr(0, message.find("RecompileRequest:")) << '\n'; } } } if (failures != 0) return 1; - std::cout << "LDS sentinel budget tests passed\n"; + std::cout << "LDS lock budget tests passed\n"; return 0; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp index 3db59e8190..7c35530d9f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp @@ -192,7 +192,7 @@ std::uint64_t WorkgroupBytes(std::span words) { std::vector pointers; for (std::size_t at = 5; at < words.size();) { const auto count = words[at] >> 16u; - Require(count != 0u && count <= words.size() - at, "lds sentinel budget: malformed SPIR-V"); + Require(count != 0u && count <= words.size() - at, "lds lock budget: malformed SPIR-V"); const auto instruction = words.subspan(at, count); const auto opcode = instruction[0] & 0xffffu; if (opcode == spv::OpTypeInt || opcode == spv::OpTypeArray || opcode == spv::OpTypePointer) types[instruction[1]] = instruction; @@ -203,11 +203,11 @@ std::uint64_t WorkgroupBytes(std::span words) { std::uint64_t total = 0; for (const auto pointer : pointers) { const auto definition = types.at(pointer); - Require((definition[0] & 0xffffu) == spv::OpTypePointer, "lds sentinel budget: Workgroup storage has no pointer type"); + Require((definition[0] & 0xffffu) == spv::OpTypePointer, "lds lock budget: Workgroup storage has no pointer type"); const auto array = types.at(definition[3]); - Require((array[0] & 0xffffu) == spv::OpTypeArray, "lds sentinel budget: unexpected Workgroup allocation shape"); + Require((array[0] & 0xffffu) == spv::OpTypeArray, "lds lock budget: unexpected Workgroup allocation shape"); const auto element = types.at(array[2]); - Require((element[0] & 0xffffu) == spv::OpTypeInt && element[2] == 32u, "lds sentinel budget: Workgroup array is not dwords"); + Require((element[0] & 0xffffu) == spv::OpTypeInt && element[2] == 32u, "lds lock budget: Workgroup array is not dwords"); total += static_cast(constants.at(array[3])) * sizeof(std::uint32_t); } return total; @@ -232,10 +232,10 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, c if (expectedDeviceDwords) { const auto bytes = WorkgroupBytes(result.spirv.Words()); const auto limit = request.target.maxWorkgroupSharedMemoryBytes; - Require(bytes <= limit, "lds sentinel budget: guest LDS, lock and sentinel exceed the device limit before dispatch"); - Require(result.workgroupMemoryDwords == *expectedDeviceDwords, "lds sentinel budget: unexpected device-backed LDS allocation before dispatch"); - if (*expectedDeviceDwords == 0u) Require(bytes == limit, "lds sentinel budget: exact-fit case did not retain its lock and sentinel"); - std::printf("lds sentinel budget: %s declarations verified; guest=%u; workgroup=%llu; device=%u\n", name, ldsDwords * 4u, static_cast(bytes), result.workgroupMemoryDwords); + Require(bytes <= limit, "lds lock budget: guest LDS and lock exceed the device limit before dispatch"); + Require(result.workgroupMemoryDwords == *expectedDeviceDwords, "lds lock budget: unexpected device-backed LDS allocation before dispatch"); + if (*expectedDeviceDwords == 0u) Require(bytes == limit, "lds lock budget: exact-fit case did not retain its lock"); + std::printf("lds lock budget: %s declarations verified; guest=%u; workgroup=%llu; device=%u\n", name, ldsDwords * 4u, static_cast(bytes), result.workgroupMemoryDwords); } for (std::uint32_t run = 0; run < repeats; ++run) { Output.fill(0xdeadbeefu); @@ -243,19 +243,19 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, c device.WaitIdle(); Check(name, operations, run); } - if (expectedDeviceDwords) std::printf("lds sentinel budget: %s GPU readback verified across 256 lanes\n", name); + if (expectedDeviceDwords) std::printf("lds lock budget: %s GPU readback verified across 256 lanes\n", name); } -void SentinelBudget(AgcDriver::VulkanDevice& device) { +void LockBudget(AgcDriver::VulkanDevice& device) { const auto target = device.Target(); const auto limit = target.maxWorkgroupSharedMemoryBytes; - std::printf("lds sentinel budget: robustBufferAccess2=%u; maxWorkgroupSharedMemoryBytes=%u\n", static_cast(target.robustBufferAccess2), limit); - if (std::getenv("ANYPS5_REQUIRE_ROBUST2") != nullptr) Require(target.robustBufferAccess2, "lds sentinel budget: robustBufferAccess2 was not enabled"); - Require(limit >= 32u && limit % 4u == 0u, "lds sentinel budget: unexpected device shared-memory limit"); + std::printf("lds lock budget: robustBufferAccess2=%u; maxWorkgroupSharedMemoryBytes=%u\n", static_cast(target.robustBufferAccess2), limit); + if (std::getenv("ANYPS5_REQUIRE_ROBUST2") != nullptr) Require(target.robustBufferAccess2, "lds lock budget: robustBufferAccess2 was not enabled"); + Require(limit >= 32u && limit % 4u == 0u, "lds lock budget: unexpected device shared-memory limit"); const auto operations = static_cast(Threads) * Iterations; - Run(device, UniformCode, "limit-4", operations, 1, (limit - 4u) / 4u, (limit - 4u) / 4u + 1u); - Run(device, UniformCode, "limit-8", operations, 1, (limit - 8u) / 4u, 0u); - std::puts("lds sentinel budget tests passed"); + Run(device, UniformCode, "limit", operations, 1, limit / 4u, limit / 4u + 1u); + Run(device, UniformCode, "limit-4", operations, 1, (limit - 4u) / 4u, 0u); + std::puts("lds lock budget tests passed"); } } @@ -264,8 +264,8 @@ int main(int argc, char** argv) { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - if (argc == 2 && std::string_view(argv[1]) == "--sentinel-budget") { - SentinelBudget(*device); + if (argc == 2 && std::string_view(argv[1]) == "--lock-budget") { + LockBudget(*device); return 0; } Require(argc == 1, "lds mixed width atomics: unexpected argument"); diff --git a/core/shader/recompiler/Deserialize.cpp b/core/shader/recompiler/Deserialize.cpp index 61ccd6d5f9..6f1b3a0ca9 100644 --- a/core/shader/recompiler/Deserialize.cpp +++ b/core/shader/recompiler/Deserialize.cpp @@ -167,7 +167,7 @@ int main(int argc, char** argv) { translateOptions.userDataBaseRegister = request.context.userDataBaseRegister; translateOptions.userDataCount = static_cast(request.context.userData.size()); translateOptions.scratchDwords = request.context.compute.has_value() ? request.context.compute->scratchDwords : 0u; - translateOptions.sharedMemoryBytes = request.context.compute.has_value() && static_cast(request.context.compute->ldsSizeDwords) * 4u + 8u > request.target.maxWorkgroupSharedMemoryBytes ? request.context.compute->ldsSizeDwords * 4u : 0u; + translateOptions.sharedMemoryBytes = request.context.compute.has_value() && static_cast(request.context.compute->ldsSizeDwords) * 4u + 4u > request.target.maxWorkgroupSharedMemoryBytes ? request.context.compute->ldsSizeDwords * 4u : 0u; translateOptions.inputInfo = inputInfo; InstructionTranslator translator; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 2481fb8554..d08bc8a148 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -123,7 +123,7 @@ ShaderStageInputInfo RequestInputInfo(const RecompileRequest& request) { std::uint32_t DeviceMemoryLdsBytes(const RecompileRequest& request) { if (!request.context.compute.has_value()) return 0u; const auto bytes = static_cast(request.context.compute->ldsSizeDwords) * 4u; - return bytes + 8u > request.target.maxWorkgroupSharedMemoryBytes ? static_cast(bytes) : 0u; + return bytes + 4u > request.target.maxWorkgroupSharedMemoryBytes ? static_cast(bytes) : 0u; } std::atomic frontEndBuilds{0}; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index d996faf30e..836b58a04c 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -118,7 +118,7 @@ std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryRe bool InFunctionStorage(const SpirvEmitterState& state, ResourceKind kind); std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); std::uint32_t EmitValueOrZero(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t value); -std::uint32_t LdsSentinelIndex(const SpirvEmitterState& state); +std::uint32_t LdsStorageDwords(const SpirvEmitterState& state); bool LoadsSelectable(const SpirvEmitterState& state, const MemoryResourceAccess& access); std::uint32_t GuardedElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t condition, std::uint32_t index); std::uint32_t EmitStorageBufferElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index, std::uint32_t pointerType); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp index 5fa98f5610..d208290cac 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp @@ -20,7 +20,7 @@ namespace { if (ShaderWorkgroupInput(state) == nullptr) { FailEmit("function LDS was not prepared before function emission"); } - state.ldsVariable = state.module.DefineGlobalVariable(TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsSentinelIndex(state) + 1u), spv::StorageClassWorkgroup); + state.ldsVariable = state.module.DefineGlobalVariable(TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsStorageDwords(state)), spv::StorageClassWorkgroup); state.module.AddName(state.ldsVariable, "lds_dwords"); } @@ -177,18 +177,19 @@ std::uint32_t EmitValueOrZero(SpirvEmitterState& state, std::uint32_t condition, return Select(state, TypeU32(state), condition, value, ConstantU32(state, 0u)); } -std::uint32_t LdsSentinelIndex(const SpirvEmitterState& state) { +std::uint32_t LdsStorageDwords(const SpirvEmitterState& state) { return LdsDwordCount(state) + (state.requirements.ldsLock ? 1u : 0u); } bool LoadsSelectable(const SpirvEmitterState& state, const MemoryResourceAccess& access) { if (access.kind == ResourceKind::Scratch) return true; - if (access.kind == ResourceKind::Lds) return !LdsInDeviceMemory(state); + if (access.kind == ResourceKind::Lds) return false; return state.robustBufferAccess2; } std::uint32_t GuardedElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t condition, std::uint32_t index) { - const auto fallback = access.kind == ResourceKind::Scratch ? 0u : access.kind == ResourceKind::Lds ? LdsSentinelIndex(state) : 0xfffffffeu; + if (access.kind == ResourceKind::Lds) FailEmit("LDS loads keep their branch and have no guarded element"); + const auto fallback = access.kind == ResourceKind::Scratch ? 0u : 0xfffffffeu; return Select(state, TypeU32(state), condition, index, ConstantU32(state, fallback)); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index cd1f8c8ddc..33ce878077 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -1146,7 +1146,7 @@ void EmitProgram(SpirvEmitterState& state) { state.module.AddFunction(spv::OpFunction, TypeVoid(state), state.meshGuestFunc != 0u ? state.meshGuestFunc : state.mainFunc, spv::FunctionControlMaskNone, TypeFunction(state)); EmitLabel(state, state.entryLabel); if (state.requirements.functionLds) { - state.module.AddFunction(spv::OpVariable, TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsSentinelIndex(state) + 1u), state.ldsVariable, spv::StorageClassFunction); + state.module.AddFunction(spv::OpVariable, TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsStorageDwords(state)), state.ldsVariable, spv::StorageClassFunction); } if (state.requirements.functionScratch) { for (std::uint32_t half = 0; half < state.laneCount; half++) { diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 7b7e78f8b8..5ad66baace 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -147,5 +147,4 @@ - [Flag structurization](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a graph that every structurizer attempt refuses, irreducible and non-terminating graphs included, is rebuilt with one reach flag per block and one again flag per loop (`structurizeWithFlags`): each block keeps one copy and runs behind a guard on its flag, and each strongly connected component becomes a loop whose single latch repeats it while a back edge set its flag. The output has about 8 times the input's blocks and one route variable per block, and ends in an empty Return that never runs. Only blocks that start the top-level order unconditionally run unguarded, so resource planning also sees the static paths that skip a guarded block and can plan extra descriptor candidates for them. The attempts still run first. Shared merge splitting refuses an attempt once the graph has grown past 16 times its block count at the start of splitting (at least 64 blocks); succeeding attempts grew at most 10.6 times on random graphs of up to 128 blocks, and an attempt that would succeed only beyond the budget leaves the graph to a later attempt or to this construction. Splitting that adds one block per round stays within that budget until its 4 rounds per block run out, so the attempts can still take more than 10 s (9 of 2,000 random graphs of up to 128 blocks, 63 s at most). `APS5_FORCE_STRUCTURIZER_FALLBACK=1` builds every graph this way, for tests - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) use only the base address, stride and record count for resources accessed exclusively by scalar loads; their fourth word is normalized to zero, following AMD RDNA2 ISA section 7.2.2. Vector and mixed accesses retain descriptor validation. Native Vulkan readbacks cover ignored fourth-word fields on Windows/NVIDIA; their console behavior has not been measured. - [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - `APS5_SPANNED_WAVE=1`, off by default, runs a single-wave wave64 compute program at one lane per invocation on a 32-lane host subgroup and exchanges ballots, shuffles and reductions between the two host subgroups through workgroup memory and barriers. The driver turns it on only where compute subgroups can be pinned at 32 full lanes (`subgroupSizeControl` and `computeFullSubgroups`), passes it as `SpirvTarget::spannedWaves` (part of the cache key and the serialized request), and creates the pipelines of spanned programs with `REQUIRE_FULL_SUBGROUPS`, since the lane number `SubgroupId * 32 + SubgroupLocalInvocationId` needs two full subgroups. Those barriers are correct only where every invocation of the workgroup reaches them, which holds for the guest's wave-uniform control flow. The emulated 64-bit LDS atomics ([LockedLdsUpdate](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp)) run only on the lanes the exec mask keeps, so their lock loop ballots within its own host subgroup, as its elect already does. Any other cross-lane exchange emitted inside per-lane control flow would hang a spanned wave in the same way, and the emitter does not reject one: there is no uniformity check. The only check so far is manual: disassemble the module (`spirv-dis`), list the selection and loop constructs that enclose each `OpControlBarrier`, and read the condition of each enclosing selection. In Ghost of Yōtei's program 0x80003af600 (PPSA26344), 108 of 111 barriers sat only in loops on wave-uniform conditions and 3 inside per-invocation conditionals, all in the lock loop. Other programs and titles have not been scanned. An earlier, cheaper ballot (one barrier instead of three) also lost the device for a reason that is not understood, which may be another barrier placement problem. Measured on an RTX 4090 (Windows): an offline replay of that program and two Yōtei runs that previously lost the device (NVIDIA TDR) now run past it, and [LdsMaskWaterfall.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp) runs both modes and covers a half-inactive wave. Stray (PPSA02101) on Intel ANV and a Demon's Souls (PPSA01342) request replay showed no fault, before this fix -- [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - LDS, scratch and direct storage-buffer loads (`LoadWord`, `LoadSubword`, `LoadWideShared`, `readBuffer` and `EmitRuntimeScalarBufferLoad` in [SpirvRuntimeBuffer.cpp](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp)) load element 0 when the lane is inactive or the address is out of bounds and select 0 as the result, instead of branching around the load; stores, atomics, formatted and physical-address (BDA) accesses still branch. A branch guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. The unconditional load depends on element 0 being addressable: SPIR-V function and workgroup arrays have at least one element, and a storage buffer with no elements reads out of bounds, which is defined only with `robustBufferAccess` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it). -- [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; LDS in workgroup or function memory reads a sentinel dword appended after the program's LDS (and after the lock dword), which no bounds-checked write reaches; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it), and for LDS kept in device memory, those loads keep their selections; stores, atomics, formatted and physical-address (BDA) accesses always do. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. +- [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it) those loads keep their selections; LDS loads, stores, atomics, formatted and physical-address (BDA) accesses always do. LDS keeps the branch because NVIDIA's compiler predicates a branched LDS load at no cost, while the select form costs about 2.5 instructions per dword and splits a `ds_read_b128` into four loads (RTX 4060 measurements by @karanvora2599 in #4048); Intel's static counts favour the select only for wide reads, and run time was not measured on either. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. From 2b9c131e0cd8a7a1de34eb3e73df14eae548a8aa Mon Sep 17 00:00:00 2001 From: Alexius0815 <79040541+Alexius0815@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:24:53 +0200 Subject: [PATCH 037/103] fix(shader): forward gl_ClipDistance through the generated rect-list stages Since #3878 every vertex program that exports a position also writes gl_ClipDistance for the all-zero position distance. The tessellation stages generated for rect lists declared only gl_Position in gl_in, so their input interface no longer matched the vertex outputs; on MoltenVK the control stage then reads all-zero positions and every rect-list draw stops with "rect-list requires finite nondegenerate axis-aligned positions" (reported in #1). The control and evaluation stages now declare the vertex program's gl_ClipDistance array in their per-vertex blocks and forward it. The generated fourth corner gets the clip distances interpolated with the same weights as its position, so the all-zero position clip keeps working for rect lists. Validation accepts gl_ClipDistance as a tessellation input and as a tessellation-control output. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QXdkh9PHBCPqGcGQPeikAe --- .../Graphics/src/ShaderValidation.cpp | 9 ++-- .../SpirvBackend/SpirvClipDistances.hpp | 46 +++++++++++++++++++ .../SpirvBackend/src/RectListShaders.cpp | 38 +++++++++++++-- 3 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvClipDistances.hpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index f2c1970381..34bd3349af 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -115,7 +115,7 @@ struct Module { return; } // One gl_ClipDistance array holds the guest clip distances and the recompiler's all-zero position distance. - if (!input && ((value == spv::BuiltInClipDistance && (vertex || stage == Stage::TessellationEvaluation || stage == Stage::Mesh)) || (value == spv::BuiltInCullDistance && (vertex || stage == Stage::Mesh)))) { + if (!input && ((value == spv::BuiltInClipDistance && (vertex || stage == Stage::TessellationControl || stage == Stage::TessellationEvaluation || stage == Stage::Mesh)) || (value == spv::BuiltInCullDistance && (vertex || stage == Stage::Mesh)))) { const auto signature = Signature(type); auto& count = value == spv::BuiltInClipDistance ? clipDistances : cullDistances; Require(count == 0 && signature.size() == 6 && signature.starts_with("f32[") && signature[4] >= '1' && signature[4] <= '8' && signature[5] == ']', "invalid or duplicate clip or cull distance output"); @@ -131,7 +131,8 @@ struct Module { return; } if (stage == Stage::TessellationControl || stage == Stage::TessellationEvaluation) { - Require(input && ((value == spv::BuiltInPosition && signature == "f32x4") || (value == spv::BuiltInTessCoord && stage == Stage::TessellationEvaluation && signature == "f32x3") || ((value == spv::BuiltInInvocationId || value == spv::BuiltInPrimitiveId || value == spv::BuiltInPatchVertices) && (signature == "i32" || signature == "u32"))), "unsupported tessellation built-in"); + const bool clipInput = value == spv::BuiltInClipDistance && signature.size() == 6 && signature.starts_with("f32[") && signature[4] >= '1' && signature[4] <= '8' && signature[5] == ']'; + Require(input && (clipInput || (value == spv::BuiltInPosition && signature == "f32x4") || (value == spv::BuiltInTessCoord && stage == Stage::TessellationEvaluation && signature == "f32x3") || ((value == spv::BuiltInInvocationId || value == spv::BuiltInPrimitiveId || value == spv::BuiltInPatchVertices) && (signature == "i32" || signature == "u32"))), "unsupported tessellation built-in"); return; } if (!input && value == spv::BuiltInPrimitiveLineIndicesEXT) primitiveIndices = 2u; @@ -252,7 +253,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi fragment && capability == spv::CapabilityStencilExportEXT; - if ((vertex || evaluation || mesh) && (capability == spv::CapabilityClipDistance || capability == spv::CapabilityCullDistance)) { + if ((vertex || control || evaluation || mesh) && (capability == spv::CapabilityClipDistance || capability == spv::CapabilityCullDistance)) { char word[16]; std::snprintf(word, sizeof(word), "0x%08x", state.paClVsOutCntl); Require(capability == spv::CapabilityClipDistance ? shaderClipDistance : (vertex || mesh) && shaderCullDistance, std::string("vertex clip or cull distance needs shaderClipDistance or shaderCullDistance, which the device lacks (PA_CL_VS_OUT_CNTL=") + word + ")"); @@ -264,7 +265,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi capability == spv::CapabilityImage1D || capability == spv::CapabilityImageGatherExtended || capability == spv::CapabilityImageQuery || - ((vertex || evaluation || mesh) && shaderClipDistance && capability == spv::CapabilityClipDistance) || + ((vertex || control || evaluation || mesh) && shaderClipDistance && capability == spv::CapabilityClipDistance) || (vertex && capability == spv::CapabilityDrawParameters) || (fragment && geometryShader && capability == spv::CapabilityGeometry) || (fragment && sampleRateShading && capability == spv::CapabilitySampleRateShading) || diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvClipDistances.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvClipDistances.hpp new file mode 100644 index 0000000000..2d915b38eb --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvClipDistances.hpp @@ -0,0 +1,46 @@ +#ifndef CORE_SHADER_RECOMPILIER_SPIRVBACKEND_INCLUDE_SPIRVBACKEND_SPIRVCLIPDISTANCES_HPP +#define CORE_SHADER_RECOMPILIER_SPIRVBACKEND_INCLUDE_SPIRVBACKEND_SPIRVCLIPDISTANCES_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { + +[[nodiscard]] inline std::uint32_t VertexClipDistanceCount(std::span words) { + std::set builtins; + std::map pointees; + std::map arrays; + std::map constants; + std::map outputs; + for (std::size_t i = 5; i < words.size();) { + const auto count = words[i] >> 16u; + const auto opcode = words[i] & 0xffffu; + if (count == 0u || i + count > words.size()) throw std::runtime_error("Clip distances: malformed vertex SPIR-V"); + const auto* word = words.data() + i; + if (opcode == spv::OpDecorate && count == 4u && word[2] == spv::DecorationBuiltIn && word[3] == spv::BuiltInClipDistance) builtins.insert(word[1]); + if (opcode == spv::OpTypePointer && count == 4u) pointees[word[1]] = word[3]; + if (opcode == spv::OpTypeArray && count == 4u) arrays[word[1]] = word[3]; + if (opcode == spv::OpConstant && count == 4u) constants[word[2]] = word[3]; + if (opcode == spv::OpVariable && count >= 4u && word[3] == spv::StorageClassOutput) outputs[word[2]] = word[1]; + i += count; + } + for (const auto variable : builtins) { + const auto output = outputs.find(variable); + if (output == outputs.end()) continue; + const auto pointee = pointees.find(output->second); + const auto array = pointee == pointees.end() ? arrays.end() : arrays.find(pointee->second); + const auto length = array == arrays.end() ? constants.end() : constants.find(array->second); + if (length == constants.end() || length->second == 0u || length->second > 8u) throw std::runtime_error("Clip distances: the vertex gl_ClipDistance output is not an array of one to eight elements"); + return length->second; + } + return 0u; +} + +} + +#endif diff --git a/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp b/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp index 59757fa18c..7dad183575 100644 --- a/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp +++ b/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp @@ -1,6 +1,7 @@ #include "Recompiler.hpp" #include "BdaAbi.hpp" #include "RuntimeAbi.hpp" +#include "SpirvBackend/SpirvClipDistances.hpp" #include "SpirvBackend/SpirvModule.hpp" #if ANYPS5_ENABLE_SPIRV_TOOLS #include "SpirvBackend/SpirvOptimizer.hpp" @@ -31,7 +32,7 @@ void require(bool condition, const char* message) { class RectListEmitter { public: - RectListEmitter(const std::vector& parameters, spv::ExecutionModel model, std::uint32_t version, std::uint32_t faultBinding) : builder(version), faultBinding(faultBinding), version(version), parameters(parameters) { + RectListEmitter(const std::vector& parameters, spv::ExecutionModel model, std::uint32_t version, std::uint32_t faultBinding, std::uint32_t clipDistances) : builder(version), faultBinding(faultBinding), version(version), clipDistances(clipDistances), parameters(parameters) { builder.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450); voidType = type(spv::OpTypeVoid); @@ -41,7 +42,14 @@ public: vec4FloatType = type(spv::OpTypeVector, floatType, 4u); functionType = type(spv::OpTypeFunction, voidType); - perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpMemberDecorate, {0u, spv::DecorationInvariant}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType); + if (clipDistances != 0u) { + clipArrayType = array(floatType, clipDistances); + perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpMemberDecorate, {0u, spv::DecorationInvariant}}, {spv::OpMemberDecorate, {1u, spv::DecorationBuiltIn, spv::BuiltInClipDistance}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType, clipArrayType); + ptrInputClip = pointer(spv::StorageClassInput, clipArrayType); + ptrOutputClip = pointer(spv::StorageClassOutput, clipArrayType); + } else { + perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpMemberDecorate, {0u, spv::DecorationInvariant}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType); + } ptrInputVec4Float = pointer(spv::StorageClassInput, vec4FloatType); ptrOutputVec4Float = pointer(spv::StorageClassOutput, vec4FloatType); @@ -116,6 +124,20 @@ public: const auto position = result(spv::OpSelect, vec4FloatType, isFourth, position3, load(vec4FloatType, access(ptrInputVec4Float, glIn, index, intConstant(0)))); store(access(ptrOutputVec4Float, glOut, invocation, intConstant(0)), position); + if (clipDistances != 0u) { + std::array clips {}; + for (std::uint32_t i = 0; i < clips.size(); i++) clips[i] = load(clipArrayType, access(ptrInputClip, glIn, intConstant(i), intConstant(1))); + const auto own = load(clipArrayType, access(ptrInputClip, glIn, index, intConstant(1))); + std::vector values; + for (std::uint32_t k = 0; k < clipDistances; k++) { + auto sum = constant(floatType, 0); + for (std::uint32_t i = 0; i < clips.size(); i++) sum = result(spv::OpFAdd, floatType, sum, result(spv::OpFMul, floatType, result(spv::OpCompositeExtract, floatType, clips[i], k), barycentric[i])); + values.push_back(result(spv::OpSelect, floatType, fourth, sum, result(spv::OpCompositeExtract, floatType, own, k))); + } + const auto clip = builder.AllocateId(); + builder.AddFunction(spv::OpCompositeConstruct, clipArrayType, clip, std::span(values)); + store(access(ptrOutputClip, glOut, invocation, intConstant(1)), clip); + } for (std::uint32_t i = 0; i < parameters.size(); i++) { if (parameters[i].missing) { @@ -149,6 +171,7 @@ public: const auto position = load(vec4FloatType, access(ptrInputVec4Float, glIn, index, intConstant(0))); store(access(ptrOutputVec4Float, glOut, intConstant(0)), position); + if (clipDistances != 0u) store(access(ptrOutputClip, glOut, intConstant(1)), load(clipArrayType, access(ptrInputClip, glIn, index, intConstant(1)))); for (std::uint32_t i = 0; i < parameters.size(); i++) { store(outputs[i], load(vec4FloatType, access(ptrInputVec4Float, inputs[i], index))); } @@ -271,6 +294,7 @@ private: void defineEntry(spv::ExecutionModel model) { builder.EmitCapability(spv::CapabilityShader); builder.EmitCapability(spv::CapabilityTessellation); + if (clipDistances != 0u) builder.EmitCapability(spv::CapabilityClipDistance); main = result(spv::OpFunction, voidType, spv::FunctionControlMaskNone, functionType); if (model == spv::ExecutionModelTessellationControl) { builder.AddExecutionMode(main, spv::ExecutionModeOutputVertices, 4u); @@ -343,6 +367,7 @@ private: SpirvModule builder; std::uint32_t faultBinding; std::uint32_t version; + std::uint32_t clipDistances; std::uint32_t fault = 0; const std::vector& parameters; std::vector interfaces; @@ -360,6 +385,9 @@ private: std::uint32_t vec4FloatType = 0; std::uint32_t functionType = 0; std::uint32_t perVertexType = 0; + std::uint32_t clipArrayType = 0; + std::uint32_t ptrInputClip = 0; + std::uint32_t ptrOutputClip = 0; std::uint32_t ptrInputFloat = 0; std::uint32_t ptrInputVec4Float = 0; std::uint32_t ptrOutputFloat = 0; @@ -394,9 +422,11 @@ RectListShaders BuildRectListShaders(const RecompileResult& vertex, const Recomp } const auto components = static_cast((parameters.size() + 1) * 4); require(limits.maxPatchSize >= 4 && components <= limits.maxControlPerVertexInputComponents && components <= limits.maxControlPerVertexOutputComponents && components <= limits.maxEvaluationInputComponents && components <= limits.maxEvaluationOutputComponents && limits.maxControlPerPatchOutputComponents >= 6 && components * 4 + 6 <= limits.maxControlTotalOutputComponents, "tessellation interface exceeds device limits"); + const auto clipDistances = VertexClipDistanceCount(vertex.spirv.Words()); + require(clipDistances == 0u || std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), spv::CapabilityClipDistance) != target.supportedCapabilities.end(), "the vertex clip distances need shaderClipDistance"); const auto faultBinding = RuntimeAbi::BindingNumber(RuntimeAbi::Stage::TessellationControl, RuntimeAbi::Binding::FaultBuffer); - RectListEmitter control(parameters, spv::ExecutionModelTessellationControl, target.spirvVersion, faultBinding); - RectListEmitter evaluation(parameters, spv::ExecutionModelTessellationEvaluation, target.spirvVersion, faultBinding); + RectListEmitter control(parameters, spv::ExecutionModelTessellationControl, target.spirvVersion, faultBinding, clipDistances); + RectListEmitter evaluation(parameters, spv::ExecutionModelTessellationEvaluation, target.spirvVersion, faultBinding, clipDistances); RectListShaders shaders; shaders.control.spirv = control.EmitControl(); shaders.control.bindings.push_back({DescriptorKind::StorageBuffer, DescriptorRole::FaultBuffer, 0, faultBinding, 1, {}}); From 9dc65429991c7619e13c7827867063c6811aba50 Mon Sep 17 00:00:00 2001 From: mattogepeto Date: Thu, 15 Oct 2026 23:18:14 +0200 Subject: [PATCH 038/103] fix(libkernel): report EINVAL for getdirentries on a non-directory sceKernelGetdirentries on a descriptor that is not a directory returned ENOTDIR from the host getdents64 on Linux, and ENOTDIR for every descriptor missing from the directory table on Windows, including invalid ones. FreeBSD 11.4's kern_getdirentries returns EINVAL when the vnode is not a directory. Return EINVAL for a regular file on both hosts and EBADF for an invalid descriptor on Windows. Test getdents and getdirentries on a regular file and list the pipe case, which still fails with ESPIPE on Linux, in the library's technical debt file. Written with AI assistance (Claude). Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libkernel/File/src/Stdio.cpp | 6 +++++- core/libs/tests/GuestFilesystem.cpp | 9 +++++++++ docs/dev/debt/libkernel.md | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 510c992a55..862aa1f7e8 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -985,6 +985,10 @@ int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* bas File::RefuseArchivedFile(fd, "getdirentries"); if (buf == nullptr) return SceErrorFromErrno(GUEST_EFAULT); if (nbytes <= 0) return SceErrorFromErrno(GUEST_EINVAL); + if (!File::DirectoryDescriptorPath(fd)) { + if (WithoutParameterHandler([fd] { return ::_get_osfhandle(fd); }) == -1) return SceErrorFromErrno(GUEST_EBADF); + return SceErrorFromErrno(GUEST_EINVAL); + } if (basep != nullptr) *basep = 0; return File::ReadDirectoryDescriptor(fd, buf, nbytes); } @@ -1007,7 +1011,7 @@ int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* bas if (basep != nullptr) *basep = static_cast(base); std::vector host(static_cast(nbytes) < 4096 ? 4096 : static_cast(nbytes)); const auto read = ::syscall(SYS_getdents64, fd, host.data(), host.size()); - if (read < 0) return SceErrorFromErrno(errno); + if (read < 0) return SceErrorFromErrno(errno == ENOTDIR ? GUEST_EINVAL : errno); std::size_t written = 0; off_t resume = base; for (long offset = 0; offset < read;) { diff --git a/core/libs/tests/GuestFilesystem.cpp b/core/libs/tests/GuestFilesystem.cpp index b87cf1344c..aca41f4a8a 100644 --- a/core/libs/tests/GuestFilesystem.cpp +++ b/core/libs/tests/GuestFilesystem.cpp @@ -55,6 +55,8 @@ int APS5_VABI mkdir_nid_postfix(const char*, unsigned short); int APS5_VABI sceKernelOpen(const char*, int, unsigned short); int APS5_VABI sceKernelClose(int); int APS5_VABI sceKernelGetdents(int, char*, int); +int APS5_VABI getdents_nid_postfix(int, char*, int); +int APS5_VABI getdirentries_nid_postfix(int, char*, int, int64_t*); std::int64_t APS5_VABI sceKernelRead(int, void*, std::size_t); std::int64_t APS5_VABI sceKernelReadv(int, const void*, int); std::int64_t APS5_VABI sceKernelPread(int, void*, std::size_t, std::int64_t); @@ -833,6 +835,13 @@ int main() { Require(sceKernelPwrite(directoryDescriptor, "x", 1, 0) == static_cast(0x80020009u)); std::vector listing(4096); Require(sceKernelGetdents(directoryDescriptor, listing.data(), static_cast(listing.size())) > 0); + const int plainFile = open_nid_postfix(presentName.c_str(), 0, 0); + Require(plainFile >= 0); + Require(sceKernelGetdents(plainFile, listing.data(), static_cast(listing.size())) == static_cast(0x80020016u)); + Require(getdents_nid_postfix(plainFile, listing.data(), static_cast(listing.size())) == -1 && *__error_nid_postfix() == 22); + int64_t plainBase = 0; + Require(getdirentries_nid_postfix(plainFile, listing.data(), static_cast(listing.size()), &plainBase) == -1 && *__error_nid_postfix() == 22); + Require(close_nid_postfix(plainFile) == 0); Require(close_nid_postfix(directoryDescriptor) == 0); const int flaggedDirectory = sceKernelOpen(rootName.c_str(), 0x20000, 0); Require(flaggedDirectory >= 0 && sceKernelClose(flaggedDirectory) == 0); diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 5791f7ab5e..da7053539c 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -122,6 +122,7 @@ - [system](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) runs no command. It follows FreeBSD 11's `lib/libc/stdlib/system.c` for a sandbox without `/bin/sh`: a null command returns 1 unconditionally, as that version does, and any command returns the wait status of a child that could not exec the shell and called `_exit(127)` (`0x7f00`). Whether the console's application sandbox has no shell, and whether `vfork` succeeds there (FreeBSD returns -1 when it fails), is unverified - [sceKernelWriteThrottlingStatus](../../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown - [Windows directory descriptors](../../../core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp) (libkernel) take one snapshot of the directory, on the first `getdents`, `getdirentries` or non-zero `lseek` (`SEEK_SET` or `SEEK_CUR`), and keep it until it is discarded; `lseek` to offset 0 only discards the snapshot, and the next read lists the directory again, so every rewind costs O(entries) on the next read. `lseek` takes the offset as an entry index and `SEEK_CUR` with offset 0 only reports the index. `SEEK_END` throws, where the NUL descriptor used to return 0, because FreeBSD defines it relative to the directory size, which is not modelled. A negative result returns `EINVAL` and a `SEEK_CUR` offset that overflows returns `EOVERFLOW`, as FreeBSD `vn_seek` reports them. Names whose UTF-8 encoding exceeds the one-byte `d_namlen` limit are not representable in this guest directory-record layout: `getdents` returns `ENAMETOOLONG` without consuming the entry. Repeated calls therefore report the same error and entries after that name cannot be reached through this descriptor; FreeBSD's `getdents` behavior for host filenames that exceed its `MAXNAMLEN` is not verified. `getdirentries` still reports 0 through `basep` instead of the position of the block it read, so only 0 and values returned by `lseek` are valid offsets; FreeBSD offsets are filesystem cookies +- [getdents and getdirentries](../../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) return `EINVAL` for a descriptor that is not a directory, as [`kern_getdirentries`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/kern/vfs_syscalls.c) does for a vnode that is not `VDIR`. On Linux a host pipe or FIFO still fails with `ESPIPE`, from the `lseek` that reads the base offset before `getdents64`, where `getvnode` returns `EINVAL` for a non-vnode descriptor - [/dev/random and /dev/urandom](../../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) (libkernel) opened for reading return a descriptor backed by the host null device whose reads are filled with host OS entropy (`getentropy`, `rand_s` on Windows); both paths behave the same, never block, and positioned reads ignore the offset (FreeBSD `random(4)`). Only the exact paths are recognised, `fstat`, `lseek` and `mmap` see the null device, and opening them for writing throws - [pthread_mutexattr_getpshared and pthread_mutexattr_setpshared](../../../core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp) (libkernel) store and report private or shared attributes, following FreeBSD 11 libthr [thr_mutexattr.c](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/libthr/thread/thr_mutexattr.c). Invalid values return `EINVAL` without changing the attribute. Guest threads execute in one host process; sharing mutexes across processes is not modeled. Console behavior and invalid-attribute error precedence are unverified. - [pthread_attr_getinheritsched, pthread_attr_setinheritsched, pthread_attr_getscope, pthread_attr_setscope, pthread_attr_getstackaddr](../../../core/libs/prx/libkernel/Pthread/Posix/Attr.cpp) (libkernel) follow FreeBSD 11 libthr ([thr_attr.c](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/lib/libthr/thread/thr_attr.c), [thr_create.c](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/lib/libthr/thread/thr_create.c)), not measured on the console: the scope defaults to `PTHREAD_SCOPE_SYSTEM` (2), only it and `PTHREAD_SCOPE_PROCESS` (0) are accepted and other values return `EINVAL`, and an inherit value other than `PTHREAD_INHERIT_SCHED` (4) or `PTHREAD_EXPLICIT_SCHED` (0) returns `ENOTSUP`. `pthread_attr_getstackaddr` returns the configured stack address pointer (null by default). A null output of `pthread_attr_getinheritsched` throws, since libthr writes through it unchecked. A thread keeps the scope it was created with, taken from the creating thread under `PTHREAD_INHERIT_SCHED`, and `pthread_attr_get_np` and `scePthreadAttrGet` report it, but every guest thread is a host thread at system scope, so process scope changes nothing From 9834a9f851a2599e7110f2babafa31aa662da26c Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:59:40 +0400 Subject: [PATCH 039/103] fix: reject library archive file and directory conflicts --- tools/build_web.py | 8 ++++++-- tools/tests/test_build_web.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tools/build_web.py b/tools/build_web.py index 7af1cc6992..cd19113603 100644 --- a/tools/build_web.py +++ b/tools/build_web.py @@ -63,14 +63,18 @@ def publish_libraries(output): files = archive.infolist() if not files or len(files) > 4096: raise ValueError("Invalid library archive") - names = set() + names = {} for file in files: if not file.filename.startswith("libs/") or any(part in ("..", ".", "") for part in file.filename.rstrip("/").split("/")) or "\\" in file.filename: raise ValueError("Invalid library archive layout") name = file.filename.rstrip("/").lower() if name in names: raise ValueError(f"Duplicate library archive path: {file.filename}") - names.add(name) + names[name] = file.is_dir() + for name in names: + parts = name.split("/") + if any(names.get("/".join(parts[:index])) is False for index in range(1, len(parts))): + raise ValueError(f"Library archive file is also used as a directory: {name}") expanded = sum(file.file_size for file in files) if expanded > MAX_EXPANDED_BYTES: raise ValueError("Release archive exceeds maximum expanded size") diff --git a/tools/tests/test_build_web.py b/tools/tests/test_build_web.py index 47c057babc..3a56e49429 100644 --- a/tools/tests/test_build_web.py +++ b/tools/tests/test_build_web.py @@ -41,6 +41,26 @@ class LibraryPublicationTests(unittest.TestCase): report = json.loads(self.manifest.read_text(encoding="utf-8")) self.assertTrue(all(asset["expandedBytes"] == 4096 for asset in report["platforms"].values())) + def test_file_ancestors_are_rejected_before_publication(self): + valid = library_zip([("libs/libc.prx", b"valid")]) + for entries in (("libs/nested", "libs/nested/libc.prx"), + ("libs/NESTED/libc.prx", "libs/nested")): + with self.subTest(entries=entries): + invalid = library_zip([(name, b"content") for name in entries]) + with self.assertRaisesRegex(ValueError, "file is also used as a directory"): + self.publish(invalid, valid) + self.assertEqual(self.manifest.read_bytes(), self.previous) + self.assertEqual(list(self.libraries.iterdir()), [self.existing]) + + + def test_explicit_and_implicit_archive_directories_are_valid(self): + for entries in ([('libs/sub/libc.prx', b'content')], + [('libs/sub/', b''), ('libs/sub/libc.prx', b'content')]): + with self.subTest(entries=entries): + payload = library_zip(entries) + self.publish(payload, payload) + self.assertEqual(json.loads(self.manifest.read_text(encoding="utf-8"))["tag"], "v1") + def setUp(self): temporary = tempfile.TemporaryDirectory() From 8eecd5060055d01fa7a29547e58aff963ef9e0e2 Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:01:02 +0400 Subject: [PATCH 040/103] fix: exclude test export aliases from import audit classification --- tools/import_audit.py | 2 +- tools/tests/test_import_audit.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/import_audit.py b/tools/import_audit.py index ce551a47a7..a75e4384b8 100644 --- a/tools/import_audit.py +++ b/tools/import_audit.py @@ -454,7 +454,7 @@ def source_stubs(source): shared = {} for library in sorted(item for item in Path(source).iterdir() if item.is_dir()): aliases = {} - for file in library.rglob("*.cpp"): + for file in nid_names.source_files(library): for nid, name in nid_names.source_aliases(file.read_text(errors="ignore")): aliases[name] = nid group = progress.scan_library(library) diff --git a/tools/tests/test_import_audit.py b/tools/tests/test_import_audit.py index 9fd1ec2cac..10125129fa 100644 --- a/tools/tests/test_import_audit.py +++ b/tools/tests/test_import_audit.py @@ -224,6 +224,20 @@ class ImportAuditTests(unittest.TestCase): registry.write_text(json.dumps([dict(nid="AAAAAAAAAAA", library="libSceA.prx", **fields)]), encoding="utf-8") self.assertEqual(import_audit.read_registry(registry), [("AAAAAAAAAAA", "libSceA.prx", expected)]) + def test_test_file_aliases_cannot_override_production_stubs(self): + source = self.work.source + text = 'APS5_EXPORT("AAAAAAAAAAA", sceExample);\n' + STUB_SOURCE.format(name="sceExample") + write_source(source, "libSceExample", text) + test = source / "libSceExample" / "tests" / "Alias.cpp" + test.parent.mkdir() + test.write_text('APS5_EXPORT("BBBBBBBBBBB", sceExample);\n', encoding="utf-8") + stubs = import_audit.source_stubs(source) + self.assertEqual(stubs, {("libSceExample.prx", "AAAAAAAAAAA"): "sceExample"}) + records, _ = import_audit.audit([("AAAAAAAAAAA", "libSceExample.prx", False)], + {"AAAAAAAAAAA": ["libSceExample.prx"]}, stubs, + {"libSceExample.prx"}, set()) + self.assertEqual(records[0]["class"], "stub") + def setUp(self): self.work = Workspace() From 26c3a82e8d185d8b22c2b5bd376a4d9bd9bf1c95 Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:08:43 +0400 Subject: [PATCH 041/103] fix(conventions): ignore non-code duplicate export matches --- tools/check_conventions.py | 11 +++++-- ..._check_conventions_wrapped_declarations.py | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/tools/check_conventions.py b/tools/check_conventions.py index 2d62d2dce1..dc4fb2ad5b 100644 --- a/tools/check_conventions.py +++ b/tools/check_conventions.py @@ -173,10 +173,17 @@ class Check: result = subprocess.run(["git", "-c", "core.quotePath=false", "grep", "-nE", "--full-name", pattern, self.head, "--", ":(top)core/libs/prx"], capture_output=True, encoding="utf-8", errors="replace") hits = [line.split(":", 3)[1:] for line in filter(None, result.stdout.split("\n"))] self.read(path for path, _, _ in hits) + sources = {} for path, number, text in hits: - name = EXPORT.search(text).group(1) + if path not in sources: + sources[path] = cpp_text(self.show(path)).split("\n") + source = sources[path] + match = next((match for match in EXPORT.finditer(source[int(number) - 1]) if match.group(1) in exports), None) + if match is None: + continue + name = match.group(1) own, own_number = exports[name] - if library(path) != library(own) and "tests" not in PurePosixPath(path).parts and function_body(self.show(path).splitlines(), int(number) - 1) is not None: + if library(path) != library(own) and "tests" not in PurePosixPath(path).parts and function_body(source, int(number) - 1) is not None: self.report("duplicate-export", own, own_number, f"{name} is also in {library(path)}") def build(self): diff --git a/tools/tests/test_check_conventions_wrapped_declarations.py b/tools/tests/test_check_conventions_wrapped_declarations.py index c05d890dbb..62c6f9642c 100644 --- a/tools/tests/test_check_conventions_wrapped_declarations.py +++ b/tools/tests/test_check_conventions_wrapped_declarations.py @@ -63,6 +63,39 @@ class WrappedDeclarationTests(unittest.TestCase): self.assertEqual(result.returncode, 1, result.stdout + result.stderr) self.assertIn("[duplicate-export]", result.stdout) + def test_multiline_raw_literal_is_not_a_duplicate(self): + source = 'const char* example = R"example(\n' + DEFINITION + ')example";\n' + result = self.check("Export.cpp", source) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("[duplicate-export]", result.stdout) + + def test_block_comment_is_not_a_duplicate(self): + result = self.check("Export.cpp", "/*\n" + DEFINITION + "*/\n") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("[duplicate-export]", result.stdout) + + def test_literal_before_unrelated_export_is_not_a_duplicate(self): + source = 'const char* example = "APS5_VABI sceFooBar()"; int APS5_VABI sceOther() { return 1; }\n' + result = self.check("Export.cpp", source) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("[duplicate-export]", result.stdout) + + def test_line_comment_before_definition_is_not_a_duplicate(self): + result = self.check("Export.cpp", "// " + DEFINITION + "int unrelated() { return 42; }\n") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("[duplicate-export]", result.stdout) + + def test_definition_after_unicode_raw_literal_is_a_duplicate(self): + source = 'const char* text = R"text(first\u2028+middle\u2029+last)text";\n' + DEFINITION + result = self.check("Export.cpp", source) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("[duplicate-export]", result.stdout) + + def test_definition_after_commented_example_is_a_duplicate(self): + result = self.check("Export.cpp", "/*\n" + DEFINITION + "*/\n" + DEFINITION) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(result.stdout.count("[duplicate-export]"), 1, result.stdout) + if __name__ == "__main__": unittest.main() From b808658488ad2ae4022da877f27afbd577c03202 Mon Sep 17 00:00:00 2001 From: mattogepeto Date: Sat, 17 Oct 2026 16:35:15 +0200 Subject: [PATCH 042/103] test(libkernel): expect the FreeBSD errors in the Windows directory tests guest_directory_close expected ENOTDIR from getdents on a descriptor number reused by a regular file, and guest_directory_buffer expected ENOTDIR for descriptor -1. kern_getdirentries returns EINVAL for a vnode that is not a directory and getvnode returns EBADF for a descriptor that is not open, which is what sceKernelGetdirentries now reports on Windows. Written with AI assistance (Claude). Co-Authored-By: Claude Opus 5.5 --- core/libs/tests/GuestDirectoryBuffer.cpp | 2 +- core/libs/tests/GuestDirectoryClose.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/libs/tests/GuestDirectoryBuffer.cpp b/core/libs/tests/GuestDirectoryBuffer.cpp index bfd1beb0dd..8ceba21364 100644 --- a/core/libs/tests/GuestDirectoryBuffer.cpp +++ b/core/libs/tests/GuestDirectoryBuffer.cpp @@ -66,7 +66,7 @@ static void CheckName(const std::u8string& utf8Name) { Require(sceKernelGetdents(directory, buffer.data(), 8) == 0); Require(sceKernelGetdents(directory, buffer.data(), 256) == 0); Require(sceKernelClose(directory) == 0); - Require(sceKernelGetdents(-1, buffer.data(), 256) == SCE_KERNEL_ERROR_ENOTDIR); + Require(sceKernelGetdents(-1, buffer.data(), 256) == SCE_KERNEL_ERROR_EBADF); std::filesystem::remove_all(root); } diff --git a/core/libs/tests/GuestDirectoryClose.cpp b/core/libs/tests/GuestDirectoryClose.cpp index b4ab7b0fc2..68d3d60176 100644 --- a/core/libs/tests/GuestDirectoryClose.cpp +++ b/core/libs/tests/GuestDirectoryClose.cpp @@ -25,7 +25,7 @@ static void Check(bool value, int line) { } #define Require(value) Check((value), __LINE__) -static constexpr int ErrorEnotdir = static_cast(0x80020014u); +static constexpr int ErrorEinval = static_cast(0x80020016u); static void VerifyCloseReuse(const std::filesystem::path& root, const char* scenario, bool useUnderscoreClose) { @@ -49,7 +49,7 @@ static void VerifyCloseReuse(const std::filesystem::path& root, const char* scen Require(file == directory); std::array entries{}; - Require(sceKernelGetdents(file, entries.data(), static_cast(entries.size())) == ErrorEnotdir); + Require(sceKernelGetdents(file, entries.data(), static_cast(entries.size())) == ErrorEinval); Require((useUnderscoreClose ? _close_nid_postfix(file) : close_nid_postfix(file)) == 0); } From d915e5835a6d7c1fda2f53e216a45fbc4a3d8c60 Mon Sep 17 00:00:00 2001 From: mattogepeto Date: Sat, 17 Oct 2026 16:41:22 +0200 Subject: [PATCH 043/103] fix(libkernel): check for a directory with fstat before getdents on Linux The Linux path mapped ENOTDIR from getdents64 to EINVAL, but a pipe or FIFO still failed with ESPIPE from the lseek that reads the base offset first. FreeBSD 11.4's kern_getdirentries returns EINVAL for any descriptor that is not a directory vnode, pipes included, and EBADF for one that is not open. Check the descriptor with fstat before the lseek: EBADF and other host errors are passed through, and anything that is not a directory returns EINVAL. The getdents64 error line is unchanged again, and the technical debt entry is removed since no difference is left. Written with AI assistance (Claude). Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libkernel/File/src/Stdio.cpp | 5 ++++- docs/dev/debt/libkernel.md | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 862aa1f7e8..90767bf005 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -1006,12 +1006,15 @@ int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* bas constexpr std::size_t GuestMaxName = 255; if (buf == nullptr) return SceErrorFromErrno(GUEST_EFAULT); if (nbytes <= 0) return SceErrorFromErrno(GUEST_EINVAL); + struct stat status{}; + if (::fstat(fd, &status) != 0) return SceErrorFromErrno(errno); + if (!S_ISDIR(status.st_mode)) return SceErrorFromErrno(GUEST_EINVAL); const off_t base = ::lseek(fd, 0, SEEK_CUR); if (base < 0) return SceErrorFromErrno(errno); if (basep != nullptr) *basep = static_cast(base); std::vector host(static_cast(nbytes) < 4096 ? 4096 : static_cast(nbytes)); const auto read = ::syscall(SYS_getdents64, fd, host.data(), host.size()); - if (read < 0) return SceErrorFromErrno(errno == ENOTDIR ? GUEST_EINVAL : errno); + if (read < 0) return SceErrorFromErrno(errno); std::size_t written = 0; off_t resume = base; for (long offset = 0; offset < read;) { diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index da7053539c..5791f7ab5e 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -122,7 +122,6 @@ - [system](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) runs no command. It follows FreeBSD 11's `lib/libc/stdlib/system.c` for a sandbox without `/bin/sh`: a null command returns 1 unconditionally, as that version does, and any command returns the wait status of a child that could not exec the shell and called `_exit(127)` (`0x7f00`). Whether the console's application sandbox has no shell, and whether `vfork` succeeds there (FreeBSD returns -1 when it fails), is unverified - [sceKernelWriteThrottlingStatus](../../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown - [Windows directory descriptors](../../../core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp) (libkernel) take one snapshot of the directory, on the first `getdents`, `getdirentries` or non-zero `lseek` (`SEEK_SET` or `SEEK_CUR`), and keep it until it is discarded; `lseek` to offset 0 only discards the snapshot, and the next read lists the directory again, so every rewind costs O(entries) on the next read. `lseek` takes the offset as an entry index and `SEEK_CUR` with offset 0 only reports the index. `SEEK_END` throws, where the NUL descriptor used to return 0, because FreeBSD defines it relative to the directory size, which is not modelled. A negative result returns `EINVAL` and a `SEEK_CUR` offset that overflows returns `EOVERFLOW`, as FreeBSD `vn_seek` reports them. Names whose UTF-8 encoding exceeds the one-byte `d_namlen` limit are not representable in this guest directory-record layout: `getdents` returns `ENAMETOOLONG` without consuming the entry. Repeated calls therefore report the same error and entries after that name cannot be reached through this descriptor; FreeBSD's `getdents` behavior for host filenames that exceed its `MAXNAMLEN` is not verified. `getdirentries` still reports 0 through `basep` instead of the position of the block it read, so only 0 and values returned by `lseek` are valid offsets; FreeBSD offsets are filesystem cookies -- [getdents and getdirentries](../../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) return `EINVAL` for a descriptor that is not a directory, as [`kern_getdirentries`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/kern/vfs_syscalls.c) does for a vnode that is not `VDIR`. On Linux a host pipe or FIFO still fails with `ESPIPE`, from the `lseek` that reads the base offset before `getdents64`, where `getvnode` returns `EINVAL` for a non-vnode descriptor - [/dev/random and /dev/urandom](../../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) (libkernel) opened for reading return a descriptor backed by the host null device whose reads are filled with host OS entropy (`getentropy`, `rand_s` on Windows); both paths behave the same, never block, and positioned reads ignore the offset (FreeBSD `random(4)`). Only the exact paths are recognised, `fstat`, `lseek` and `mmap` see the null device, and opening them for writing throws - [pthread_mutexattr_getpshared and pthread_mutexattr_setpshared](../../../core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp) (libkernel) store and report private or shared attributes, following FreeBSD 11 libthr [thr_mutexattr.c](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/libthr/thread/thr_mutexattr.c). Invalid values return `EINVAL` without changing the attribute. Guest threads execute in one host process; sharing mutexes across processes is not modeled. Console behavior and invalid-attribute error precedence are unverified. - [pthread_attr_getinheritsched, pthread_attr_setinheritsched, pthread_attr_getscope, pthread_attr_setscope, pthread_attr_getstackaddr](../../../core/libs/prx/libkernel/Pthread/Posix/Attr.cpp) (libkernel) follow FreeBSD 11 libthr ([thr_attr.c](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/lib/libthr/thread/thr_attr.c), [thr_create.c](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/lib/libthr/thread/thr_create.c)), not measured on the console: the scope defaults to `PTHREAD_SCOPE_SYSTEM` (2), only it and `PTHREAD_SCOPE_PROCESS` (0) are accepted and other values return `EINVAL`, and an inherit value other than `PTHREAD_INHERIT_SCHED` (4) or `PTHREAD_EXPLICIT_SCHED` (0) returns `ENOTSUP`. `pthread_attr_getstackaddr` returns the configured stack address pointer (null by default). A null output of `pthread_attr_getinheritsched` throws, since libthr writes through it unchecked. A thread keeps the scope it was created with, taken from the creating thread under `PTHREAD_INHERIT_SCHED`, and `pthread_attr_get_np` and `scePthreadAttrGet` report it, but every guest thread is a host thread at system scope, so process scope changes nothing From f83b6c0ee0c257bc405ce621a530612ee7d4305d Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:41:57 +0400 Subject: [PATCH 044/103] fix: report oracle CLI input errors before running tools --- tools/hw-oracle/hw_oracle.py | 24 +++++++++++++++--------- tools/hw-oracle/test_hw_oracle.py | 27 ++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/tools/hw-oracle/hw_oracle.py b/tools/hw-oracle/hw_oracle.py index b81f53d966..0f43800a93 100644 --- a/tools/hw-oracle/hw_oracle.py +++ b/tools/hw-oracle/hw_oracle.py @@ -100,7 +100,7 @@ def assemble(body, work, wave64, *, ieee, denorm32, denorm16, dx10_clamp, round3 return work / "k.co" -def run(body, rows, extra=b"", wave64=False, coarse=False, *, ieee, denorm32, denorm16, dx10_clamp, round32, round16, fp16_overflow, lds=DEFAULT_LDS): +def validated_rows(rows): validated = [] for index, row in enumerate(rows, 1): try: @@ -110,7 +110,11 @@ def run(body, rows, extra=b"", wave64=False, coarse=False, *, ieee, denorm32, de if len(values) != 4 or any(value < 0 or value > 0xffffffff for value in values): raise ValueError(f"row {index} needs 4 unsigned 32-bit integers") validated.append(values) - rows = validated + return validated + + +def run(body, rows, extra=b"", wave64=False, coarse=False, *, ieee, denorm32, denorm16, dx10_clamp, round32, round16, fp16_overflow, lds=DEFAULT_LDS): + rows = validated_rows(rows) wave = 64 if wave64 else 32 padded = rows + [(0, 0, 0, 0)] * (-len(rows) % wave) env = dict(os.environ) @@ -161,13 +165,15 @@ def main(): parser.add_argument("--outs", type=int, choices=range(1, 17), default=16, metavar="N", help="print v10..v(10+N-1)") parser.add_argument("--json", action="store_true", help="emit inputs, results and run settings as JSON; target is the assembly target, not a GPU model or driver") args = parser.parse_args() - rows = [tuple(parse_word(v) for v in line.split()) for line in args.rows.read_text().splitlines() if line.strip()] - if not rows: - parser.error("rows file must contain at least one input row") - if any(len(r) != 4 for r in rows): - sys.exit("every row needs 4 values") - extra = args.extra.read_bytes() if args.extra else b"" - body = args.body.read_text() + try: + rows = validated_rows(tuple(parse_word(v) for v in line.split()) + for line in args.rows.read_text(encoding="utf-8").splitlines() if line.strip()) + if not rows: + parser.error("rows file must contain at least one input row") + extra = args.extra.read_bytes() if args.extra else b"" + body = args.body.read_text(encoding="utf-8") + except (OSError, ValueError) as error: + parser.error(str(error)) modes = {name: getattr(args, name) for name in ("ieee", "denorm32", "denorm16", "dx10_clamp", "round32", "round16", "fp16_overflow")} results = run(body, rows, extra, args.wave64, args.coarse, **modes, lds=args.lds) if args.json: diff --git a/tools/hw-oracle/test_hw_oracle.py b/tools/hw-oracle/test_hw_oracle.py index a3321dc3e1..d40e731880 100644 --- a/tools/hw-oracle/test_hw_oracle.py +++ b/tools/hw-oracle/test_hw_oracle.py @@ -79,6 +79,30 @@ class OracleCacheTests(unittest.TestCase): class InputRowsTests(unittest.TestCase): + def test_cli_reports_invalid_input_before_tool_lookup(self): + with tempfile.TemporaryDirectory() as tmp: + body, rows = Path(tmp) / "body.s", Path(tmp) / "rows.txt" + body.write_text("s_nop 0", encoding="utf-8") + argv = ["hw_oracle.py", str(body), str(rows)] + argv += [part for name, value in MODES.items() + for part in ["--" + name.replace("_", "-"), str(value)]] + for contents in (None, b"not a number\n", b"1 2 3\n", b"-1 0 0 0\n", b"4294967296 0 0 0\n", b"\xff"): + with self.subTest(contents=contents): + rows.unlink(missing_ok=True) + if contents is not None: + rows.write_bytes(contents) + with patch("sys.argv", argv), patch.object(hw_oracle, "assemble") as assemble, \ + patch.object(hw_oracle, "oracle") as oracle, \ + contextlib.redirect_stdout(io.StringIO()) as output, \ + contextlib.redirect_stderr(io.StringIO()) as errors, \ + self.assertRaises(SystemExit) as exited: + hw_oracle.main() + self.assertEqual(exited.exception.code, 2) + self.assertIn("error:", errors.getvalue()) + self.assertEqual(output.getvalue(), "") + assemble.assert_not_called() + oracle.assert_not_called() + def test_successful_dispatch_without_output_cannot_reuse_previous_results(self): with tempfile.TemporaryDirectory() as directory: runner = Path(directory) / "runner.py" @@ -257,8 +281,9 @@ class JsonReportTests(unittest.TestCase): self.rows.write_text("0x100000000 0 0 0\n", encoding="utf-8") output = io.StringIO() with patch.object(hw_oracle, "target", side_effect=AssertionError("target lookup")) as target: - with self.assertRaisesRegex(ValueError, "row 1"): + with self.assertRaises(SystemExit) as exited: self.cli("--json", output=output) + self.assertEqual(exited.exception.code, 2) target.assert_not_called() self.assertEqual(output.getvalue(), "") From 2578949658293847db4357c57ede28fbeb7f8c21 Mon Sep 17 00:00:00 2001 From: milescoviello Date: Sat, 17 Oct 2026 14:46:37 +0000 Subject: [PATCH 045/103] fix(agc): run the AGC GPU tests in an ANYPS5_ENABLE_ASAN build After #7147 the CPU-side tests pass under AddressSanitizer, but 18 of the 518 AGC driver and shader tests still failed there with ASAN_OPTIONS=detect_leaks=0: - Guest memory is raw memory, not a C++ object. ReadRawShader reads a shader's first 256 bytes before it knows the program's length, and ShaderMemory copies the 64-byte block around each word it is asked for. The tests place shaders and descriptors in small globals and locals, so those reads ran past them into ASan's redzones (13 tests, global- and stack-buffer-overflow). GuestMemory::CopyGuestBytes copies guest bytes with memcpy, and in an ASan build without instrumentation; Read and ShaderMemory's block load use it. - The depth surface reuse checks in agc_driver_graphics use the made-up guest addresses 0x80000000 and 0x81000000, which lie in ASan's low shadow, so the HTILE read there looked mapped and read shadow memory. They now use 0x30000000 and 0x31000000. - agc_driver_flat_store maps its watched blocks at 8 TiB and 32 TiB to cover page numbers on both sides of 16 TiB. 8 TiB is in ASan's high shadow; the ASan build uses 0x60000000 for the low one. - agc_driver_gpu_map_on_fiber requires the fiber to use at most 8 KiB of its stack while its mapping is imported on a host thread. ASan's redzones make the same frames 9200 bytes; the ASan build allows 16 KiB. It also requires a mapping on an ordinary thread to import on that thread's stack. Like MeasureStackReserve before #7115, the driver's inline-import check (Lifecycle.cpp) and onOwnLiveStack (GuestMemory.cpp) took the address of a local, which ASan's fake stack moves off the thread's stack, so the ASan build always imported on a host thread. Both now measure from __builtin_frame_address(0). Co-Authored-By: Claude Opus 5.5 --- .../Execution/include/GuestMemory.hpp | 1 + .../Execution/src/Driver/Lifecycle.cpp | 3 +-- .../Execution/src/GuestMemory.cpp | 22 ++++++++++++++++--- .../Execution/src/ShaderMemory.cpp | 2 +- .../tests/DepthSurfaceReuse.cpp | 4 ++-- .../tests/execution/FlatStore.cpp | 4 +++- .../tests/execution/GpuMapOnFiber.cpp | 4 ++++ 7 files changed, 31 insertions(+), 9 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index 2388684997..5e8ce98cc3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -26,6 +26,7 @@ struct Commitment { bool whole = false; }; Commitment DescribeCommitted(std::uint64_t address, std::size_t bytes, bool writable = false); +void CopyGuestBytes(void* destination, const void* source, std::size_t bytes); void Read(std::uint64_t address, std::span destination, std::size_t alignment = 1); // Variants for resources in heaps the guest commits on demand (partially resident textures, GPU // scratch heaps): uncommitted pages read as zeros, are not compared, and are never stored. diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp index 91c50d6f3f..5a5ff651f3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp @@ -73,8 +73,7 @@ bool StackHoldsImport() { const auto low = bounds.low; const auto high = bounds.high; #endif - const volatile unsigned char marker = 0; - const auto frame = reinterpret_cast(&marker); + const auto frame = reinterpret_cast(__builtin_frame_address(0)); return frame >= low && frame < high && frame - low >= InlineImportStackBytes; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index d0859ab061..c6199b4bfe 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -657,8 +657,7 @@ bool onOwnLiveStack(std::uintptr_t address, std::size_t bytes) { } }; thread_local const Bounds bounds; - const volatile unsigned char marker = 0; - const auto frame = reinterpret_cast(&marker); + const auto frame = reinterpret_cast(__builtin_frame_address(0)); return bounds.high != 0 && frame >= bounds.low && frame < bounds.high && address >= frame && address < bounds.high && bytes <= bounds.high - address; } @@ -2100,9 +2099,26 @@ void Read(std::uint64_t address, std::span destination, std::size_t a if (trace) std::fprintf(stderr, "[unreadable] read from +0x%llx\n", ModuleOffset(__builtin_return_address(0))); throw; } - std::memcpy(destination.data(), source, destination.size()); + CopyGuestBytes(destination.data(), source, destination.size()); } +#ifdef ANYPS5_ENABLE_ASAN +[[gnu::no_sanitize("address")]] void CopyGuestBytes(void* destination, const void* source, std::size_t bytes) { + auto* to = static_cast(destination); + const auto* from = static_cast(source); + std::size_t index = 0; + for (; index + sizeof(std::uint64_t) <= bytes; index += sizeof(std::uint64_t)) { + const std::uint64_t word = *reinterpret_cast(from + index); + std::memcpy(to + index, &word, sizeof(word)); + } + for (; index < bytes; ++index) to[index] = static_cast(from)[index]; +} +#else +void CopyGuestBytes(void* destination, const void* source, std::size_t bytes) { + std::memcpy(destination, source, bytes); +} +#endif + namespace { void copyCommitted(std::uint64_t address, std::span destination) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index ffbecd5b70..d658572a98 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -219,7 +219,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val const auto index = static_cast((address % PageBytes) / sizeof(*value)); if (!page.valid.Test(index) && page.lazy) { const auto first = index / BlockWords * BlockWords; - std::memcpy(&page.words[first], reinterpret_cast((address & ~static_cast(PageBytes - 1)) + first * sizeof(std::uint32_t)), BlockWords * sizeof(std::uint32_t)); + GuestMemory::CopyGuestBytes(&page.words[first], reinterpret_cast((address & ~static_cast(PageBytes - 1)) + first * sizeof(std::uint32_t)), BlockWords * sizeof(std::uint32_t)); page.valid.SetBlock(index); } if (!page.valid.Test(index)) { diff --git a/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp b/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp index ebb60efe01..006e4efd53 100644 --- a/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp @@ -458,8 +458,8 @@ void RunDepthSurfaceReuseTests() { const auto otherWithA = pairing(otherDepth, stencilA); Require(imageCopies.size() == 1 && imageCopies[0] == std::make_tuple(withA, otherWithA, static_cast(VK_IMAGE_ASPECT_STENCIL_BIT)), "a stencil plane paired with another depth plane must carry its stencil over"); { - constexpr std::uint64_t arrayBase = 0x80000000; - constexpr std::uint64_t htile = 0x81000000; + constexpr std::uint64_t arrayBase = 0x30000000; + constexpr std::uint64_t htile = 0x31000000; constexpr VkExtent2D extent{1024, 512}; const auto sliceHtile = HtileMetadataBytes(extent); DepthTarget first{arrayBase, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp index f249083ba2..2d6da41fd0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp @@ -31,7 +31,9 @@ constexpr std::uint32_t MaskedThreads = 16; constexpr std::size_t BlockBytes = 65536; constexpr std::uint32_t CheckedBytes = 0x2000; constexpr std::uint8_t Fill = 0xcd; -#ifdef __linux__ +#if defined(__linux__) && defined(ANYPS5_ENABLE_ASAN) +constexpr std::array GuestWindowAddresses{0x60000000, 0x200000000000}; +#elif defined(__linux__) constexpr std::array GuestWindowAddresses{0x80000000000, 0x200000000000}; #endif diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GpuMapOnFiber.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GpuMapOnFiber.cpp index aefdae5293..5a4da94b89 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GpuMapOnFiber.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GpuMapOnFiber.cpp @@ -34,7 +34,11 @@ using AgcDriver::Graphics::Require; constexpr std::size_t HostStackContextBytes = 16 * 1024; constexpr std::size_t ContextBytes = 64 * 1024; constexpr std::size_t MappingBytes = 4 * 65536; +#ifdef ANYPS5_ENABLE_ASAN +constexpr std::size_t FiberStackLimit = 16 * 1024; +#else constexpr std::size_t FiberStackLimit = 8 * 1024; +#endif constexpr std::uint64_t Fill = 0x5aa5c33c5aa5c33cull; alignas(16) std::array FiberContext; From 6ce6850d12a8b55d80381095ca23736f9dd7116c Mon Sep 17 00:00:00 2001 From: simonbronk Date: Sat, 17 Oct 2026 17:23:32 +0200 Subject: [PATCH 046/103] fix(libSceNet): guard recvfrom sender outputs before receiving AI-assisted: OpenAI Codex. Windows native checks: 17 related CTests and 10 regression repetitions pass. No affected title verified. --- core/libs/prx/libSceNet/Export.cpp | 6 ++++ core/libs/tests/GuestSocketHostWrite.cpp | 45 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index 8fd8578762..e5537bf33d 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -987,6 +987,8 @@ int64_t APS5_VABI sceNetRecvfrom(int s, void* buf, size_t len, int flags, void* if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; } + const GuestArena::HostWrite lengthOutput(from == nullptr ? nullptr : fromlen, from == nullptr ? 0 : sizeof(*fromlen)); + if (!lengthOutput.Open()) return fail(NET_EFAULT); #ifdef _WIN32 if (const int wait = wait_for_local_address(s, socket, len == 0)) { if (wait != RECEIVE_NOTHING) return fail(wait); @@ -994,6 +996,10 @@ int64_t APS5_VABI sceNetRecvfrom(int s, void* buf, size_t len, int flags, void* return 0; } #endif + const auto addressBytes = from != nullptr && socket.type == NET_SOCK_DGRAM ? + std::min(*fromlen, socket.family == NET_AF_INET ? 16u : 28u) : 0u; + const GuestArena::HostWrite addressOutput(addressBytes == 0 ? nullptr : from, addressBytes); + if (!addressOutput.Open()) return fail(NET_EFAULT); const GuestArena::HostWrite destination(len == 0 ? nullptr : buf, len); if (!destination.Open()) return fail(NET_EFAULT); sockaddr_storage peer{}; diff --git a/core/libs/tests/GuestSocketHostWrite.cpp b/core/libs/tests/GuestSocketHostWrite.cpp index 06e23b986e..982eb625d7 100644 --- a/core/libs/tests/GuestSocketHostWrite.cpp +++ b/core/libs/tests/GuestSocketHostWrite.cpp @@ -243,6 +243,51 @@ int main(int argc, char** argv) { if (mask & 1) Require(Protection(address) == PAGE_READONLY, "sender address rearmed"); if (mask & 2) Require(Protection(length) == PAGE_READONLY, "sender length rearmed"); } + if (api == 5) { + for (const bool readOnlyAddress : {true, false}) { + auto* address = static_cast(outputMapping) + 16; + auto* length = reinterpret_cast(static_cast(outputMapping) + page + 16); + { + const GuestArena::HostWrite addressDestination(address, 32); + const GuestArena::HostWrite lengthDestination(length, sizeof(*length)); + Require(addressDestination.Open() && lengthDestination.Open(), "initialize refused sender outputs"); + std::memset(address, 0xa5, 32); + *length = 32; + } + void* refused = static_cast(outputMapping) + (readOnlyAddress ? 0 : page); + Require(sceKernelMprotect(refused, page, 1) == 0, "make one sender output read-only"); + send(); + waitForPacket(); + std::array recovery{}; + Require(receive(recovery.data(), recovery.size(), 0, address, length) == static_cast(0x8041010eu) && *error == 14, + "read-only sender output fails with EFAULT"); + Require(*length == 32 && std::all_of(address, address + 32, [](unsigned char value) { return value == 0xa5; }), + "refused sender outputs remain unchanged"); + Require(receive(recovery.data(), recovery.size(), 0, nullptr, nullptr) == sizeof(payload) && + std::memcmp(recovery.data(), payload, sizeof(payload)) == 0, "refused sender output preserves queued data"); + Require(sceKernelMprotect(refused, page, 3) == 0, "restore sender output permission"); + } + auto* address = static_cast(outputMapping) + 16; + Require(sceKernelMprotect(outputMapping, page, 1) == 0, "make an unused sender address read-only"); + std::uint32_t zero = 0; + std::array recovery{}; + send(); + waitForPacket(); + Require(receive(recovery.data(), recovery.size(), 0, address, &zero) == sizeof(payload) && zero == 0 && + std::memcmp(recovery.data(), payload, sizeof(payload)) == 0, "zero-capacity sender address needs no write access"); + Require(sceKernelMprotect(outputMapping, page, 3) == 0, "restore unused sender address permission"); + address = static_cast(outputMapping) + page - senderAddress.size(); + Require(sceKernelMprotect(static_cast(outputMapping) + page, page, 1) == 0, + "make bytes beyond the sender address read-only"); + std::uint32_t oversized = 32; + send(); + waitForPacket(); + Require(receive(recovery.data(), recovery.size(), 0, address, &oversized) == sizeof(payload) && + oversized == senderAddress.size() && std::memcmp(address, senderAddress.data(), oversized) == 0, + "guard only the sender address bytes the receive writes"); + Require(sceKernelMprotect(static_cast(outputMapping) + page, page, 3) == 0, + "restore bytes beyond the sender address"); + } Require(sceKernelMunmap(outputTwin, page * 2) == 0, "unmap the second sender output mapping"); Require(sceKernelMunmap(outputMapping, page * 2) == 0, "unmap sender outputs"); Require(sceKernelReleaseDirectMemory(outputPhysical, page * 2) == 0, "release sender output memory"); From c652880f49e6a8d37185cfde5f87e36e995fbec8 Mon Sep 17 00:00:00 2001 From: thibaut Date: Sat, 17 Oct 2026 17:08:38 +0200 Subject: [PATCH 047/103] perf(agc): grow the resource cache while the keys it evicts come back Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XBXM4pgkMHYEtqN1EQni2P --- .../Graphics/include/ShaderResources.hpp | 11 ++ .../Graphics/src/ShaderResources.cpp | 49 +++++- .../tests/execution/ResourceCacheGrowth.cpp | 139 ++++++++++++++++++ 3 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 8e25d227c9..30e1012671 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -629,12 +629,17 @@ public: static std::uint64_t Touches(); void Clear(); std::size_t Size() const; + std::size_t Capacity() const; std::uint64_t TableElements() const; static std::uint64_t TableElementBudget(); + static constexpr std::size_t InitialEntries = 1024; + static constexpr std::size_t MaximumEntries = 16384; private: void erase(const Key& key); void noteMiss(const Key& key); + void noteEvicted(const Key& key); + void noteReturn(const Key& key); struct KeyHash { std::size_t operator()(const Key& key) const noexcept { std::uint64_t hash = 14695981039346656037ull; @@ -645,6 +650,12 @@ private: mutable std::mutex mutex; std::list>> entries; std::unordered_map index; + std::size_t capacity = 0; + bool fixedCapacity = false; + std::list evictedOrder; + std::unordered_map evictedKeys; + std::size_t evictionsSinceCheck = 0; + std::size_t returnsSinceCheck = 0; }; // The process-wide cache the device and the draw path share (keys name the device; a device clears diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index ab7d9b47ce..fc09e2f367 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -3145,6 +3145,7 @@ std::shared_ptr ResourceCache::Find(const Key& key) { std::lock_guard lock(mutex); const auto found = index.find(key); if (found == index.end()) { + noteReturn(key); noteMiss(key); return nullptr; } @@ -3152,6 +3153,35 @@ std::shared_ptr ResourceCache::Find(const Key& key) { return found->second->second; } +void ResourceCache::noteEvicted(const Key& key) { + if (fixedCapacity) return; + const auto hash = KeyHash{}(key); + if (const auto found = evictedKeys.find(hash); found != evictedKeys.end()) evictedOrder.erase(found->second); + evictedOrder.push_front(hash); + evictedKeys[hash] = evictedOrder.begin(); + while (evictedOrder.size() > capacity) { + evictedKeys.erase(evictedOrder.back()); + evictedOrder.pop_back(); + } + if (++evictionsSinceCheck < capacity / 4u) return; + const bool churning = returnsSinceCheck * 2u > evictionsSinceCheck; + evictionsSinceCheck = 0; + returnsSinceCheck = 0; + if (!churning || capacity >= MaximumEntries) return; + const auto previous = capacity; + capacity = std::min(capacity * 2u, MaximumEntries); + std::fprintf(stderr, "[rescache] capacity %zu -> %zu entries: keys it evicted keep coming back\n", previous, capacity); +} + +void ResourceCache::noteReturn(const Key& key) { + if (fixedCapacity) return; + const auto found = evictedKeys.find(KeyHash{}(key)); + if (found == evictedKeys.end()) return; + evictedOrder.erase(found->second); + evictedKeys.erase(found); + ++returnsSinceCheck; +} + namespace { void traceMiss(ChurnProfile& churn, bool draw, std::uint64_t variants, const ContentKeyDiff* parts) { @@ -3279,12 +3309,14 @@ void ResourceCache::Insert(const Key& key, std::shared_ptr reso } entries.emplace_front(key, std::move(resources)); index.emplace(key, entries.begin()); - static const std::size_t capacity = [] { + if (capacity == 0) { const char* value = std::getenv("APS5_RESOURCE_CACHE_ENTRIES"); - const auto parsed = value ? std::strtoull(value, nullptr, 10) : 1024ull; - return static_cast(parsed != 0 ? parsed : 1024ull); - }(); + const auto parsed = value ? std::strtoull(value, nullptr, 10) : 0ull; + fixedCapacity = parsed != 0; + capacity = fixedCapacity ? static_cast(parsed) : InitialEntries; + } while (entries.size() > capacity) { + noteEvicted(entries.back().first); if (evicted != nullptr) evicted->push_back(std::move(entries.back().second)); index.erase(entries.back().first); entries.pop_back(); @@ -3348,6 +3380,10 @@ void ResourceCache::Clear() { std::lock_guard lock(mutex); index.clear(); entries.clear(); + evictedKeys.clear(); + evictedOrder.clear(); + evictionsSinceCheck = 0; + returnsSinceCheck = 0; } std::size_t ResourceCache::Size() const { @@ -3355,6 +3391,11 @@ std::size_t ResourceCache::Size() const { return entries.size(); } +std::size_t ResourceCache::Capacity() const { + std::lock_guard lock(mutex); + return capacity == 0 ? InitialEntries : capacity; +} + std::uint64_t ResourceCache::TableElementBudget() { static const std::uint64_t elements = [] { const char* value = std::getenv("APS5_RESOURCE_CACHE_TABLE_ELEMENTS"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp new file mode 100644 index 0000000000..fbb0ccec87 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp @@ -0,0 +1,139 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::uint32_t Dispatches = 1536; +constexpr std::uint32_t Passes = 4; +constexpr std::size_t SrtBytes = 64; +constexpr std::size_t OutputBytes = 256; +constexpr std::size_t SrtOffset = 4096; +constexpr std::size_t OutputOffset = SrtOffset + static_cast(Dispatches) * SrtBytes; +constexpr std::size_t BlockBytes = (OutputOffset + static_cast(Dispatches) * OutputBytes + 65535) & ~static_cast(65535); +constexpr std::size_t InitialEntries = 1024; + +constexpr std::array StoreCode{0xf4080100u, 0xfa000000u, 0xbf8c0000u, 0x7e020281u, 0xe0700000u, 0x80010100u, 0xbf810000u}; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); +#else + constexpr std::uintptr_t alignment = 65536; + void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast(aligned + BlockBytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes); + } +#endif + Require(block != nullptr, "resource cache growth: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint64_t At(std::size_t offset) const { return static_cast(reinterpret_cast(block + offset)); } + std::uint8_t* Bytes(std::size_t offset) const { return block + offset; } + +private: + std::uint8_t* block = nullptr; +}; + +std::vector Pm4(std::uint32_t opcode, std::initializer_list body) { + std::vector words{0xc0000000u | (static_cast(body.size() - 1) << 16u) | (opcode << 8u)}; + words.insert(words.end(), body); + return words; +} + +void Prepare(const GuestBlock& block) { + std::memcpy(block.Bytes(0), StoreCode.data(), sizeof(StoreCode)); + for (std::uint32_t dispatch = 0; dispatch < Dispatches; ++dispatch) { + const auto output = block.At(OutputOffset + static_cast(dispatch) * OutputBytes); + const std::array descriptor{static_cast(output), static_cast((output >> 32u) & 0xffffu), static_cast(OutputBytes), 0x31016facu}; + std::memcpy(block.Bytes(SrtOffset + static_cast(dispatch) * SrtBytes), descriptor.data(), sizeof(descriptor)); + } + std::memset(block.Bytes(OutputOffset), 0, static_cast(Dispatches) * OutputBytes); +} + +void RunPass(const GuestBlock& block) { + const auto kernel = block.At(0); + std::vector words; + for (std::uint32_t dispatch = 0; dispatch < Dispatches; ++dispatch) { + const auto srt = block.At(SrtOffset + static_cast(dispatch) * SrtBytes); + for (const auto& packet : {Pm4(0x76, {0x20c, static_cast(kernel >> 8u)}), Pm4(0x76, {0x20d, static_cast(kernel >> 40u) & 0xffu}), Pm4(0x76, {0x207, 64}), Pm4(0x76, {0x208, 1}), Pm4(0x76, {0x209, 1}), Pm4(0x76, {0x213, 2u << 1u}), Pm4(0x76, {0x240, static_cast(srt)}), Pm4(0x76, {0x241, static_cast(srt >> 32u)}), Pm4(0x15, {1, 1, 1, 0x41})}) { + words.insert(words.end(), packet.begin(), packet.end()); + } + } + Packet dcb{words.data(), static_cast(words.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&dcb) == 0, "resource cache growth: the dispatch submission failed"); + AgcDriverWaitIdle_nid_postfix(); +} + +void WorkingSetLargerThanTheInitialBound(const GuestBlock& block) { + auto& cache = AgcDriver::Graphics::SharedResourceCache(); + for (std::uint32_t pass = 0; pass < Passes; ++pass) { + RunPass(block); + if (pass != 0) continue; + for (std::uint32_t dispatch = 0; dispatch < Dispatches; ++dispatch) { + std::uint32_t stored = 0; + AgcDriver::GuestMemory::Read(block.At(OutputOffset + static_cast(dispatch) * OutputBytes), std::as_writable_bytes(std::span(&stored, 1)), 4); + Require(stored == 1u, "resource cache growth: dispatch " + std::to_string(dispatch) + " stored " + std::to_string(stored) + ", expected 1"); + } + } + const auto kept = cache.Size(); + Require(kept >= Dispatches, "resource cache growth: " + std::to_string(Dispatches) + " dispatches with distinct resources repeated " + std::to_string(Passes) + " times in turn, but the resource cache kept only " + std::to_string(kept) + " entries (initial bound " + std::to_string(InitialEntries) + "), so every pass rebuilt every set"); +} + +} + +int main() { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + GuestBlock block; + Prepare(block); + WorkingSetLargerThanTheInitialBound(block); + AgcDriverShutdown_nid_postfix(); + std::puts("resource cache growth tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + AgcDriverShutdown_nid_postfix(); + return 1; + } +} From 6e0886d76d0300960139ee806bbee7b2167f206d Mon Sep 17 00:00:00 2001 From: milescoviello Date: Sat, 17 Oct 2026 16:32:08 +0000 Subject: [PATCH 048/103] perf(shader): drop the result flush of a product behind its tiny-product check A v_mul_f32 in a FLOAT_MODE that flushes f32 denormals emits the nearest-even tininess check, the NaN result selection and then a result flush. The check already turns every product that rounds below 2^-126 into a zero of its sign, so the flush can't change what it keeps. DenormalFlushEliminator now proves that: a select arm guarded by the tiny-product check of the same denormal-free factors, an arm a select takes only when its own bits are NaN, and an and with a mask that has no significand bits are denormal-free. Co-Authored-By: Claude Opus 5.5 --- .../tests/DenormalFlushElimination.cpp | 123 ++++++++++++++++++ .../src/DenormalFlushEliminator.cpp | 62 ++++++++- docs/dev/debt/shader.md | 1 + 3 files changed, 183 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp b/core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp index 3db6a8ce08..cc952f5ea9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp @@ -1,11 +1,20 @@ #include "IntermediateRepresentation/IrProgram.hpp" #include "Optimization/DenormalFlushEliminator.hpp" +#include "Recompiler.hpp" +#include +#include #include #include #include #include #include +#include +#include +#include +#include +#include #include +#include using namespace ShaderRecompiler; @@ -80,6 +89,46 @@ bool run(const char* name, const std::function& test) { } } +std::vector Recompiled(std::span code) { + const std::array userData{0x10000000u, 0x00100000u, 0x40u, 0x00027facu}; + const std::array capabilities{1u, 61u}; + const std::array extensions{"SPV_KHR_storage_buffer_storage_class"}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}}; + request.context.waveSize = 32; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00403000u; + request.target.spirvVersion = 0x00010600u; + request.target.subgroupSize = 32u; + request.target.bdaAbiVersion = 1; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.maxWorkgroupSize = {1024u, 1024u, 64u}; + request.target.maxWorkgroupInvocations = 1024; + request.target.maxWorkgroupSharedMemoryBytes = 49152; + request.layout = {0, 0, 0, 128}; + request.useCache = false; + const auto& words = Recompile(request).spirv.Words(); + return {words.begin(), words.end()}; +} + +std::size_t ExponentMasks(std::span words) { + if (words.size() < 5 || words[0] != spv::MagicNumber) throw std::runtime_error("the recompiler returned no SPIR-V module"); + std::set exponents; + std::size_t masks = 0; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> spv::WordCountShift; + if (count == 0 || count > words.size() - offset) throw std::runtime_error("the recompiler returned a malformed SPIR-V instruction"); + const auto opcode = words[offset] & spv::OpCodeMask; + if (opcode == spv::OpConstant && count == 4 && words[offset + 3] == 0x7f800000u) exponents.insert(words[offset + 2]); + if (opcode == spv::OpBitwiseAnd && count == 5 && (exponents.contains(words[offset + 3]) || exponents.contains(words[offset + 4]))) ++masks; + offset += count; + } + return masks; +} + } int main() { @@ -252,5 +301,79 @@ int main() { return loop(true) && !loop(false); }); + passed &= run("a product behind its round-to-nearest tiny-product guard needs no flush, one behind another guard does", [] { + Builder b; + const auto value = [&](IrValue& bits) -> IrValue& { return b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&bits}); }; + const auto bitsOf = [&](IrValue& number) -> IrValue& { return b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&number}); }; + const auto guarded = [&](IrValue& lhs, IrValue& rhs, IrValue& guardLhs, IrValue& guardRhs, std::uint32_t threshold) -> IrValue& { + auto& product = b.emit(IrOpcode::FPMul32, IrType::F32, {&lhs, &rhs}); + auto& scaled = b.emit(IrOpcode::FPMul32, IrType::F32, {&b.emit(IrOpcode::FPMul32, IrType::F32, {&guardLhs, &b.constant(0x5f800000u)}), &guardRhs}); + auto& magnitude = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&bitsOf(scaled), &b.constant(0x7fffffffu)}); + auto& tiny = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.emit(IrOpcode::ISub32, IrType::U32, {&magnitude, &b.constant(1u)}), &b.constant(threshold)}); + auto& sign = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::BitwiseXor32, IrType::U32, {&bitsOf(lhs), &bitsOf(rhs)}), &b.constant(0x80000000u)}); + return b.flush(bitsOf(b.emit(IrOpcode::SelectF32, IrType::F32, {&tiny, &value(sign), &product}))); + }; + auto& x = value(b.flush(b.unknown())); + auto& y = value(b.flush(b.unknown())); + auto& raw = value(b.unknown()); + IrValue* removed[] = {&guarded(x, y, x, y, 0x207fffffu), &guarded(x, y, y, x, 0x207fffffu)}; + IrValue* kept[] = {&guarded(x, y, x, raw, 0x207fffffu), &guarded(x, y, x, y, 0x20800000u), &guarded(x, raw, x, raw, 0x207fffffu)}; + for (auto* flushed : removed) b.keep(*flushed); + for (auto* flushed : kept) b.keep(*flushed); + const bool counted = b.eliminate() == 2u; + bool flushes = true; + for (auto* flushed : removed) flushes &= !flushed->HasUses(); + for (auto* flushed : kept) flushes &= flushed->HasUses(); + return counted && flushes; + }); + + passed &= run("an arm a select takes only when its bits are NaN needs no flush, one under another value's NaN test does", [] { + Builder b; + auto& x = b.unknown(); + auto& other = b.unknown(); + const auto magnitude = [&](IrValue& bits) -> IrValue& { return b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&bits, &b.constant(0x7fffffffu)}); }; + const auto isNan = [&](IrValue& bits) -> IrValue& { return b.emit(IrOpcode::UGreaterThan32, IrType::Bool, {&magnitude(bits), &b.constant(0x7f800000u)}); }; + const auto chosen = [&](IrValue& condition, IrValue& arm) -> IrValue& { return b.flush(b.emit(IrOpcode::SelectU32, IrType::U32, {&condition, &arm, &b.flush(b.unknown())})); }; + IrValue* removed[] = { + &chosen(isNan(x), x), + &chosen(isNan(x), b.emit(IrOpcode::BitwiseOr32, IrType::U32, {&x, &b.constant(0x00400000u)})), + &chosen(b.emit(IrOpcode::FPIsNan32, IrType::Bool, {&b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&x})}), x), + }; + IrValue* kept[] = { + &chosen(isNan(other), x), + &chosen(b.emit(IrOpcode::UGreaterThan32, IrType::Bool, {&b.constant(0x7f800000u), &magnitude(x)}), x), + }; + for (auto* flushed : removed) b.keep(*flushed); + for (auto* flushed : kept) b.keep(*flushed); + const bool counted = b.eliminate() == 3u; + bool flushes = true; + for (auto* flushed : removed) flushes &= !flushed->HasUses(); + for (auto* flushed : kept) flushes &= flushed->HasUses(); + return counted && flushes; + }); + + passed &= run("the sign or exponent bits of any value need no flush, a mask that keeps significand bits does", [] { + Builder b; + auto& sign = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.unknown(), &b.constant(0x80000000u)})); + auto& exponent = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.constant(0xff800000u), &b.unknown()})); + auto& significand = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.unknown(), &b.constant(0x807fffffu)})); + b.keep(sign); + b.keep(exponent); + b.keep(significand); + return b.eliminate() == 2u && !sign.HasUses() && !exponent.HasUses() && significand.HasUses(); + }); + + passed &= run("two chained v_mul_f32 keep only the input flush of the first source", [] { + const std::array code{ + 0x10020100u, + 0x10040301u, + 0xe0700000u, 0x80000200u, + 0xbf810000u, + }; + const auto masks = ExponentMasks(Recompiled(code)); + if (masks != 1u) std::fprintf(stderr, " %zu f32 exponent masks, expected 1\n", masks); + return masks == 1u; + }); + return passed ? 0 : 1; } diff --git a/core/shader/recompiler/Optimization/src/DenormalFlushEliminator.cpp b/core/shader/recompiler/Optimization/src/DenormalFlushEliminator.cpp index d448da5e6a..f8a013e4b7 100644 --- a/core/shader/recompiler/Optimization/src/DenormalFlushEliminator.cpp +++ b/core/shader/recompiler/Optimization/src/DenormalFlushEliminator.cpp @@ -19,6 +19,58 @@ const IrValue* maskedOperand(const IrValue* value, IrOpcode opcode, std::uint32_ return nullptr; } +const IrValue* rightOperand(const IrValue* value, IrOpcode opcode, std::uint32_t bits) { + if (value->Opcode() != opcode || value->ArgumentCount() != 2u) return nullptr; + return isImmediate(value->Argument(1)->Resolve(), bits) ? value->Argument(0)->Resolve() : nullptr; +} + +const IrValue* bitsOf(const IrValue* value) { + value = value->Resolve(); + while ((value->Opcode() == IrOpcode::BitCastU32F32 || value->Opcode() == IrOpcode::BitCastF32U32) && value->ArgumentCount() == 1u) value = value->Argument(0)->Resolve(); + return value; +} + +bool testsForNan(const IrValue* condition, const IrValue* bits) { + condition = condition->Resolve(); + if (condition->Opcode() == IrOpcode::FPIsNan32 && condition->ArgumentCount() == 1u) return bitsOf(condition->Argument(0)) == bits; + const IrValue* magnitude = rightOperand(condition, IrOpcode::UGreaterThan32, 0x7f800000u); + if (magnitude != nullptr) magnitude = maskedOperand(magnitude, IrOpcode::BitwiseAnd32, 0x7fffffffu); + return magnitude != nullptr && bitsOf(magnitude) == bits; +} + +bool takenOnlyAsNan(const IrValue* condition, const IrValue* arm) { + const IrValue* bits = bitsOf(arm); + if (const IrValue* quieted = maskedOperand(bits, IrOpcode::BitwiseOr32, 0x00400000u)) bits = bitsOf(quieted); + return testsForNan(condition, bits); +} + +const IrValue* tinyGuardedProduct(const IrValue* condition, const IrValue* arm) { + const IrValue* product = arm->Resolve(); + if (product->Opcode() != IrOpcode::FPMul32 || product->ArgumentCount() != 2u) return nullptr; + const IrValue* offset = rightOperand(condition->Resolve(), IrOpcode::ULessThan32, 0x207fffffu); + const IrValue* magnitude = offset != nullptr ? rightOperand(offset, IrOpcode::ISub32, 1u) : nullptr; + const IrValue* scaled = magnitude != nullptr ? maskedOperand(magnitude, IrOpcode::BitwiseAnd32, 0x7fffffffu) : nullptr; + if (scaled == nullptr) return nullptr; + scaled = bitsOf(scaled); + if (scaled->Opcode() != IrOpcode::FPMul32 || scaled->ArgumentCount() != 2u) return nullptr; + const IrValue* lhs = product->Argument(0)->Resolve(); + const IrValue* rhs = product->Argument(1)->Resolve(); + for (std::size_t index = 0; index < 2u; ++index) { + const IrValue* factor = maskedOperand(scaled->Argument(index)->Resolve(), IrOpcode::FPMul32, 0x5f800000u); + const IrValue* other = scaled->Argument(1u - index)->Resolve(); + if (factor != nullptr && ((factor == lhs && other == rhs) || (factor == rhs && other == lhs))) return product; + } + return nullptr; +} + +bool clearsSignificand(const IrValue& value) { + for (std::size_t index = 0; index < value.ArgumentCount(); ++index) { + const IrValue* mask = value.Argument(index)->Resolve(); + if (mask->HasImmediate() && (mask->ImmediateU32() & 0x007fffffu) == 0u) return true; + } + return false; +} + IrValue* flushedSource(const IrValue& value) { if (value.Opcode() != IrOpcode::SelectU32 || value.ArgumentCount() != 3u) return nullptr; IrValue* source = value.Argument(2)->Resolve(); @@ -65,10 +117,14 @@ private: return false; case IrOpcode::BitwiseAnd32: if (const IrValue* operand = maskedOperand(&value, IrOpcode::BitwiseAnd32, 0x7fffffffu)) return Contains(operand); - return false; + return value.ArgumentCount() == 2u && clearsSignificand(value); case IrOpcode::SelectU32: - case IrOpcode::SelectF32: - return Contains(value.Argument(1)) && Contains(value.Argument(2)); + case IrOpcode::SelectF32: { + const IrValue* condition = value.Argument(0); + const IrValue* product = tinyGuardedProduct(condition, value.Argument(2)); + const bool taken = takenOnlyAsNan(condition, value.Argument(1)) || Contains(value.Argument(1)); + return taken && (product != nullptr ? Contains(product->Argument(0)) && Contains(product->Argument(1)) : Contains(value.Argument(2))); + } case IrOpcode::FPMin32: case IrOpcode::FPMax32: case IrOpcode::FPMinTri32: diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 89fa8f70d7..3f379e5cca 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -42,6 +42,7 @@ - Comparison sampling of a color texture through an [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) S# throws at the first dispatch or draw that binds it. - An [image table](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) key with no key domain whose `key * stride` wraps past 2^32 to an offset between records faults only when the build already expects a fault; otherwise the fault buffer is not read and it reads zeros. - [MaskedSelectEliminator](../../../core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp) can discard inactive source lanes through `ReadFirstLane` only when its read mask implies the write mask and its result is unobserved outside the read mask. A loop-local read mask requires result uses in the same block without phi transport. The empty-EXEC lane-zero fallback follows AMD's [Vega ISA, section 12.8, V_READFIRSTLANE_B32, page 127](https://www.amd.com/content/dam/amd/en/documents/radeon-tech-docs/instruction-set-architectures/vega-shader-instruction-set-architecture.pdf) and the existing SPIR-V emitter. The optimization is covered by synthetic IR tests but has not been compared with console hardware; a proof that needs a read mask of a kind that is already being solved is refused. +- [DenormalFlushEliminator](../../../core/shader/recompiler/Optimization/src/DenormalFlushEliminator.cpp) drops the result flush of a `v_mul_f32` or `v_mul_legacy_f32` product behind its nearest-even tininess check (see f32 denormals) when both factors are denormal-free: a product the check keeps is then zero, infinite, NaN or at least 2^-126 in magnitude, provided the driver rounds it as it rounds the check's multiply. - [Shader modules built at registration](../../../core/shader/recompiler/Recompiler.cpp) (shader recompiler): on devices with graphics pipeline libraries a vertex or pixel module reads buffer V# words, image modes, swizzles and mip ranges, depth-compare state, export mappings and vertex component selectors at run time instead of specializing on them per draw, so its libraries can be built when the shader is registered. A module that would exceed `SpirvTarget::registrationWordLimit` (200000 SPIR-V words) is emitted specialized as before: on RADV (Radeon 660M, Mesa 26.2.4) a sample of 60 Astro's Playroom pixel shaders compiled 1.9 to 2.2 times slower in registration form below 141k words and 3.8 to 5.7 times slower above 280k words (none in between), so the limit sits in that gap; it was not measured on other drivers. Texel-offset folding, the format word of a buffer accessed with the descriptor's format (`buffer_load_format_*`; as runtime values every format's decoder is compiled, which made CI's lavapipe JIT ten times slower) and the vertex attribute numeric class stay specialized, so shaders with the first two, and pixel shaders whose image needs a multisampled descriptor, get no registration result; these, and every draw whose fixed-function state differs from the prebuild's, still build their libraries at the draw. Compute and the other stages stay specialized: in registration form the `agc_driver_dword_pattern_fill` compute shader hung CI's lavapipe (Mesa 25.2.8) when built with `ANYPS5_ENABLE_SPIRV_TOOLS=ON`, not on RADV, and the cause was not found. - [SPIRV-Tools optimization](../../../core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp) skips CCP and Simplification for modules with storage-only 8- or 16-bit types because the pinned folder produces illegal narrow constants. This also disables unrelated constant folding; the restricted full pipeline mirrors the pinned performance recipe. - [SRT signed comparison](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp) evaluates `SLessThan32` as signed 32-bit less-than, matching the existing SPIR-V backend and [SPIR-V `OpSLessThan`](https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpSLessThan). Signed boundary descriptor expressions are tested on the host; native RDNA2/PS5 execution and title behavior remain unverified. From 227d1e44a10edadbec0373339cc4b3fea888e167 Mon Sep 17 00:00:00 2001 From: milescoviello Date: Sat, 17 Oct 2026 17:03:09 +0000 Subject: [PATCH 049/103] perf(shader): fold a select nested in a select on the same condition select(c, a, select(c, b, d)) is select(c, a, d), and select(c, select(c, a, b), d) is select(c, a, d). The recompiler leaves these behind when a register is written more than once under the same lane mask and when the f32 NaN selection tests the same source twice, and nothing folded them. The constant folder now takes the inner arm the outer condition already decides, then folds a select whose arms became equal. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 13 ++ .../libSceAgcDriver/tests/ConstantSelect.cpp | 121 ++++++++++++++++++ .../Optimization/src/ConstantFolder.cpp | 16 ++- 3 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/ConstantSelect.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index c6950a121b..89f98916db 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2493,6 +2493,19 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_constant_select_tests EXCLUDE_FROM_ALL tests/ConstantSelect.cpp) +target_link_libraries(agc_constant_select_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_constant_select_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_constant_select_tests) +if(BUILD_TESTING) + set_target_properties(agc_constant_select_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_constant_select COMMAND agc_constant_select_tests) + if(WIN32) + set_tests_properties(agc_constant_select PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + if(BUILD_TESTING) get_directory_property(agcRegisteredTests TESTS) list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal|agc_driver_pipeline_journal)$") diff --git a/core/libs/prx/libSceAgcDriver/tests/ConstantSelect.cpp b/core/libs/prx/libSceAgcDriver/tests/ConstantSelect.cpp new file mode 100644 index 0000000000..883659b077 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ConstantSelect.cpp @@ -0,0 +1,121 @@ +#include "IntermediateRepresentation/IrBuilder.hpp" +#include "Optimization/ConstantFolder.hpp" +#include "Optimization/DeadCodeEliminator.hpp" +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +struct Program { + IrProgram program; + IrBuilder builder{program}; + IrValue* lane = nullptr; + IrValue* condition = nullptr; + IrValue* other = nullptr; + std::uint32_t values = 0; + + Program() { + auto& block = program.CreateBlock(); + program.BlockOrder().push_back(&block); + BlockInfo info; + info.id = 0; + info.terminator.kind = TerminatorKind::Return; + program.Metadata().blockInfo.push_back(info); + program.SetEntryBlock(block); + builder.SetInsertionPoint(block); + lane = &builder.Emit(IrOpcode::LaneId, IrType::U32, {}); + condition = &builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {lane, &builder.Constant(16u)}); + other = &builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {lane, &builder.Constant(8u)}); + } + + IrValue& value() { + return builder.Emit(IrOpcode::IAdd32, IrType::U32, {lane, &builder.Constant(++values)}); + } + + IrValue& select(IrValue& selector, IrValue& onTrue, IrValue& onFalse) { + return builder.Emit(IrOpcode::SelectU32, IrType::U32, {&selector, &onTrue, &onFalse}); + } + + const IrValue* folded(IrValue& result) { + auto& reference = builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&result}); + ValidateProgram(program, true); + ConstantFolder{}.Fold(program); + DeadCodeEliminator{}.RemoveIdentities(program); + DeadCodeEliminator{}.Eliminate(program); + ValidateProgram(program, true); + return reference.Argument(0)->Resolve(); + } +}; + +bool Selects(const IrValue* value, const IrValue* selector, const IrValue* onTrue, const IrValue* onFalse) { + return value->Opcode() == IrOpcode::SelectU32 && value->Argument(0)->Resolve() == selector && value->Argument(1)->Resolve() == onTrue && value->Argument(2)->Resolve() == onFalse; +} + +void CheckFalseArm() { + Program p; + auto& kept = p.value(); + auto& dropped = p.value(); + auto& fallback = p.value(); + auto& inner = p.select(*p.condition, dropped, fallback); + const auto* result = p.folded(p.select(*p.condition, kept, inner)); + Require(Selects(result, p.condition, &kept, &fallback), "a select in the false arm of a select on the same condition was not folded to its false arm"); + Require(!inner.HasUses(), "the folded inner select is still used"); +} + +void CheckTrueArm() { + Program p; + auto& kept = p.value(); + auto& dropped = p.value(); + auto& fallback = p.value(); + const auto* result = p.folded(p.select(*p.condition, p.select(*p.condition, kept, dropped), fallback)); + Require(Selects(result, p.condition, &kept, &fallback), "a select in the true arm of a select on the same condition was not folded to its true arm"); +} + +void CheckChain() { + Program p; + auto& last = p.value(); + auto& fallback = p.value(); + auto* chain = &p.select(*p.condition, p.value(), fallback); + chain = &p.select(*p.condition, p.value(), *chain); + const auto* result = p.folded(p.select(*p.condition, last, *chain)); + Require(Selects(result, p.condition, &last, &fallback), "a chain of selects on one condition did not fold to the last write and the fallback"); +} + +void CheckEqualArms() { + Program p; + auto& same = p.value(); + const auto* result = p.folded(p.select(*p.condition, same, p.select(*p.condition, p.value(), same))); + Require(result == &same, "a select whose arms become equal after folding was not replaced by that value"); +} + +void CheckOtherCondition() { + Program p; + auto& kept = p.value(); + auto& inner = p.select(*p.other, p.value(), p.value()); + const auto* result = p.folded(p.select(*p.condition, kept, inner)); + Require(Selects(result, p.condition, &kept, &inner), "a select nested under a different condition was folded"); +} + +} + +int main() { + try { + CheckFalseArm(); + CheckTrueArm(); + CheckChain(); + CheckEqualArms(); + CheckOtherCondition(); + std::puts("constant select tests passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp index 52643c8990..ea97d46833 100644 --- a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp +++ b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp @@ -200,17 +200,25 @@ bool replaceBinaryIdentity(IrValue& inst, IrType type, std::uint64_t identity) { bool foldSelect(IrValue& inst) { auto& condition = resolveArg(inst, 0); - auto& trueValue = resolveArg(inst, 1); - auto& falseValue = resolveArg(inst, 2); if (isImmediate(condition, IrType::U1)) { - replaceWith(inst, condition.ImmediateBool() ? trueValue : falseValue); + replaceWith(inst, resolveArg(inst, condition.ImmediateBool() ? 1 : 2)); return true; } + bool folded = false; + for (std::size_t arm = 1; arm <= 2; arm++) { + auto& nested = resolveArg(inst, arm); + if (nested.Opcode() == inst.Opcode() && nested.ArgumentCount() == 3u && &resolveArg(nested, 0) == &condition) { + inst.ReplaceArgument(arm, &resolveArg(nested, arm)); + folded = true; + } + } + auto& trueValue = resolveArg(inst, 1); + auto& falseValue = resolveArg(inst, 2); if (trueValue == falseValue) { replaceWith(inst, trueValue); return true; } - return false; + return folded; } bool foldPhi(IrValue& inst) { From 206aaa56cbd0710f6bfbf20b07b084207d4d51c3 Mon Sep 17 00:00:00 2001 From: oskarlgs Date: Sat, 17 Oct 2026 20:58:29 +0000 Subject: [PATCH 050/103] fix(relinker): decode near branches with an operand-size prefix as AMD does 66 E8, 66 E9 and 66 0F 80-8F without REX.W take a 16-bit displacement on AMD processors, the console included. Every near relative branch with that prefix truncates its target to 16 bits, so the decoder reports no branch target for it and stubs refuse to move it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016TFMtUC4L4wMS4eeEqaNSR --- .../codegen/include/codegen/CodegenTypes.hpp | 1 + .../codegen/src/x86/StubBodyBuilder.cpp | 2 ++ .../codegen/src/x86/X64InstructionDecoder.cpp | 20 ++++++++++++-- .../codegen/tests/DecoderLengthCases.hpp | 8 ++++++ .../codegen/tests/DecoderLengthTests.cpp | 14 ++++++++++ .../codegen/tests/DecoderMetadataCases.hpp | 27 +++++++++++++++++++ docs/dev/debt/relinker.md | 2 +- 7 files changed, 71 insertions(+), 3 deletions(-) diff --git a/core/relinker/codegen/include/codegen/CodegenTypes.hpp b/core/relinker/codegen/include/codegen/CodegenTypes.hpp index 43e5a7110b..72bace4267 100644 --- a/core/relinker/codegen/include/codegen/CodegenTypes.hpp +++ b/core/relinker/codegen/include/codegen/CodegenTypes.hpp @@ -71,6 +71,7 @@ struct DecodedInstructionInfo { std::uint8_t ModRmByte; bool HasBranchTarget; std::int64_t BranchDisp; + bool TruncatesBranchTarget; bool IsTwoByteOpcode; std::uint8_t Opcode; std::uint8_t ModRmRegField; diff --git a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp index e68805215f..f3e18ab213 100644 --- a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp +++ b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp @@ -210,6 +210,8 @@ void StubBodyBuilder::MoveControlFlow(const std::span instru const auto end = _siteOffset + static_cast(instruction.size()); if (info.FlowKind == ControlFlowKind::IndirectCall) throw CodegenException("A register or memory call cannot move into a stub"); + if (info.TruncatesBranchTarget) + throw CodegenException("A relative branch with this prefix cannot move into a stub"); if (info.FlowKind == ControlFlowKind::Call && info.HasRipRelativeDisp) { _pushReturnAddress(end); std::vector jump(instruction.begin(), instruction.end()); diff --git a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp index a3f7f3962d..c6cc9de4ac 100644 --- a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp +++ b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp @@ -207,7 +207,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t } if (opcode == OneByteCallRel32 || opcode == OneByteJmpRel32) { - immediateSize = ImmSize32; + immediateSize = operandImmediateSize; } if (opcode == OneByteJmpRel8 || opcode == OneByteJrcxz || @@ -241,7 +241,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t } hasModRm = true; } else if (opcode >= TwoByteJccRel32Min && opcode <= TwoByteJccRel32Max) { - immediateSize = ImmSize32; + immediateSize = operandImmediateSize; } else if (opcode == TwoByteMovImm8ModRm || opcode == TwoByte3DNow) { hasModRm = true; immediateSize = ImmSize8; @@ -355,6 +355,7 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( info.Length = Decode(data, available); std::size_t pos = 0; + bool operandSizeOverride = false; while (pos < info.Length) { std::uint8_t b = data[pos]; @@ -371,6 +372,8 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( info.SegmentPrefix = b; if (b == PrefixAddressSize) info.AddressSize32 = true; + if (b == PrefixOperandSize) + operandSizeOverride = true; info.RexPrefix = 0; ++pos; continue; @@ -417,6 +420,19 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( info.Opcode = op; info.IsTwoByteOpcode = twoByteOpcode; + const bool relativeNearBranch = twoByteOpcode + ? op >= TwoByteJccRel32Min && op <= TwoByteJccRel32Max + : op == OneByteCallRel32 || op == OneByteJmpRel32 || op == OneByteJmpRel8 || + (op >= OneByteJccRel8Min && op <= OneByteJccRel8Max) || + op == OneByteJrcxz || (op >= OneByteLoop && op <= OneByteLoopMax); + if (relativeNearBranch && operandSizeOverride && (info.RexPrefix & RexWBit) == 0) { + info.TruncatesBranchTarget = true; + info.FlowKind = twoByteOpcode || (op != OneByteCallRel32 && op != OneByteJmpRel32 && op != OneByteJmpRel8) + ? ControlFlowKind::ConditionalBranch + : op == OneByteCallRel32 ? ControlFlowKind::Call : ControlFlowKind::UnconditionalJump; + return info; + } + if (!twoByteOpcode) { if (op == OneByteCallRel32) { std::int32_t disp = 0; diff --git a/core/relinker/codegen/tests/DecoderLengthCases.hpp b/core/relinker/codegen/tests/DecoderLengthCases.hpp index 10747c9a61..1de81e75ab 100644 --- a/core/relinker/codegen/tests/DecoderLengthCases.hpp +++ b/core/relinker/codegen/tests/DecoderLengthCases.hpp @@ -420,6 +420,9 @@ inline constexpr DecoderLengthCase kDecoderLengthCases[] = { {"660f79c1", 4}, {"660f7c846511223344", 9}, {"660f7d846511223344", 9}, + {"660f801122", 5}, + {"660f841122", 5}, + {"660f8f1122", 5}, {"660fd0846511223344", 9}, {"660fd6846511223344", 9}, {"660fe6846511223344", 9}, @@ -430,6 +433,7 @@ inline constexpr DecoderLengthCase kDecoderLengthCases[] = { {"66358465", 4}, {"663d8465", 4}, {"66480584651122", 7}, + {"66480f8411223344", 8}, {"66480d84651122", 7}, {"66481584651122", 7}, {"66481d84651122", 7}, @@ -450,6 +454,8 @@ inline constexpr DecoderLengthCase kDecoderLengthCases[] = { {"6648be8465112233445566", 11}, {"6648bf8465112233445566", 11}, {"6648c784651122334455667708", 13}, + {"6648e811223344", 7}, + {"6648e911223344", 7}, {"6648f784651122334455667708", 13}, {"66688465", 4}, {"66698465112233445566", 10}, @@ -464,6 +470,8 @@ inline constexpr DecoderLengthCase kDecoderLengthCases[] = { {"66be8465", 4}, {"66bf8465", 4}, {"66c78465112233445566", 10}, + {"66e81122", 4}, + {"66e91122", 4}, {"66f78465112233445566", 10}, {"676984651122334455667708", 12}, {"678184651122334455667708", 12}, diff --git a/core/relinker/codegen/tests/DecoderLengthTests.cpp b/core/relinker/codegen/tests/DecoderLengthTests.cpp index 54dd90df2b..63bca4bfe0 100644 --- a/core/relinker/codegen/tests/DecoderLengthTests.cpp +++ b/core/relinker/codegen/tests/DecoderLengthTests.cpp @@ -99,6 +99,20 @@ int main() { require(info.HasBranchTarget == testCase.HasBranchTarget, "Wrong branch target presence for " + name); require(!testCase.HasBranchTarget || info.BranchDisp == testCase.BranchDisp, "Wrong branch displacement for " + name); } + for (const auto& testCase : kDecoderTruncatedBranchCases) { + const std::string name = testCase.Bytes; + auto bytes = parse(testCase.Bytes); + require(bytes.size() == testCase.Length, "Truncated branch case " + name + " does not hold its own length"); + bytes.insert(bytes.end(), 15, 0x90); + const auto info = decoder.DecodeInstruction(bytes.data(), bytes.size()); + require(info.Length == testCase.Length, "Wrong length for " + name); + require(info.FlowKind == testCase.FlowKind, "Wrong control flow kind for " + name); + require(info.TruncatesBranchTarget && !info.HasBranchTarget, "Branch target of " + name + " was reported without its 16-bit truncation"); + } + for (const auto& testCase : kDecoderBranchCases) { + const auto bytes = parse(testCase.Bytes); + require(!decoder.DecodeInstruction(bytes.data(), bytes.size()).TruncatesBranchTarget, "Branch target of " + std::string(testCase.Bytes) + " was reported as truncated"); + } const auto threeDNow = parse("0f0f0d11223344a6"); const auto threeDNowInfo = decoder.DecodeInstruction(threeDNow.data(), threeDNow.size()); require(threeDNowInfo.HasModRm && threeDNowInfo.ModRmByte == 0x0d, "3DNow! ModRM was not reported"); diff --git a/core/relinker/codegen/tests/DecoderMetadataCases.hpp b/core/relinker/codegen/tests/DecoderMetadataCases.hpp index c685b184ea..d29d0c0191 100644 --- a/core/relinker/codegen/tests/DecoderMetadataCases.hpp +++ b/core/relinker/codegen/tests/DecoderMetadataCases.hpp @@ -787,6 +787,33 @@ inline constexpr DecoderBranchCase kDecoderBranchCases[] = { {"ff28", 2, Codegen::ControlFlowKind::IndirectJump, false, 0}, {"ffd0", 2, Codegen::ControlFlowKind::IndirectCall, false, 0}, {"ffe0", 2, Codegen::ControlFlowKind::IndirectJump, false, 0}, + {"66480f84c0111111", 8, Codegen::ControlFlowKind::ConditionalBranch, true, 286331328}, + {"6648e8c0111111", 7, Codegen::ControlFlowKind::Call, true, 286331328}, + {"6648e9c0111111", 7, Codegen::ControlFlowKind::UnconditionalJump, true, 286331328}, + {"6648ebc0", 4, Codegen::ControlFlowKind::UnconditionalJump, true, -64}, + {"664875c0", 4, Codegen::ControlFlowKind::ConditionalBranch, true, -64}, +}; + +struct DecoderTruncatedBranchCase { + const char* Bytes; + std::size_t Length; + Codegen::ControlFlowKind FlowKind; +}; + +inline constexpr DecoderTruncatedBranchCase kDecoderTruncatedBranchCases[] = { + {"660f80c011", 5, Codegen::ControlFlowKind::ConditionalBranch}, + {"660f8fc011", 5, Codegen::ControlFlowKind::ConditionalBranch}, + {"6670c0", 3, Codegen::ControlFlowKind::ConditionalBranch}, + {"6675c0", 3, Codegen::ControlFlowKind::ConditionalBranch}, + {"667fc0", 3, Codegen::ControlFlowKind::ConditionalBranch}, + {"66e0c0", 3, Codegen::ControlFlowKind::ConditionalBranch}, + {"66e2c0", 3, Codegen::ControlFlowKind::ConditionalBranch}, + {"66e3c0", 3, Codegen::ControlFlowKind::ConditionalBranch}, + {"66e8c011", 4, Codegen::ControlFlowKind::Call}, + {"66e9c011", 4, Codegen::ControlFlowKind::UnconditionalJump}, + {"66ebc0", 3, Codegen::ControlFlowKind::UnconditionalJump}, + {"6667e8c011", 5, Codegen::ControlFlowKind::Call}, + {"6640e8c011", 5, Codegen::ControlFlowKind::Call}, }; #endif diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index acab2b4e54..770bab7545 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -9,7 +9,7 @@ - [Guest import registries](../../../core/relinker/relinker/src/guest/GuestModuleBuilder.cpp) - declared host imports of bundled modules are recorded with their module name and relocation target, even when another guest module exports the same stripped NID. No-op relocations are omitted. Registries describe import bindings, not implementation correctness or runtime reachability. - Converted Windows main executables [omit `DYNAMIC_BASE`](../../../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) by default, not only with `--windows-fixed-base`: on the console the executable lies inside the 40-bit window that the title's own `sceLibcMspaceCreate` checks (#3043). Relocations are kept, so if `0x140000000` is taken Windows relocates the image, possibly above that window. Verified only on Windows 10. - [SDK memory-management syscall traps](../../../core/relinker/relinker/src/analysis/SyscallScanner.cpp) under `--trap-sdk-syscall-stubs` recognize three exact instruction sequences from [ps5-payload-dev SDK libc/mman.c](https://github.com/ps5-payload-dev/sdk/blob/12cf3dda40/libc/mman.c), observed in Nuvio Maxd 0.5.15 (PPSA99288): syscall 74 and the two syscall 477 paths, with their argument preparation and carry-flag error handling. Matching requires a decoded instruction boundary and the complete sequence inside the scanned code range. Other SDK/compiler output remains rejected. The matched syscall becomes `UD2`; this does not implement mmap or mprotect and a call still faults. -- The [instruction decoder](../../../core/relinker/codegen/src/x86/X64InstructionDecoder.cpp) decodes a near CALL, JMP or Jcc with an operand-size prefix (`66`) as Intel does, with a 32-bit or 8-bit displacement and a 64-bit target. AMD processors, the console included, read a 16-bit displacement for `66 E8`, `66 E9` and `66 0F 80`-`8F` and truncate the target of every near branch with that prefix, rel8 forms included, to 16 bits. Compilers do not emit these forms; the relinker's length and target for them do not match the console. +- The [instruction decoder](../../../core/relinker/codegen/src/x86/X64InstructionDecoder.cpp) reads a 16-bit displacement for `66 E8`, `66 E9` and `66 0F 80`-`8F` without REX.W, as AMD processors, the console included, do (binutils `objdump -M amd64`; REX.W selects the 32-bit form). AMD also truncates the target of every near relative branch with that prefix, rel8, LOOP and JRCXZ forms included, to 16 bits. Instruction metadata marks these branches as truncated and reports no branch target for them, and stubs refuse to move them. Compilers do not emit these forms. - `--to-intel` does not lower RDPRU/MCOMMIT; the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) fails the relink instead. - The length-changing path of the [instruction rewriter](../../../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../../../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length. - A `PT_SCE_RELRO` segment that no `PT_LOAD` covers, the PS4 layout, is [loaded](../../../core/relinker/relinker/src/parsing/ElfReader.cpp) as a read-write `PT_LOAD` and stays writable after relocation; FreeBSD rtld makes `PT_GNU_RELRO` read-only once relocation is done, and what the console does with `PT_SCE_RELRO` is not checked. From 70ff799fdbd526b1d27bd7a976b0dd92a1e1f7fc Mon Sep 17 00:00:00 2001 From: T3cm0 Date: Sat, 17 Oct 2026 16:17:05 -0500 Subject: [PATCH 051/103] feat(libkernel): implement three coredump exports from the console module Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libkernel/System/src/Coredump.cpp | 17 +++++++++-------- core/libs/tests/GuestCoredumpAttach.cpp | 17 +++++++++++++++++ docs/dev/debt/libkernel.md | 5 +++-- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/core/libs/prx/libkernel/System/src/Coredump.cpp b/core/libs/prx/libkernel/System/src/Coredump.cpp index a644a630f1..3bc229914c 100644 --- a/core/libs/prx/libkernel/System/src/Coredump.cpp +++ b/core/libs/prx/libkernel/System/src/Coredump.cpp @@ -11,6 +11,7 @@ static std::atomic g_coredumpHandler{0}; static std::atomic g_coredumpContext{0}; +static constexpr int SCE_COREDUMP_ERROR_INVALID_ARGUMENT = static_cast(0x81180000u); static constexpr int SCE_COREDUMP_ERROR_NOT_IN_COREDUMP_HANDLER = static_cast(0x81180003u); static void requireRegionOutsideHandler(const char* function, const void* mem, size_t size) { @@ -55,9 +56,10 @@ int APS5_VABI sceCoredumpAttachUserFile(uint32_t userValue, const char* path) { return SCE_COREDUMP_ERROR_NOT_IN_COREDUMP_HANDLER; } -int APS5_VABI sceCoredumpGetStopInfoGpu_Agc(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceCoredumpGetStopInfoGpu_Agc(void* info, size_t size) { + (void)size; + if (info == nullptr) return SCE_COREDUMP_ERROR_INVALID_ARGUMENT; + return SCE_COREDUMP_ERROR_NOT_IN_COREDUMP_HANDLER; } int APS5_VABI sceCoredumpAttachMemoryRegionAsUserFile(uint32_t userValue, const void* mem, size_t size, const char* name) { @@ -99,14 +101,13 @@ int APS5_VABI sceCoredumpGetThreadContextInfo(void* threadContextInfo, size_t th return SCE_COREDUMP_ERROR_NOT_IN_COREDUMP_HANDLER; } -int APS5_VABI sceCoredumpWriteUserString() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceCoredumpWriteUserString(uint32_t first, const char* text, int third) { + if (text == nullptr || first > 15 || third < 0 || third > 256) return SCE_COREDUMP_ERROR_INVALID_ARGUMENT; return 0; } -int APS5_VABI sceCoredumpConfigDumpMode() { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceCoredumpConfigDumpMode(int mode) { + return mode == 1 ? 0 : SCE_COREDUMP_ERROR_INVALID_ARGUMENT; } int APS5_VABI sceCoredumpAttachUserMemoryFile(uint32_t userValue, const void* mem, size_t size) { diff --git a/core/libs/tests/GuestCoredumpAttach.cpp b/core/libs/tests/GuestCoredumpAttach.cpp index 1fb6e65f1e..43e32238e0 100644 --- a/core/libs/tests/GuestCoredumpAttach.cpp +++ b/core/libs/tests/GuestCoredumpAttach.cpp @@ -14,9 +14,13 @@ int APS5_VABI sceCoredumpAttachMemoryRegion(uint32_t userValue, const void* mem, int APS5_VABI sceCoredumpAttachUserMemoryFile(uint32_t userValue, const void* mem, size_t size); int APS5_VABI sceCoredumpGetStopInfoCpu(void* info, size_t size); int APS5_VABI sceCoredumpGetThreadContextInfo(void* threadContextInfo, size_t threadContextInfoSize); +int APS5_VABI sceCoredumpGetStopInfoGpu_Agc(void* info, size_t size); +int APS5_VABI sceCoredumpWriteUserString(uint32_t first, const char* text, int third); +int APS5_VABI sceCoredumpConfigDumpMode(int mode); } namespace { +constexpr int InvalidArgument = static_cast(0x81180000u); constexpr int NotInCoredumpHandler = static_cast(0x81180003u); constexpr uint32_t UserFileTruncateIfExceedLimit = 0x80000000u; bool handlerRan = false; @@ -90,5 +94,18 @@ int main() { Require(sceCoredumpGetThreadContextInfo(context.data() + 8, 144) == NotInCoredumpHandler); Require(region == original && context == originalContext); Require(!handlerRan); + + Require(sceCoredumpGetStopInfoGpu_Agc(region.data() + 8, 8) == NotInCoredumpHandler); + Require(sceCoredumpGetStopInfoGpu_Agc(region.data() + 8, 3) == NotInCoredumpHandler); + Require(sceCoredumpGetStopInfoGpu_Agc(nullptr, 8) == InvalidArgument); + Require(sceCoredumpWriteUserString(1, "text", 4) == 0); + Require(sceCoredumpWriteUserString(0, "", 0) == 0); + Require(sceCoredumpWriteUserString(1, nullptr, 4) == InvalidArgument); + Require(sceCoredumpWriteUserString(15, "x", 256) == 0 && sceCoredumpWriteUserString(0, "x", 0) == 0); + Require(sceCoredumpWriteUserString(16, "x", 1) == InvalidArgument && sceCoredumpWriteUserString(0x80000000u, "x", 1) == InvalidArgument); + Require(sceCoredumpWriteUserString(1, "x", 257) == InvalidArgument && sceCoredumpWriteUserString(1, "x", -1) == InvalidArgument); + Require(region == original); + Require(sceCoredumpConfigDumpMode(1) == 0); + for (const int mode : {0, 2, -1}) Require(sceCoredumpConfigDumpMode(mode) == InvalidArgument); return 0; } diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 2fb9d30137..25f2c1a23d 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -33,8 +33,9 @@ - [AMM remap, multimap and protection changes](../../../core/libs/prx/libkernel/Apr/src/Apr.cpp) (libSceAmpr) - signatures and the protection and mask checks taken from ampr_emu; the effects are inferred from the names, not confirmed on a title. A remap moves every page of a fully mapped range to the new address with the given protection and leaves the old range reserved; a multimap maps the same pages at a second address, anywhere in the AMM ranges, and a pool page goes back to the pool once its last mapping is unmapped; a multimap onto its own source range throws. A protection change sets the masked bits on every page of a fully mapped range; the memory type is recorded and ignored. A command on a range that is not fully mapped throws - [AMM PRT ranges](../../../core/libs/prx/libkernel/Apr/src/Apr.cpp) (libSceAmpr) - signatures, checks and their order taken from ampr_emu, where `MapAsPrt` is a map with the PRT flag and protection 0 and `AllocatePaForPrt` a memory type and protection change with mask 1019, both checking the buffer memory after its size; the effects are inferred from the names, not confirmed on a title. Unbacked PRT pages are zero-filled pages with GPU read access, so CPU reads see zeros instead of faulting and GPU writes fault instead of being dropped. `AllocatePaForPrt` backs each unbacked page with a pool page and sets the type and protection of backed ones; `RemapIntoPrt` moves a fully mapped range into a PRT range, its opcode argument (1011 when 0, as in ampr_emu) is recorded and ignored; `UnmapToPrt` returns the pages to the pool and zero-fills them again. A plain map, unmap or remap over a PRT range ends it; `AllocatePaForPrt`, `RemapIntoPrt` or `UnmapToPrt` outside a PRT range throws - [AMM kernel write commands](../../../core/libs/prx/libkernel/Apr/src/Apr.cpp) (libkernel) - signatures of `sceKernelWriteModifyMtypeProtectCommand` and `sceKernelWriteRemapIntoPrtCommand` taken from [ampr_emu's libkernel hook targets](https://github.com/drakmor/ampr_emu/blob/cfa85df379f6eeeb165d7badf9b648e266fe77b7/include/ampr_libkernel_hook_targets.h), which forward to the real kernel; the measure mode (`dst` null reports the size, `outSize` null skips the report) follows ampr_emu's measure-then-record calls, not confirmed on a title. The range, protection and mask checks mirror the `libSceAmpr` siblings, and the recorded bytes match them exactly so the submitted buffer executes on the existing paths -- [sceCoredumpWriteUserString](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) - imported by PPSA16064, not called during its start-up and added without an implementation; unknown signature -- [sceCoredumpConfigDumpMode](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel, NID `gzLt9Qrauk0`) - imported by PPSA16833, whose executable does not load without it; added without an implementation; unknown signature +- [sceCoredumpWriteUserString](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel, NID `32KQRUK13kI`) - `(uint32_t, const char* text, int)` follows the console's `libkernel.sprx` (SDK 4.5x build of `J03950001`) at 0x13050: a null `text` returns `0x81180000`; otherwise the three arguments go to syscall 601 (command 10), whose `EPERM` becomes `SCE_COREDUMP_ERROR_NOT_IN_COREDUMP_HANDLER` (0x81180003), `EINVAL` `0x81180000`, and a success returns the value the kernel wrote back. Measured on a PS5 (firmware 13.60) from a homebrew payload with no core dump handler: the call succeeds with 0, the kernel rejects a first argument above 15 or a third outside 0 to 256 with `0x81180000`, and a 2047-character text is accepted. Those checks are reproduced and anything else returns 0; nothing is recorded, as there is no dump, and what the two numbers mean is unknown. PPSA16064 imports it +- [sceCoredumpConfigDumpMode](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel, NID `gzLt9Qrauk0`) - `(int mode)` follows the same `libkernel.sprx` at 0x134D0: a mode other than 1 returns `0x81180000`; mode 1 issues syscall 601 (command 16, arguments 0 and 1) and returns 0 whether or not it fails, since only a raw return of 22 would map to `0x81180000`. No dump is produced here, so mode 1 returns 0 without the syscall. PPSA16833 imports it +- [sceCoredumpGetStopInfoGpu_Agc](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel, NID `Jrs7UUkGOFo`) - `(void* info, size_t size)` follows the same `libkernel.sprx` at 0x137E0: a null `info` returns `0x81180000`; syscall 601 (command 21) failing with `EPERM` returns `NOT_IN_COREDUMP_HANDLER`; otherwise two `libSceAgcDriver` imports (NIDs `+NIAY3DWpOw` and `LgctzJSGy7s`, names unknown) decide between `0x81180005` and, for a `size` of 8, storing the second one's 64-bit result and returning 0, with any other size returning `0x81180000`. The handler is never invoked here, so a non-null `info` returns `NOT_IN_COREDUMP_HANDLER` without writing, assuming `EPERM` outside the handler as above; the AGC branch is not reached - [sceKernelInternalMemoryGetModuleSegmentInfo](../../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) (libkernel) - the `(InternalSegmentInfo* result)` signature, the 16-byte `{address, size}` result and the choice of the main program's first segment follow [fpPS4](https://github.com/red-prig/fpPS4/blob/04cefd43e6fddd1ab033e7980cd356d14c964905/kernel/ps4_libkernel.pas#L343-L355) only, which fills it through its `sceKernelGetLibkernelTextLocation` from the first segment of `eboot.bin`; whether the console reports that segment, the text of libkernel itself or the calling module's is unknown. The segment is the first one `sceKernelGetModuleInfo` reports for the main program, and 0 is returned. A null result throws. 44 audited titles that bundle their own `libc.prx` import it; no call has been observed - [sceKernelGetModuleInfoForUnwind](../../../core/libs/prx/libkernel/Module/src/Module.cpp) (libkernel) - on Linux the tables come from `sceKernelGetModuleInfoFromAddr`, whose `eh_frame_size` includes the zero terminator, as measured on firmware 13.60 (reported in [#2419](https://github.com/boykopovar/AnyPS5/issues/2419), not reproduced here); on Windows it is assumed to stop before the terminator. LLVM libunwind's `CFI_Parser::findFDE` stops at the terminator or the length, whichever comes first, so it finds the same FDEs either way. Only the main executable and `.guest.prx` images report tables; other host images report none, as on Windows - [sceKernelGetModuleInfoForUnwind](../../../core/libs/prx/libkernel/Module/src/Module.cpp) (libkernel) - on Windows the `.ehmeta` tables and the image file name are cached per allocation base and dropped after a loader free (`dlclose`, `sceKernelStopUnloadModule`, the thread atexit release) when the module is no longer loaded. A table changed in place, or an image freed by anything other than the loader, is not noticed. An address inside an image of the Windows loader's module list whose tables are cached is resolved with `RtlPcToFileHeader` and no `VirtualQuery`; other addresses, and images without tables, still use `VirtualQuery`. The `.eh_frame` walk that finds its end is done once per image instead of once per call From 3802f887f2ca04d8b5f8a904d0fa14ba95cde865 Mon Sep 17 00:00:00 2001 From: T3cm0 Date: Sat, 17 Oct 2026 16:17:22 -0500 Subject: [PATCH 052/103] fix(libc): take sceLibcMspaceReallocalign's alignment before its size Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libc/src/Mspace.cpp | 14 ++++++-------- core/libs/tests/Mspace.cpp | 19 ++++++++++++------- docs/dev/debt/libc.md | 1 + 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/core/libs/prx/libc/src/Mspace.cpp b/core/libs/prx/libc/src/Mspace.cpp index 143d50eba2..dbd74b1203 100644 --- a/core/libs/prx/libc/src/Mspace.cpp +++ b/core/libs/prx/libc/src/Mspace.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include +#include #include #include #include @@ -265,19 +266,15 @@ void* APS5_VABI sceLibcMspaceAlignedAlloc_nid_postfix(void* handle, std::size_t return sceLibcMspaceMemalign_nid_postfix(handle, alignment, size); } -void* APS5_VABI sceLibcMspaceReallocalign_nid_postfix(void* handle, void* pointer, std::size_t size, std::size_t alignment) { - if (alignment == 0 || (alignment & (alignment - 1))) { - Error(22); - return nullptr; - } - const auto effective = std::max(alignment, 16); +void* APS5_VABI sceLibcMspaceReallocalign_nid_postfix(void* handle, void* pointer, std::size_t alignment, std::size_t size) { + if (alignment >> 63) return nullptr; + const auto effective = std::bit_ceil(std::max(alignment, MinimumAlignment)); std::lock_guard lock(arenaMutex); auto* arena = Find(handle); if (!pointer) return Allocate(arena, size, effective); std::map::iterator chunk; if (!FindUsed(arena, pointer, chunk)) return nullptr; - if (!size) { Release(*arena, chunk); return nullptr; } - const auto needed = AlignUp(size, Granule); + const auto needed = AlignUp(std::max(size, 1), MinimumAlignment); if (needed < size) { Error(12); return nullptr; } const auto start = chunk->first; const auto capacity = chunk->second.end - start; @@ -311,4 +308,5 @@ std::size_t APS5_VABI sceLibcMspaceMallocUsableSize_nid_postfix(const void* poin Error(22); return 0; } + } diff --git a/core/libs/tests/Mspace.cpp b/core/libs/tests/Mspace.cpp index 386435ded6..8a3571292f 100644 --- a/core/libs/tests/Mspace.cpp +++ b/core/libs/tests/Mspace.cpp @@ -52,7 +52,7 @@ static void CheckAlignment() { Require(aligned(blocks[0])); void* small = sceLibcMspaceMemalign_nid_postfix(arena, 16, 40); Require(aligned(small)); - void* moved = sceLibcMspaceReallocalign_nid_postfix(arena, nullptr, 40, 8); + void* moved = sceLibcMspaceReallocalign_nid_postfix(arena, nullptr, 8, 40); Require(aligned(moved)); void* posix = nullptr; Require(sceLibcMspacePosixMemalign_nid_postfix(arena, &posix, 8, 40) == 0 && aligned(posix)); @@ -102,7 +102,7 @@ static void CheckMovedBlockKeepsUsableBytes() { void* other = sceLibcMspaceMalloc_nid_postfix(arena, 1); const auto otherUsable = fill(other); Require(sceLibcMspaceMalloc_nid_postfix(arena, 1) != nullptr); - void* realigned = sceLibcMspaceReallocalign_nid_postfix(arena, other, 4096, 256); + void* realigned = sceLibcMspaceReallocalign_nid_postfix(arena, other, 256, 4096); Require(realigned != nullptr && realigned != other && kept(realigned, otherUsable)); Require(sceLibcMspaceDestroy_nid_postfix(arena) == 0); } @@ -126,7 +126,7 @@ int main() { Require(sceLibcMspaceRealloc_nid_postfix(arena, first, size) == nullptr); Require(*__error_nid_postfix() == 12); *__error_nid_postfix() = 0; - Require(sceLibcMspaceReallocalign_nid_postfix(arena, first, size, 16) == nullptr); + Require(sceLibcMspaceReallocalign_nid_postfix(arena, first, 16, size) == nullptr); Require(*__error_nid_postfix() == 12); Require(sceLibcMspaceMallocUsableSize_nid_postfix(first) == 128); for (int i = 0; i < 128; ++i) Require(first[i] == i); @@ -194,12 +194,17 @@ int main() { unsigned char* realigned = static_cast(sceLibcMspaceReallocalign_nid_postfix(arena, nullptr, 64, 64)); Require(realigned && (reinterpret_cast(realigned) & 63) == 0); for (int i = 0; i < 64; ++i) realigned[i] = static_cast(i + 1); - unsigned char* regrown = static_cast(sceLibcMspaceReallocalign_nid_postfix(arena, realigned, 256, 64)); + unsigned char* regrown = static_cast(sceLibcMspaceReallocalign_nid_postfix(arena, realigned, 64, 256)); Require(regrown && (reinterpret_cast(regrown) & 63) == 0); for (int i = 0; i < 64; ++i) Require(regrown[i] == static_cast(i + 1)); - Require(sceLibcMspaceReallocalign_nid_postfix(arena, regrown, 16, 0) == nullptr); - Require(sceLibcMspaceReallocalign_nid_postfix(arena, regrown, 16, 3) == nullptr); - Require(sceLibcMspaceReallocalign_nid_postfix(arena, regrown, 0, 16) == nullptr); + *__error_nid_postfix() = 0; + Require(sceLibcMspaceReallocalign_nid_postfix(arena, regrown, 0, 16) == regrown); + Require(sceLibcMspaceReallocalign_nid_postfix(arena, regrown, 3, 16) == regrown); + Require(sceLibcMspaceReallocalign_nid_postfix(arena, regrown, std::size_t{1} << 63, 16) == nullptr); + Require(sceLibcMspaceReallocalign_nid_postfix(arena, regrown, 64, 0) == regrown && regrown[63] == 64); + Require(*__error_nid_postfix() == 0); + unsigned char* rounded = static_cast(sceLibcMspaceReallocalign_nid_postfix(arena, nullptr, 100, 0)); + Require(rounded && (reinterpret_cast(rounded) & 127) == 0); Require(sceLibcMspaceMallocStatsFast_nid_postfix(arena, &stats) == 0 && stats.currentInuseSize >= 100); sceLibcMspaceFree_nid_postfix(arena, memaligned); void* region = sceLibcMspaceMalloc_nid_postfix(arena, 8192); diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index ff2096892f..038605b678 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -18,6 +18,7 @@ - [malloc_usable_size](../../../core/libs/prx/libc/src/ApplicationHeap.cpp) (libc) - with the default allocator it returns the size the block was requested with (the range registered for it), not the rounded-up chunk size Sony's allocator reports; both satisfy "at least the requested size", and bytes past the request would not be write-watched. With an application allocator it calls entry 9 of the replacement table (`malloc_usable_size` in KytyPS5's `MallocReplace` order) and throws when the table leaves it null, like `malloc_stats_fast` - [malloc, calloc, realloc, memalign, aligned_alloc and reallocalign](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) - follow FreeBSD 11.4 `contrib/jemalloc/src/jemalloc.c` for failures, not measured on the console: a size the default heap rejects or the guest arena cannot place (including a `calloc` product that overflows) returns null with `errno` `ENOMEM`, and `memalign` and `aligned_alloc` return null with `EINVAL` for an alignment that is zero or not a power of two. FreeBSD's libc has no `memalign`, so setting `errno` there is assumed. A host commit failure for a block that fits in the arena (Windows `ERROR_NOT_ENOUGH_MEMORY`, `ERROR_OUTOFMEMORY` or `ERROR_COMMITMENT_LIMIT`, Linux `ENOMEM`) resets and releases the reserved range and returns null with `ENOMEM`; any other commit failure releases the range and throws. `reallocalign` returns null with `EINVAL` for an invalid alignment and `ENOMEM` like the others, which is an assumption as it is not in FreeBSD. `realloc(ptr, 0)` frees the block and returns null, where FreeBSD 11.4's jemalloc 4.5 resizes it to one byte and returns a block - [reallocf](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libSceLibcInternal, implemented in libc) - follows FreeBSD 11 `lib/libc/stdlib/reallocf.c` on top of this libc's `realloc`, not measured on the console: size 0 frees the block and returns null as `realloc` does here, so the block is not freed twice; when `realloc` fails because the application allocator returns null or the default heap rejects the size, the block is freed through the same allocator, `errno` is `ENOMEM` and null is returned. Other failures (a commit failure that is not out of memory, an unknown pointer) throw, as they do for `realloc` +- [sceLibcMspaceReallocalign](../../../core/libs/prx/libc/src/Mspace.cpp) (libSceLibcInternal, implemented in libc) takes `(void* heap, void* block, size_t alignment, size_t size)`, the order of the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x13880, whose internal routine at 0x12390 treats the third argument as the alignment and, for a null block, passes it to the routine `sceLibcMspaceMemalign` uses). Measured on a PS5 (firmware 13.60) from a homebrew payload: a null block with alignment 4096 and size 100 returns a 4096-aligned block; an alignment of 0 or 3 succeeds and 100 gives a 128-aligned block, as the routine raises it to at least 32 and to the next power of two; an alignment of 2^63 or more returns null, keeps the block and leaves `errno` alone; and a size of 0 returns a block (the same one when it is already suitable) instead of freeing it. A block that is aligned and large enough is kept where it is, without giving back the bytes past the new size as the console's in-place resize does - [acosl](../../../core/libs/prx/libc/src/Math.cpp) (libc) - computed by the host's math library, like the other libc math functions here, not by the console's (FreeBSD msun ld80 code in Sony's `libc.prx`): results can differ from the console's in the last bit, and on Linux glibc sets `errno` to `EDOM` for `acosl` outside [-1, 1] where msun only raises the floating-point exception. `long double` is the x87 80-bit format, as Sony's `frexpl` uses it (`fldt 0x8(%rsp)`, result in `st(0)`) - [atanl, sinl, cosl, tanl, sinhl, coshl, tanhl, expl, exp2l, logl, log10l and powl](../../../core/libs/prx/libc/src/Math.cpp) (libc) - computed by the host's math library, as `acosl` is, not by the console's FreeBSD msun ld80 code: results can differ from the console's in the last bit (glibc's `log10l(1000)` is not exactly 3), and on Linux glibc sets `errno` to `EDOM` or `ERANGE` where msun only raises the floating-point exception. `ldexpl` and `truncl` are exact operations and give the console's result; `logb` and `remainder` give the console's value too, but on Linux glibc sets `errno` (`ERANGE` for `logb(0)`, `EDOM` for `remainder` by zero or of an infinity) where msun only raises the floating-point exception - [std_execute_once](../../../core/libs/prx/libc/Export.cpp) (libc) - assumed to be the plain-named twin of `std::_Execute_once` (`_ZSt13_Execute_onceRSt9once_flagPFiPvS1_PS1_ES1_`, the callee of Dinkumware `call_once`): it delegates there and gets the [thread.once.callonce] semantics already verified for that symbol (one invocation per flag, concurrent callers block until it completes, and a failing or throwing invocation leaves the flag unready so the next caller runs it). The name equivalence is not confirmed on a title, and the callback convention (called with (null, arg, null), non-zero return meaning success) is the twin's, unverified for this symbol From 22c584acd39f4f3a40f5cd61d4ace063b135de13 Mon Sep 17 00:00:00 2001 From: T3cm0 Date: Sat, 17 Oct 2026 16:17:27 -0500 Subject: [PATCH 053/103] feat(libc): implement clearerr_unlocked, fileno_unlocked and fgetln Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libc/src/StandardStreams.cpp | 13 +++++++++++++ core/libs/tests/GuestStandardStreams.cpp | 20 ++++++++++++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 34 insertions(+) diff --git a/core/libs/prx/libc/src/StandardStreams.cpp b/core/libs/prx/libc/src/StandardStreams.cpp index 8d136d05b4..9d59cff429 100644 --- a/core/libs/prx/libc/src/StandardStreams.cpp +++ b/core/libs/prx/libc/src/StandardStreams.cpp @@ -1,6 +1,9 @@ #include "prx/libc/include/FileStream.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include +#include +#include +#include #ifdef _WIN32 #include #else @@ -134,6 +137,16 @@ int APS5_VABI fileno_nid_postfix(FileStream* stream) { stream->GuestState().descriptor = static_cast(descriptor); return descriptor; } +void APS5_VABI clearerr_unlocked_nid_postfix(FileStream* stream) { + stream->ClearError(); +} +int APS5_VABI fileno_unlocked_nid_postfix(FileStream* stream) { + if (stream == nullptr || !stream->IsOpen()) return -1; + return fileno_nid_postfix(stream); +} +[[noreturn]] char* APS5_VABI fgetln_nid_postfix(FileStream*, std::size_t*) { + throw std::runtime_error("fgetln: the console's export is a hlt instruction, which ends the process"); +} int APS5_VABI setvbuf_nid_postfix(FileStream* stream, char* buffer, int mode, std::size_t size) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } if (mode < 0 || mode > 2) return -1; diff --git a/core/libs/tests/GuestStandardStreams.cpp b/core/libs/tests/GuestStandardStreams.cpp index 9e99c39e09..dca63ca158 100644 --- a/core/libs/tests/GuestStandardStreams.cpp +++ b/core/libs/tests/GuestStandardStreams.cpp @@ -22,6 +22,9 @@ #endif extern "C" { FileStream* APS5_VABI fopen_nid_postfix(const char*, const char*); +void APS5_VABI clearerr_unlocked_nid_postfix(FileStream*); +int APS5_VABI fileno_unlocked_nid_postfix(FileStream*); +char* APS5_VABI fgetln_nid_postfix(FileStream*, std::size_t*); std::size_t APS5_VABI fread_nid_postfix(void*, std::size_t, std::size_t, FileStream*); std::size_t APS5_VABI fwrite_nid_postfix(const void*, std::size_t, std::size_t, FileStream*); FileStream* APS5_VABI freopen_nid_postfix(const char*, const char*, FileStream*); @@ -169,6 +172,22 @@ static void CheckUnlockedStreams() { Require(std::filesystem::remove(filename)); } +static void CheckUnlockedQueries() { + FileStream stream(std::tmpfile()); + Require(fputc_nid_postfix(0x7f, &stream) == 0x7f); + std::rewind(stream.GetHandle()); + Require(fgetc_nid_postfix(&stream) == 0x7f && feof_nid_postfix(&stream) == 0); + Require(fgetc_nid_postfix(&stream) == -1 && feof_nid_postfix(&stream) != 0); + clearerr_unlocked_nid_postfix(&stream); + Require(feof_nid_postfix(&stream) == 0 && ferror_nid_postfix(&stream) == 0); + Require(fileno_unlocked_nid_postfix(&stream) == fileno_nid_postfix(&stream) && fileno_unlocked_nid_postfix(&stream) >= 0); + Require(fileno_unlocked_nid_postfix(nullptr) == -1); + bool halted = false; + std::size_t length = 0; + try { fgetln_nid_postfix(&stream, &length); } catch (const std::runtime_error&) { halted = true; } + Require(halted); +} + static void CheckTryLock() { FileStream stream(std::tmpfile()); const auto tryFromOtherThread = [&] { @@ -843,5 +862,6 @@ int main(int argc, char** argv) { CheckUnlockedStreamLock(false); CheckUnlockedStreamLock(true); CheckTryLock(); + CheckUnlockedQueries(); return CheckBinaryModes() ? 0 : 1; } diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index ff2096892f..d3ca1c8970 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -47,6 +47,7 @@ - [getc_unlocked, putc_unlocked, getchar_unlocked, putchar_unlocked](../../../core/libs/prx/libc/src/StandardStreams.cpp) (libc) - byte I/O uses the host unlocked operations; guest EOF/error flags and macro counters follow `FileStream`. Linux status queries use `feof_unlocked`/`ferror_unlocked` because [glibc `feof`](https://github.com/bminor/glibc/blob/d2097651cc57834dbfcaa102ddfacae0d86cfb66/libio/feof.c) and [`ferror`](https://github.com/bminor/glibc/blob/d2097651cc57834dbfcaa102ddfacae0d86cfb66/libio/ferror.c) can acquire the stream lock. Windows uses [the CRT no-lock byte operations](https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/fgetc-nolock-fgetwc-nolock) and `feof`/`ferror`, verified under a held stream lock on UCRT. Console behavior and other host CRTs remain unverified. - [fgetws, getwc, getwchar, putwc, putwchar](../../../core/libs/prx/libc/src/FormattingWide.cpp) (libc) - wide stream I/O encodes and decodes wide characters as UTF-8 on the underlying byte stream; `fgetws` reads characters into the destination buffer until newline or EOF and null-terminates, returning null on stream error. Console wide stream buffering and encoding details remain unverified. - [ftrylockfile](../../../core/libs/prx/libc/src/RuntimeSupport.cpp) (libc) - tries the host CRT stream lock that `flockfile`, `funlockfile` and each locking host stdio call on the stream take, so it returns -1 while another thread holds the stream through `flockfile` or is inside such a call, and 0 when it acquires the lock or already owns it (recursively, released by as many `funlockfile` calls), as FreeBSD 11 [`_ftrylockfile`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/stdio/_flock_stub.c#L89-L104) does. A libc export holds the lock only for each host call it makes, not across several host calls or while it updates the guest `FILE` prefix. Linux uses glibc `ftrylockfile`. UCRT has no try variant of `_lock_file`, so Windows calls `TryEnterCriticalSection` on the `CRITICAL_SECTION` that `_lock_file` enters, found through the private UCRT stream layout (`__crt_stdio_stream_data` in `ucrt/inc/corecrt_internal_stdio.h`, unchanged from Windows SDK 10.0.19041 to 10.0.26100); another CRT or a change of that layout breaks it. A null or closed stream throws where FreeBSD dereferences it +- [fgetln](../../../core/libs/prx/libc/src/StandardStreams.cpp) (libSceLibcInternal, implemented in libc) throws: in the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x5FE80) the export is a `hlt` instruction, which faults in user mode and ends the process - [Wide-character conversions](../../../core/libs/prx/libc/src/Multibyte.cpp) (libc) - The wide `wcsto*` functions parse the ASCII prefix of the guest string, as the C locale accepts only ASCII white space and digits. [`wcsrtombs`](../../../core/libs/prx/libc/src/Multibyte.cpp) uses the guest's single-byte C locale through `wcrtomb`; other encodings and shift states are not supported - [realpath](../../../core/libs/prx/libc/src/RealPath.cpp) (libc) follows FreeBSD 11's [`realpath.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/libc/stdlib/realpath.c) except for symlinks. The console's guest filesystem has none, so a host symlink under the guest root is followed for existence and type but not expanded, and `..` drops the previous component lexically, as `Resolve` does for every guest path. Through a host link `linkdir -> sub/deep`, `linkdir/../x` gives `x` beside the link, not beside its target. - [strtod, strtof, strtold](../../../core/libs/prx/libc/src/Strings.cpp) (libc) - `NAN(n-char-sequence)` converts to the quiet NaN that `nan` and `nanf` return, negated after a minus sign, and ignores the sequence as they ignore their tag. This is inferred from C17 7.12.11.2p2, which makes `nan("n")` equivalent to `strtod("NAN(n)", NULL)`, and not observed on the console. No console `nanl` was observed, so `strtold` returns the x87 default quiet NaN (exponent `0x7fff`, significand `0xc000000000000000`) From aed1ffbae85041ec22e1e27b93b4bc1d69c80522 Mon Sep 17 00:00:00 2001 From: oskarlgs Date: Sat, 17 Oct 2026 21:55:21 +0000 Subject: [PATCH 054/103] fix(relinker): keep the Windows TLS rewrite from moving truncated branches A short TLS access followed by a near branch with an operand-size prefix decided the branch was relative from its flow kind and opcode alone, then re-encoded it to the fall-through address or wrote its rel32 over the prefix and opcode. Such a branch now counts as one that cannot move. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016TFMtUC4L4wMS4eeEqaNSR --- core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp | 4 ++-- core/relinker/relinker/tests/test_tls_function_coverage.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp index e50f17c0ce..19ff13e378 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp @@ -158,8 +158,8 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co throw Domain::RelinkerException("Short guest TLS instruction ends its segment", header.Offset + offset); const auto next = decoder.DecodeInstruction(bytes + length, header.FileSize - offset - length); const bool relativeBranch = - (next.FlowKind == Codegen::ControlFlowKind::ConditionalBranch && (next.IsTwoByteOpcode ? next.Opcode >= 0x80 && next.Opcode <= 0x8f : next.Opcode >= 0x70 && next.Opcode <= 0x7f)) || - (next.FlowKind == Codegen::ControlFlowKind::UnconditionalJump && !next.HasRipRelativeDisp && !next.IsTwoByteOpcode && (next.Opcode == 0xe9 || next.Opcode == 0xeb)); + (next.FlowKind == Codegen::ControlFlowKind::ConditionalBranch && !next.TruncatesBranchTarget && (next.IsTwoByteOpcode ? next.Opcode >= 0x80 && next.Opcode <= 0x8f : next.Opcode >= 0x70 && next.Opcode <= 0x7f)) || + (next.FlowKind == Codegen::ControlFlowKind::UnconditionalJump && !next.TruncatesBranchTarget && !next.HasRipRelativeDisp && !next.IsTwoByteOpcode && (next.Opcode == 0xe9 || next.Opcode == 0xeb)); if (!relativeBranch && (next.FlowKind != Codegen::ControlFlowKind::Sequential || next.HasBranchTarget || next.SegmentPrefix != 0)) throw Domain::RelinkerException("Short guest TLS instruction is followed by an instruction that cannot move", header.Offset + offset + length); std::vector movedBytes(bytes + length, bytes + length + next.Length); diff --git a/core/relinker/relinker/tests/test_tls_function_coverage.py b/core/relinker/relinker/tests/test_tls_function_coverage.py index 6c30aada25..ef20c1c1a1 100644 --- a/core/relinker/relinker/tests/test_tls_function_coverage.py +++ b/core/relinker/relinker/tests/test_tls_function_coverage.py @@ -747,6 +747,10 @@ def main(): "short-before-tls": bytes.fromhex("64 48 8b 03 64 48 8b 03 c3"), "short-alu-before-return": bytes.fromhex("64 48 33 03 c3"), "short-before-loop": bytes.fromhex("64 48 33 03 e2 00"), + "short-before-word-jcc8": bytes.fromhex("64 48 33 03 66 75 00 c3"), + "short-before-word-jmp8": bytes.fromhex("64 48 33 03 66 eb 00 c3"), + "short-before-word-jmp16": bytes.fromhex("64 48 33 03 66 e9 00 00 c3"), + "short-before-word-jcc16": bytes.fromhex("64 48 33 03 66 0f 85 00 00 c3"), }.items(): convert(name, make_image("register", "unwind", body=instruction), "Short guest TLS instruction is followed by an instruction that cannot move", error_offset=0x1244) From f4b3f035803c62f53380c21ab0f3e531f11b1636 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 18 Oct 2026 02:16:42 +0400 Subject: [PATCH 055/103] refactor(shader): gather the emulated barycentric interface in one place --- .../src/BarycentricGeometryShader.cpp | 58 ++++++++++++------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp b/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp index 7c86bc826b..1ab923070f 100644 --- a/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp +++ b/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp @@ -24,6 +24,36 @@ struct PerVertexParameter { std::uint32_t location; }; +struct EmulatedInterface { + std::vector passed; + std::vector perVertex; + BarycentricEmulationLayout layout; + std::uint32_t barycentrics = 0; + std::uint32_t sources = 0; +}; + +EmulatedInterface CollectEmulatedInterface(const RecompileResult& vertex, const RecompileResult& fragment) { + require(fragment.barycentricEmulation.active, "the fragment shader does not emulate barycentrics"); + const auto exported = [&](std::uint32_t location) { return std::find(vertex.parameterExports.begin(), vertex.parameterExports.end(), location) != vertex.parameterExports.end(); }; + EmulatedInterface emulated; + emulated.layout = LayoutBarycentricEmulation(fragment.fragmentParameters, fragment.barycentricEmulation); + for (const auto& parameter : fragment.fragmentParameters) { + require(exported(parameter.sourceLocation), "fragment parameter " + std::to_string(parameter.location) + " is not exported by the vertex shader"); + if (!parameter.perVertex) { + emulated.passed.push_back(parameter.sourceLocation); + continue; + } + const auto relocated = std::find_if(emulated.layout.perVertexLocations.begin(), emulated.layout.perVertexLocations.end(), [&](const auto& entry) { return entry.first == parameter.location; }); + require(relocated != emulated.layout.perVertexLocations.end(), "per-vertex parameter " + std::to_string(parameter.location) + " has no emulated location"); + emulated.perVertex.push_back({parameter.sourceLocation, relocated->second}); + } + emulated.barycentrics = (emulated.layout.smoothLocation != BarycentricEmulationLayout::NoLocation ? 1u : 0u) + (emulated.layout.linearLocation != BarycentricEmulationLayout::NoLocation ? 1u : 0u); + std::set sources(emulated.passed.begin(), emulated.passed.end()); + for (const auto& parameter : emulated.perVertex) sources.insert(parameter.sourceLocation); + emulated.sources = static_cast(sources.size()); + return emulated; +} + class BarycentricGeometryEmitter { public: explicit BarycentricGeometryEmitter(std::uint32_t version) : builder(version) { @@ -171,30 +201,16 @@ RecompileResult BuildBarycentricGeometryShader(const RecompileResult& vertex, co require(limits.has_value(), "geometry shaders are unavailable"); require(target.spirvVersion >= 0x00010300u && target.spirvVersion <= 0x00010500u, "unsupported SPIR-V target version"); require(std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), spv::CapabilityGeometry) != target.supportedCapabilities.end(), "geometry capability is unavailable"); - const auto exported = [&](std::uint32_t location) { return std::find(vertex.parameterExports.begin(), vertex.parameterExports.end(), location) != vertex.parameterExports.end(); }; - const auto layout = LayoutBarycentricEmulation(fragment.fragmentParameters, fragment.barycentricEmulation); - std::vector passed; - std::vector perVertex; - for (const auto& parameter : fragment.fragmentParameters) { - require(exported(parameter.sourceLocation), "fragment parameter " + std::to_string(parameter.location) + " is not exported by the vertex shader"); - if (!parameter.perVertex) { - passed.push_back(parameter.sourceLocation); - continue; - } - const auto relocated = std::find_if(layout.perVertexLocations.begin(), layout.perVertexLocations.end(), [&](const auto& entry) { return entry.first == parameter.location; }); - require(relocated != layout.perVertexLocations.end(), "per-vertex parameter " + std::to_string(parameter.location) + " has no emulated location"); - perVertex.push_back({parameter.sourceLocation, relocated->second}); - } - const auto barycentrics = (layout.smoothLocation != BarycentricEmulationLayout::NoLocation ? 1u : 0u) + (layout.linearLocation != BarycentricEmulationLayout::NoLocation ? 1u : 0u); - const auto outputComponents = 4u + 4u * static_cast(passed.size()) + 3u * barycentrics + 12u * static_cast(perVertex.size()); - std::set sources(passed.begin(), passed.end()); - for (const auto& parameter : perVertex) sources.insert(parameter.sourceLocation); - const auto inputComponents = 4u + 4u * static_cast(sources.size()); - const auto fragmentComponents = 4u * (static_cast(passed.size()) + barycentrics + 3u * static_cast(perVertex.size())); + const auto emulated = CollectEmulatedInterface(vertex, fragment); + const auto passed = static_cast(emulated.passed.size()); + const auto perVertex = static_cast(emulated.perVertex.size()); + const auto outputComponents = 4u + 4u * passed + 3u * emulated.barycentrics + 12u * perVertex; + const auto inputComponents = 4u + 4u * emulated.sources; + const auto fragmentComponents = 4u * (passed + emulated.barycentrics + 3u * perVertex); require(limits->maxGeometryOutputVertices >= 3u && outputComponents <= limits->maxGeometryOutputComponents && 3u * outputComponents <= limits->maxGeometryTotalOutputComponents && inputComponents <= limits->maxGeometryInputComponents && fragmentComponents <= limits->maxFragmentInputComponents, "the emulated barycentric interface exceeds device limits"); BarycentricGeometryEmitter emitter(target.spirvVersion); RecompileResult result; - result.spirv = emitter.Emit(passed, perVertex, layout); + result.spirv = emitter.Emit(emulated.passed, emulated.perVertex, emulated.layout); #if ANYPS5_ENABLE_SPIRV_TOOLS result.spirv = ValidateAndOptimizeSpirv(result.spirv, target.vulkanVersion, target.spirvVersion); #endif From 02a4bdf08ada428b2f210d605ddb942c1068168c Mon Sep 17 00:00:00 2001 From: David Date: Sun, 18 Oct 2026 02:16:51 +0400 Subject: [PATCH 056/103] feat(shader): build the barycentric emulation stages from tessellation On a device without geometry shaders, each triangle becomes a three-point patch at tessellation level 1. The evaluation stage takes the corner gl_TessCoord points at as its vertex and writes it as the weights, and passes every vertex's parameters for raw reads. The vertex clip distances are forwarded, so MoltenVK's stage interface matches the vertex output. --- core/shader/recompiler/CMakeLists.txt | 1 + core/shader/recompiler/Recompiler.hpp | 7 + .../src/BarycentricGeometryShader.cpp | 220 ++++++++++++++++++ 3 files changed, 228 insertions(+) diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index a14e40ce48..b196d53f5b 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -221,6 +221,7 @@ function(add_shader_recompiler target runtimeTarget) endif() set(shaderCacheReviewedNames BuildBarycentricGeometryShader + BuildBarycentricTessellationShaders BuildPreparedShaderKey BuildRectListShaders CaptureResources diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 123c488f40..39fc70798c 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -697,6 +697,13 @@ struct GeometryStageLimits { [[nodiscard]] RecompileResult BuildBarycentricGeometryShader(const RecompileResult& vertex, const RecompileResult& fragment, const SpirvTarget& target, const std::optional& limits); +struct BarycentricTessellationShaders { + RecompileResult control; + RecompileResult evaluation; +}; + +[[nodiscard]] BarycentricTessellationShaders BuildBarycentricTessellationShaders(const RecompileResult& vertex, const RecompileResult& fragment, const SpirvTarget& target, std::uint32_t maxFragmentInputComponents); + } #endif diff --git a/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp b/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp index 1ab923070f..8b9b6d5710 100644 --- a/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp +++ b/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp @@ -1,4 +1,5 @@ #include "Recompiler.hpp" +#include "SpirvBackend/SpirvClipDistances.hpp" #include "SpirvBackend/SpirvModule.hpp" #if ANYPS5_ENABLE_SPIRV_TOOLS #include "SpirvBackend/SpirvOptimizer.hpp" @@ -26,6 +27,7 @@ struct PerVertexParameter { struct EmulatedInterface { std::vector passed; + std::vector passedFlat; std::vector perVertex; BarycentricEmulationLayout layout; std::uint32_t barycentrics = 0; @@ -41,6 +43,7 @@ EmulatedInterface CollectEmulatedInterface(const RecompileResult& vertex, const require(exported(parameter.sourceLocation), "fragment parameter " + std::to_string(parameter.location) + " is not exported by the vertex shader"); if (!parameter.perVertex) { emulated.passed.push_back(parameter.sourceLocation); + emulated.passedFlat.push_back(parameter.flat); continue; } const auto relocated = std::find_if(emulated.layout.perVertexLocations.begin(), emulated.layout.perVertexLocations.end(), [&](const auto& entry) { return entry.first == parameter.location; }); @@ -170,6 +173,195 @@ private: } }; +class BarycentricTessellationEmitter { +public: + BarycentricTessellationEmitter(std::uint32_t version, std::uint32_t clipDistances) : builder(version), clipDistances(clipDistances) { + builder.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450); + voidType = builder.Type(spv::OpTypeVoid); + boolType = builder.Type(spv::OpTypeBool); + uintType = builder.Type(spv::OpTypeInt, 32u, 0u); + intType = builder.Type(spv::OpTypeInt, 32u, 1u); + floatType = builder.Type(spv::OpTypeFloat, 32u); + vec3FloatType = builder.Type(spv::OpTypeVector, floatType, 3u); + vec4FloatType = builder.Type(spv::OpTypeVector, floatType, 4u); + vertexArrayType = builder.Type(spv::OpTypeArray, vec4FloatType, constant(uintType, 3u)); + functionType = builder.Type(spv::OpTypeFunction, voidType); + if (clipDistances != 0u) { + clipArrayType = builder.Type(spv::OpTypeArray, floatType, constant(uintType, clipDistances)); + perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpMemberDecorate, {1u, spv::DecorationBuiltIn, spv::BuiltInClipDistance}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType, clipArrayType); + ptrInputClip = builder.Type(spv::OpTypePointer, spv::StorageClassInput, clipArrayType); + ptrOutputClip = builder.Type(spv::OpTypePointer, spv::StorageClassOutput, clipArrayType); + } else { + perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType); + } + perVertexArrayType = builder.Type(spv::OpTypeArray, perVertexType, constant(uintType, 3u)); + ptrInputVec4 = builder.Type(spv::OpTypePointer, spv::StorageClassInput, vec4FloatType); + ptrOutputVec4 = builder.Type(spv::OpTypePointer, spv::StorageClassOutput, vec4FloatType); + } + + std::vector EmitControl(const std::vector& sources) { + begin(); + builder.AddExecutionMode(main, spv::ExecutionModeOutputVertices, 3u); + addDomain(); + const auto invocationId = addInterface(spv::StorageClassInput, intType); + decorate(invocationId, spv::DecorationBuiltIn, spv::BuiltInInvocationId); + const auto glIn = addInterface(spv::StorageClassInput, perVertexArrayType); + const auto glOut = addInterface(spv::StorageClassOutput, perVertexArrayType); + const auto floatArray = [&](std::uint32_t size) { return builder.Type(spv::OpTypeArray, floatType, constant(uintType, size)); }; + const auto outer = addInterface(spv::StorageClassOutput, floatArray(4u)); + decorate(outer, spv::DecorationBuiltIn, spv::BuiltInTessLevelOuter); + builder.AddAnnotation(spv::OpDecorate, outer, spv::DecorationPatch); + const auto inner = addInterface(spv::StorageClassOutput, floatArray(2u)); + decorate(inner, spv::DecorationBuiltIn, spv::BuiltInTessLevelInner); + builder.AddAnnotation(spv::OpDecorate, inner, spv::DecorationPatch); + std::vector> parameters; + for (const auto location : sources) { + const auto input = addInterface(spv::StorageClassInput, vertexArrayType); + decorate(input, spv::DecorationLocation, location); + const auto output = addInterface(spv::StorageClassOutput, vertexArrayType); + decorate(output, spv::DecorationLocation, location); + parameters.emplace_back(input, output); + } + builder.EmitEntryPoint(spv::ExecutionModelTessellationControl, main, "main", interfaces); + builder.AddFunction(spv::OpLabel, builder.AllocateId()); + + const auto invocation = load(intType, invocationId); + const auto first = result(spv::OpIEqual, boolType, invocation, constant(intType, 0u)); + const auto writeLevels = builder.AllocateId(); + const auto afterLevels = builder.AllocateId(); + builder.AddFunction(spv::OpSelectionMerge, afterLevels, spv::SelectionControlMaskNone); + builder.AddFunction(spv::OpBranchConditional, first, writeLevels, afterLevels); + builder.AddFunction(spv::OpLabel, writeLevels); + const auto ptrOutputFloat = builder.Type(spv::OpTypePointer, spv::StorageClassOutput, floatType); + const auto one = constant(floatType, std::bit_cast(1.0f)); + for (std::uint32_t i = 0; i < 4u; ++i) store(result(spv::OpAccessChain, ptrOutputFloat, outer, constant(intType, i)), one); + for (std::uint32_t i = 0; i < 2u; ++i) store(result(spv::OpAccessChain, ptrOutputFloat, inner, constant(intType, i)), one); + builder.AddFunction(spv::OpBranch, afterLevels); + builder.AddFunction(spv::OpLabel, afterLevels); + store(result(spv::OpAccessChain, ptrOutputVec4, glOut, invocation, constant(intType, 0u)), load(vec4FloatType, result(spv::OpAccessChain, ptrInputVec4, glIn, invocation, constant(intType, 0u)))); + if (clipDistances != 0u) store(result(spv::OpAccessChain, ptrOutputClip, glOut, invocation, constant(intType, 1u)), load(clipArrayType, result(spv::OpAccessChain, ptrInputClip, glIn, invocation, constant(intType, 1u)))); + for (const auto& [input, output] : parameters) store(result(spv::OpAccessChain, ptrOutputVec4, output, invocation), load(vec4FloatType, result(spv::OpAccessChain, ptrInputVec4, input, invocation))); + return finish(); + } + + std::vector EmitEvaluation(const std::vector& sources, const std::vector& passed, const std::vector& passedFlat, const std::vector& perVertex, const BarycentricEmulationLayout& layout) { + begin(); + addDomain(); + const auto tessCoord = addInterface(spv::StorageClassInput, vec3FloatType); + decorate(tessCoord, spv::DecorationBuiltIn, spv::BuiltInTessCoord); + const auto glIn = addInterface(spv::StorageClassInput, perVertexArrayType); + std::array inputs {}; + for (const auto location : sources) { + inputs[location] = addInterface(spv::StorageClassInput, vertexArrayType); + decorate(inputs[location], spv::DecorationLocation, location); + } + const auto glOut = addInterface(spv::StorageClassOutput, perVertexType); + std::vector passedOutputs; + for (const auto location : passed) { + passedOutputs.push_back(addInterface(spv::StorageClassOutput, vec4FloatType)); + decorate(passedOutputs.back(), spv::DecorationLocation, location); + } + const auto barycentricOutput = [&](std::uint32_t location) { + if (location == BarycentricEmulationLayout::NoLocation) return 0u; + const auto variable = addInterface(spv::StorageClassOutput, vec3FloatType); + decorate(variable, spv::DecorationLocation, location); + return variable; + }; + const auto smooth = barycentricOutput(layout.smoothLocation); + const auto linear = barycentricOutput(layout.linearLocation); + std::vector perVertexOutputs; + for (const auto& parameter : perVertex) { + perVertexOutputs.push_back(addInterface(spv::StorageClassOutput, vertexArrayType)); + decorate(perVertexOutputs.back(), spv::DecorationLocation, parameter.location); + } + builder.EmitEntryPoint(spv::ExecutionModelTessellationEvaluation, main, "main", interfaces); + builder.AddFunction(spv::OpLabel, builder.AllocateId()); + + const auto ptrInputFloat = builder.Type(spv::OpTypePointer, spv::StorageClassInput, floatType); + const auto half = constant(floatType, std::bit_cast(0.5f)); + const auto zero = constant(floatType, 0u); + const auto one = constant(floatType, std::bit_cast(1.0f)); + std::array corner {}; + for (std::uint32_t i = 0; i < 3u; ++i) corner[i] = result(spv::OpFOrdGreaterThan, boolType, load(floatType, result(spv::OpAccessChain, ptrInputFloat, tessCoord, constant(intType, i))), half); + auto index = result(spv::OpSelect, intType, corner[2], constant(intType, 2u), constant(intType, 0u)); + index = result(spv::OpSelect, intType, corner[1], constant(intType, 1u), index); + const auto weight = [&](std::uint32_t vertex) { return result(spv::OpSelect, floatType, result(spv::OpIEqual, boolType, index, constant(intType, vertex)), one, zero); }; + const auto weights = result(spv::OpCompositeConstruct, vec3FloatType, weight(0u), weight(1u), weight(2u)); + + store(result(spv::OpAccessChain, ptrOutputVec4, glOut, constant(intType, 0u)), load(vec4FloatType, result(spv::OpAccessChain, ptrInputVec4, glIn, index, constant(intType, 0u)))); + if (clipDistances != 0u) store(result(spv::OpAccessChain, ptrOutputClip, glOut, constant(intType, 1u)), load(clipArrayType, result(spv::OpAccessChain, ptrInputClip, glIn, index, constant(intType, 1u)))); + for (std::size_t i = 0; i < passed.size(); ++i) store(passedOutputs[i], load(vec4FloatType, result(spv::OpAccessChain, ptrInputVec4, inputs[passed[i]], passedFlat[i] ? constant(intType, 0u) : index))); + if (smooth != 0u) store(smooth, weights); + if (linear != 0u) store(linear, weights); + for (std::size_t i = 0; i < perVertex.size(); ++i) { + for (std::uint32_t source = 0; source < 3u; ++source) store(result(spv::OpAccessChain, ptrOutputVec4, perVertexOutputs[i], constant(intType, source)), load(vec4FloatType, result(spv::OpAccessChain, ptrInputVec4, inputs[perVertex[i].sourceLocation], constant(intType, source)))); + } + return finish(); + } + +private: + SpirvModule builder; + std::uint32_t clipDistances; + std::vector interfaces; + std::uint32_t main = 0; + std::uint32_t voidType = 0; + std::uint32_t boolType = 0; + std::uint32_t uintType = 0; + std::uint32_t intType = 0; + std::uint32_t floatType = 0; + std::uint32_t vec3FloatType = 0; + std::uint32_t vec4FloatType = 0; + std::uint32_t vertexArrayType = 0; + std::uint32_t functionType = 0; + std::uint32_t perVertexType = 0; + std::uint32_t perVertexArrayType = 0; + std::uint32_t clipArrayType = 0; + std::uint32_t ptrInputVec4 = 0; + std::uint32_t ptrOutputVec4 = 0; + std::uint32_t ptrInputClip = 0; + std::uint32_t ptrOutputClip = 0; + + void begin() { + builder.EmitCapability(spv::CapabilityShader); + builder.EmitCapability(spv::CapabilityTessellation); + if (clipDistances != 0u) builder.EmitCapability(spv::CapabilityClipDistance); + main = result(spv::OpFunction, voidType, spv::FunctionControlMaskNone, functionType); + } + + void addDomain() { + builder.AddExecutionMode(main, spv::ExecutionModeTriangles); + builder.AddExecutionMode(main, spv::ExecutionModeSpacingEqual); + builder.AddExecutionMode(main, spv::ExecutionModeVertexOrderCw); + } + + std::vector finish() { + builder.AddFunction(spv::OpReturn); + builder.AddFunction(spv::OpFunctionEnd); + return builder.Finalize(); + } + + std::uint32_t constant(std::uint32_t type, std::uint32_t value) { return builder.Constant(spv::OpConstant, type, value); } + + template + std::uint32_t result(spv::Op opcode, std::uint32_t type, TArgs... operands) { + const auto id = builder.AllocateId(); + builder.AddFunction(opcode, type, id, operands...); + return id; + } + + std::uint32_t load(std::uint32_t type, std::uint32_t pointer) { return result(spv::OpLoad, type, pointer); } + + void store(std::uint32_t pointer, std::uint32_t value) { builder.AddFunction(spv::OpStore, pointer, value); } + + void decorate(std::uint32_t target, spv::Decoration decoration, std::uint32_t value) { builder.AddAnnotation(spv::OpDecorate, target, decoration, value); } + + std::uint32_t addInterface(spv::StorageClass storage, std::uint32_t type) { + const auto variable = builder.DefineGlobalVariable(builder.Type(spv::OpTypePointer, storage, type), storage); + interfaces.push_back(variable); + return variable; + } +}; + } BarycentricEmulationLayout LayoutBarycentricEmulation(std::span parameters, const BarycentricEmulation& emulation) { @@ -217,4 +409,32 @@ RecompileResult BuildBarycentricGeometryShader(const RecompileResult& vertex, co return result; } +BarycentricTessellationShaders BuildBarycentricTessellationShaders(const RecompileResult& vertex, const RecompileResult& fragment, const SpirvTarget& target, std::uint32_t maxFragmentInputComponents) { + require(fragment.barycentricEmulation.active, "the fragment shader does not emulate barycentrics"); + require(target.tessellation.has_value(), "tessellation shaders are unavailable"); + require(target.spirvVersion >= 0x00010300u && target.spirvVersion <= 0x00010500u, "unsupported SPIR-V target version"); + require(std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), spv::CapabilityTessellation) != target.supportedCapabilities.end(), "tessellation capability is unavailable"); + const auto emulated = CollectEmulatedInterface(vertex, fragment); + const auto passed = static_cast(emulated.passed.size()); + const auto perVertex = static_cast(emulated.perVertex.size()); + const auto clipDistances = VertexClipDistanceCount(vertex.spirv.Words()); + require(clipDistances == 0u || std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), spv::CapabilityClipDistance) != target.supportedCapabilities.end(), "the vertex clip distances need shaderClipDistance"); + const auto controlComponents = 4u + clipDistances + 4u * emulated.sources; + const auto evaluationOutputComponents = 4u + clipDistances + 4u * passed + 3u * emulated.barycentrics + 12u * perVertex; + const auto fragmentComponents = 4u * (passed + emulated.barycentrics + 3u * perVertex); + const auto& limits = *target.tessellation; + require(limits.maxPatchSize >= 3u && controlComponents <= limits.maxControlPerVertexInputComponents && controlComponents <= limits.maxControlPerVertexOutputComponents && 3u * controlComponents + 4u <= limits.maxControlTotalOutputComponents && limits.maxControlPerPatchOutputComponents >= 4u && controlComponents <= limits.maxEvaluationInputComponents && evaluationOutputComponents <= limits.maxEvaluationOutputComponents && fragmentComponents <= maxFragmentInputComponents, "the emulated barycentric interface exceeds device limits"); + std::set sourceSet(emulated.passed.begin(), emulated.passed.end()); + for (const auto& parameter : emulated.perVertex) sourceSet.insert(parameter.sourceLocation); + const std::vector sources(sourceSet.begin(), sourceSet.end()); + BarycentricTessellationShaders shaders; + shaders.control.spirv = BarycentricTessellationEmitter(target.spirvVersion, clipDistances).EmitControl(sources); + shaders.evaluation.spirv = BarycentricTessellationEmitter(target.spirvVersion, clipDistances).EmitEvaluation(sources, emulated.passed, emulated.passedFlat, emulated.perVertex, emulated.layout); +#if ANYPS5_ENABLE_SPIRV_TOOLS + shaders.control.spirv = ValidateAndOptimizeSpirv(shaders.control.spirv, target.vulkanVersion, target.spirvVersion); + shaders.evaluation.spirv = ValidateAndOptimizeSpirv(shaders.evaluation.spirv, target.vulkanVersion, target.spirvVersion); +#endif + return shaders; +} + } From 03f4f6718e5adc6d44dfc0028c34af179adf83d8 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 18 Oct 2026 02:16:51 +0400 Subject: [PATCH 057/103] feat(agc): emulate barycentrics through tessellation without geometry shaders A triangle list without primitive restart whose pixel shader emulates barycentrics draws through the generated tessellation stages when the device has tessellation but no geometry shaders, as on MoltenVK. The pipeline uses a three-point patch list for it, keeps it out of pipeline libraries and validates the generated stages' modes and interface. --- .../Execution/include/VulkanDevice.hpp | 1 + .../Execution/src/Driver/Draw/Draw.cpp | 14 ++++++++++++-- .../Execution/src/VulkanDevice.cpp | 4 ++++ .../Graphics/include/Pipeline.hpp | 1 + .../libSceAgcDriver/Graphics/src/Pipeline.cpp | 16 +++++++++++----- .../Graphics/src/ShaderValidation.cpp | 17 +++++++++++++---- 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 4aa2fa07f2..766df896cc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -55,6 +55,7 @@ public: bool CalibrateClock(double& gpuNs, std::chrono::steady_clock::time_point& cpu) const; ShaderRecompiler::SpirvTarget Target() const; std::optional GeometryLimits() const; + std::uint32_t MaxFragmentInputComponents() const; const std::uint32_t* WorkGroupCountLimit() const; std::uint32_t VendorId() const; ShaderRecompiler::SpirvTarget ComputeTarget(std::uint32_t waveSize) const; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index 5dea7922ea..b8e705e6dc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -451,8 +451,18 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe if (graphics.rectList) buildRectList(); if (!programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) { require(!graphics.rectList && graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it"); - results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), localDevice->GeometryLimits())); - stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0}); + const auto geometryLimits = localDevice->GeometryLimits(); + if (geometryLimits || !localDevice->Target().tessellation) { + results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), geometryLimits)); + stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0}); + } else { + require(graphics.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST && !graphics.primitiveRestart, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric or geometry shaders needs a triangle list without primitive restart"); + auto patches = ShaderRecompiler::BuildBarycentricTessellationShaders(*programResults[0], *programResults[1], localDevice->Target(), localDevice->MaxFragmentInputComponents()); + const auto first = results.size(); + results.push_back(std::move(patches.control)); + results.push_back(std::move(patches.evaluation)); + stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[first], 0}, {Stage::TessellationEvaluation, &results[first + 1], 0}}); + } } std::vector snapshots; const auto snapshot = [&] { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 62afeaea77..bec1e91bdc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -3507,6 +3507,10 @@ std::optional VulkanDevice::GeometryLimit return ShaderRecompiler::GeometryStageLimits{limits.maxGeometryInputComponents, limits.maxGeometryOutputComponents, limits.maxGeometryOutputVertices, limits.maxGeometryTotalOutputComponents, limits.maxFragmentInputComponents}; } +std::uint32_t VulkanDevice::MaxFragmentInputComponents() const { + return state->properties.limits.maxFragmentInputComponents; +} + bool VulkanDevice::PrimitiveListRestart() const { return state->primitiveListRestart; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp index 6bd3a8dd82..078e3f8818 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp @@ -134,6 +134,7 @@ bool HasStencil(VkFormat format); void ValidateViewport(const Context& context, const VkViewport& viewport); void ValidateDepthBounds(const Context& context, const State& state); void ValidateProvokingVertex(const Context& context, const State& state, std::span shaders = {}); +bool BarycentricPatches(const State& state, std::span shaders); bool ValidateSampleLocations(const Context& context, const State& state); void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index d9a9c96bdc..0d65ea65f6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -96,6 +96,10 @@ void ValidateDepthBounds(const Context& context, const State& state) { Require(!state.depthBoundsTest || context.depthRangeUnrestricted || (state.minDepthBounds >= 0.0f && state.minDepthBounds <= 1.0f && state.maxDepthBounds >= 0.0f && state.maxDepthBounds <= 1.0f), "depth bounds outside [0, 1] require VK_EXT_depth_range_unrestricted"); } +bool BarycentricPatches(const State& state, std::span shaders) { + return !state.rectList && state.stages.path == ShaderPath::Vertex && std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::TessellationControl; }); +} + void ValidateProvokingVertex(const Context& context, const State& state, std::span shaders) { Require(state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT || state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT, "invalid provoking vertex mode"); if (state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT) { @@ -106,7 +110,7 @@ void ValidateProvokingVertex(const Context& context, const State& state, std::sp return; } Require(context.provokingVertexLast, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled"); - Require(!state.rectList && state.stages.path == ShaderPath::Vertex && !state.stages.mesh && !state.stages.tessellation, "last provoking vertex is unsupported for generated primitive pipelines"); + Require(!state.rectList && !BarycentricPatches(state, shaders) && state.stages.path == ShaderPath::Vertex && !state.stages.mesh && !state.stages.tessellation, "last provoking vertex is unsupported for generated primitive pipelines"); Require(state.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, "last provoking vertex is unsupported for this primitive topology"); for (const auto& shader : shaders) { if (shader.stage != ShaderRecompiler::ShaderStage::Fragment) continue; @@ -192,7 +196,7 @@ void appendKey(std::vector& key, const TValue& value) { bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span shaders) { using Stage = ShaderRecompiler::ShaderStage; - if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || SplitsAlternateAlpha(state)) return false; + if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state)) return false; keys = {}; for (const auto& shader : shaders) { const auto identity = shader.program->Identity(); @@ -310,6 +314,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput Require(state.conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT || context.conservativeRasterization, "conservative rasterization requires VK_EXT_conservative_rasterization with at most 1/256 pixel of overestimation and degenerate triangles rasterized"); ValidateProvokingVertex(context, state, shaders); if (state.rectList) Require(context.tessellationShader && context.limits.maxTessellationPatchSize >= 4, "rect-list requires tessellation with four output control points"); + const bool barycentricPatches = BarycentricPatches(state, shaders); + if (barycentricPatches) Require(context.tessellationShader && state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST && !state.primitiveRestart, "emulated barycentrics through tessellation require a triangle list without primitive restart"); if (std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::Geometry; })) Require(context.geometryShader, "device does not support geometry shaders"); if (state.stages.tessellation) { Require(context.tessellationShader, "device does not support tessellation shaders"); @@ -425,7 +431,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput input.vertexAttributeDescriptionCount = static_cast(vertexInput.attributes.size()); input.pVertexAttributeDescriptions = vertexInput.attributes.data(); VkPipelineInputAssemblyStateCreateInfo assembly{VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO}; - assembly.topology = state.topology; + assembly.topology = BarycentricPatches(state, shaders) ? VK_PRIMITIVE_TOPOLOGY_PATCH_LIST : state.topology; const bool listTopology = state.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; assembly.primitiveRestartEnable = state.primitiveRestart && (!listTopology || context.primitiveListRestart) ? VK_TRUE : VK_FALSE; // Viewport and scissor are set per draw (Begin), so they do not multiply pipelines; the depth @@ -512,8 +518,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput pipelineInfo.stageCount = static_cast(stages.size()); pipelineInfo.pStages = stages.data(); VkPipelineTessellationStateCreateInfo tessellation{VK_STRUCTURE_TYPE_PIPELINE_TESSELLATION_STATE_CREATE_INFO}; - if (state.rectList || state.stages.tessellation) { - tessellation.patchControlPoints = state.rectList ? 3u : state.stages.tessellation->inputControlPoints; + if (state.rectList || state.stages.tessellation || barycentricPatches) { + tessellation.patchControlPoints = state.stages.tessellation ? state.stages.tessellation->inputControlPoints : 3u; pipelineInfo.pTessellationState = &tessellation; } pipelineInfo.pVertexInputState = state.stages.mesh ? nullptr : &input; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index 34bd3349af..2880094519 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -442,6 +442,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto it = module.modes.find(name); Require(it != module.modes.end() && it->second == operands, "missing or incompatible shader execution mode"); }; + const bool barycentricPatches = (control || evaluation) && !state.rectList && !state.stages.tessellation && state.stages.path == ShaderPath::Vertex; if (mesh) { Require(state.stages.mesh.has_value(), "mesh configuration is missing"); const auto& config = *state.stages.mesh; @@ -452,6 +453,12 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi mode(spv::ExecutionModeOutputPrimitivesEXT, {config.maxPrimitives}); mode(config.outputPrimitiveSize == 2u ? spv::ExecutionModeOutputLinesEXT : spv::ExecutionModeOutputTrianglesEXT, {}); Require(module.modes.size() == 4, "unsupported mesh execution mode"); + } else if (barycentricPatches) { + if (control) mode(spv::ExecutionModeOutputVertices, {3u}); + mode(spv::ExecutionModeTriangles, {}); + mode(spv::ExecutionModeSpacingEqual, {}); + mode(spv::ExecutionModeVertexOrderCw, {}); + Require(module.modes.size() == (control ? 4u : 3u), "unsupported generated tessellation execution mode"); } else if (control) { Require(state.rectList || state.stages.tessellation.has_value(), "tessellation configuration is missing"); mode(spv::ExecutionModeOutputVertices, {state.rectList ? 4u : state.stages.tessellation->outputControlPoints}); @@ -504,7 +511,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi } else if (geometry) { Require(length->second == 3u, "geometry interface array must hold the three vertices of a triangle"); } else { - const auto expected = state.rectList ? (control && input ? 3u : 4u) : control && input ? state.stages.tessellation->inputControlPoints : state.stages.tessellation->outputControlPoints; + const auto expected = barycentricPatches ? 3u : state.rectList ? (control && input ? 3u : 4u) : control && input ? state.stages.tessellation->inputControlPoints : state.stages.tessellation->outputControlPoints; Require(length->second == expected, "tessellation interface array disagrees with control-point count"); } typeId = outer[2]; @@ -605,7 +612,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(shader.vertexAttributes.empty(), "vertex attribute metadata is invalid for this stage"); } Require(descriptors.size() == shader.bindings.size(), "recompiler binding metadata contains undeclared resources"); - Require(push == (!shader.pushConstants.empty() || mesh || (control && !rectListControl)), "recompiler push constant metadata disagrees with SPIR-V"); + Require(push == (!shader.pushConstants.empty() || mesh || (control && !rectListControl && !barycentricPatches)), "recompiler push constant metadata disagrees with SPIR-V"); for (const auto id : module.interface) Require(module.variables.contains(id), "entry point interface contains an unknown variable"); if (mesh) Require(module.primitiveIndices == state.stages.mesh->outputPrimitiveSize, "mesh shader does not export primitive indices"); if (stage == Stage::Vertex || mesh || evaluation || geometry) Require(module.position, "vertex shader does not export position"); @@ -636,7 +643,9 @@ std::set ValidateShaders(std::span shaders, Require(state.stages.mesh.has_value() == mesh && state.stages.tessellation.has_value() == tessellation, "graphics stage configuration disagrees with its path"); Require(!state.rectList || (state.stages.path == ShaderPath::Vertex && state.topology == VK_PRIMITIVE_TOPOLOGY_PATCH_LIST && state.cullMode == VK_CULL_MODE_NONE), "invalid rect-list pipeline state"); const bool geometry = state.stages.path == ShaderPath::Vertex && !state.rectList && shaders.size() == 3u; - Require(geometry || shaders.size() == (tessellation || state.rectList ? 4u : 2u), "incorrect graphics stage count"); + const bool patches = BarycentricPatches(state, shaders) && shaders.back().program != nullptr && shaders.back().program->barycentricEmulation.active; + Require(geometry || shaders.size() == (tessellation || state.rectList || patches ? 4u : 2u), "incorrect graphics stage count"); + if (patches) Require(state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, "emulated barycentrics through tessellation require a triangle list"); if (geometry) { Require(shaders.back().program != nullptr && shaders.back().program->barycentricEmulation.active, "a geometry stage is generated only for emulated barycentrics"); Require(state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, "emulated barycentrics require triangle primitives"); @@ -645,7 +654,7 @@ std::set ValidateShaders(std::span shaders, static_cast(AssemblePushConstants(shaders)); Module previous; for (std::size_t i = 0; i < shaders.size(); ++i) { - const auto expected = state.rectList ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : geometry && i == 1 ? Stage::Geometry : i == shaders.size() - 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex; + const auto expected = state.rectList || patches ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : geometry && i == 1 ? Stage::Geometry : i == shaders.size() - 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex; Require(shaders[i].program != nullptr, "missing compiled shader"); Require(shaders[i].stage == expected, "graphics stage order disagrees"); for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shaders[i].stage == ShaderRecompiler::ShaderStage::Fragment && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "graphics resource uses a descriptor set other than its stage's"); From 48b7d3a34d5ec88bd8f7122c8a0ee0d0fb8138af Mon Sep 17 00:00:00 2001 From: David Date: Sun, 18 Oct 2026 02:16:51 +0400 Subject: [PATCH 058/103] test(agc): cover the tessellation route of the barycentric emulation --- .../prx/libSceAgcDriver/tests/Graphics.cpp | 60 +++++++++++++++ .../execution/PixelBarycentricEmulation.cpp | 75 ++++++++++++++----- 2 files changed, 115 insertions(+), 20 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index 95c09675a9..16d7bf35e8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -5550,7 +5550,67 @@ void barycentricEmulationTests() { expectFailure([&] { static_cast(BuildBarycentricGeometryShader(vertex, plain, target, limits)); }, "does not emulate barycentrics"); expectFailure([&] { static_cast(BuildBarycentricGeometryShader(vertex, fragment, target, std::nullopt)); }, "geometry shaders are unavailable"); expectFailure([&] { static_cast(BuildBarycentricGeometryShader(vertex, fragment, target, GeometryStageLimits{64u, 12u, 256u, 1024u, 128u})); }, "exceeds device limits"); + + const std::array tessellationCapabilities{spv::CapabilityShader, spv::CapabilityTessellation}; + SpirvTarget tessellationTarget = target; + tessellationTarget.supportedCapabilities = tessellationCapabilities; + tessellationTarget.tessellation = TessellationTargetLimits{32u, 128u, 128u, 120u, 4096u, 128u, 128u}; + const auto modes = [](const RecompileResult& result) { + std::vector values; + const auto& words = result.spirv.Words(); + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + if (static_cast(words[at] & 0xffffu) == spv::OpExecutionMode) values.push_back(words[at + 2]); + } + return values; + }; + auto patches = BuildBarycentricTessellationShaders(vertex, fragment, tessellationTarget, 128u); + Require(modes(patches.control) == std::vector{spv::ExecutionModeOutputVertices, spv::ExecutionModeTriangles, spv::ExecutionModeSpacingEqual, spv::ExecutionModeVertexOrderCw}, "the generated control stage does not declare three output vertices and the triangle domain"); + Require(modes(patches.evaluation) == std::vector{spv::ExecutionModeTriangles, spv::ExecutionModeSpacingEqual, spv::ExecutionModeVertexOrderCw}, "the generated evaluation stage does not declare the triangle domain"); + const std::array patchShaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::TessellationControl, &patches.control, 0}, {ShaderStage::TessellationEvaluation, &patches.evaluation, 0}, {ShaderStage::Fragment, &fragment, 0}}}; + Require(BarycentricPatches(state, patchShaders) && !BarycentricPatches(state, shaders), "generated tessellation stages were not recognised as barycentric patches"); + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + ValidateShaders(patchShaders, state, subgroup, false); + for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, VK_PRIMITIVE_TOPOLOGY_POINT_LIST}) { + state.topology = topology; + expectFailure([&] { ValidateShaders(patchShaders, state, subgroup, false); }, "require a triangle list"); + } + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + const std::array plainPatchShaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::TessellationControl, &patches.control, 0}, {ShaderStage::TessellationEvaluation, &patches.evaluation, 0}, {ShaderStage::Fragment, &plain, 0}}}; + expectFailure([&] { ValidateShaders(plainPatchShaders, state, subgroup, false); }, "stage count"); + expectFailure([&] { static_cast(BuildBarycentricTessellationShaders(vertex, plain, tessellationTarget, 128u)); }, "does not emulate barycentrics"); + auto noTessellation = tessellationTarget; + noTessellation.tessellation.reset(); + expectFailure([&] { static_cast(BuildBarycentricTessellationShaders(vertex, fragment, noTessellation, 128u)); }, "tessellation shaders are unavailable"); + expectFailure([&] { static_cast(BuildBarycentricTessellationShaders(vertex, fragment, target, 128u)); }, "tessellation shaders are unavailable"); + auto noCapability = tessellationTarget; + noCapability.supportedCapabilities = std::span(capabilities).first(1); + expectFailure([&] { static_cast(BuildBarycentricTessellationShaders(vertex, fragment, noCapability, 128u)); }, "tessellation capability is unavailable"); + auto smallPatches = tessellationTarget; + smallPatches.tessellation->maxEvaluationOutputComponents = 4u; + expectFailure([&] { static_cast(BuildBarycentricTessellationShaders(vertex, fragment, smallPatches, 128u)); }, "exceeds device limits"); + expectFailure([&] { static_cast(BuildBarycentricTessellationShaders(vertex, fragment, tessellationTarget, 0u)); }, "exceeds device limits"); + + RecompileResult clipped; + clipped.spirv = makeModule({.parameterOutput = true, .clipDistanceLength = 2}); + clipped.parameterExports = {0u}; + expectFailure([&] { static_cast(BuildBarycentricTessellationShaders(clipped, fragment, tessellationTarget, 128u)); }, "need shaderClipDistance"); + const std::array clipCapabilities{spv::CapabilityShader, spv::CapabilityTessellation, spv::CapabilityClipDistance}; + auto clipTarget = tessellationTarget; + clipTarget.supportedCapabilities = clipCapabilities; + auto clippedPatches = BuildBarycentricTessellationShaders(clipped, fragment, clipTarget, 128u); + const auto memberBuiltIn = [](const RecompileResult& result, spv::BuiltIn builtin) { + const auto& words = result.spirv.Words(); + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + if (static_cast(words[at] & 0xffffu) == spv::OpMemberDecorate && words[at + 3] == spv::DecorationBuiltIn && words[at + 4] == builtin) return true; + } + return false; + }; + Require(!memberBuiltIn(patches.control, spv::BuiltInClipDistance) && !memberBuiltIn(patches.evaluation, spv::BuiltInClipDistance), "a generated tessellation stage declares clip distances the vertex shader does not write"); + Require(memberBuiltIn(clippedPatches.control, spv::BuiltInClipDistance) && memberBuiltIn(clippedPatches.evaluation, spv::BuiltInClipDistance), "a generated tessellation stage does not forward the vertex clip distances"); + const std::array clippedShaders{{{ShaderStage::Vertex, &clipped, 0}, {ShaderStage::TessellationControl, &clippedPatches.control, 0}, {ShaderStage::TessellationEvaluation, &clippedPatches.evaluation, 0}, {ShaderStage::Fragment, &fragment, 0}}}; + ValidateShaders(clippedShaders, state, subgroup, false, false, false, false, false, false, true); vertex.parameterExports.clear(); + expectFailure([&] { static_cast(BuildBarycentricTessellationShaders(vertex, fragment, tessellationTarget, 128u)); }, "not exported by the vertex shader"); expectFailure([&] { static_cast(BuildBarycentricGeometryShader(vertex, fragment, target, limits)); }, "not exported by the vertex shader"); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp index 3a9d3178ef..9978674554 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp @@ -58,7 +58,12 @@ std::array BufferDescriptor(const void* data, std::uint32_t st return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; } -ShaderRecompiler::RecompileResult Draw(AgcDriver::VulkanDevice& device, std::span pixelCode, std::uint32_t inputs) { +enum class Emulation { + Geometry, + Tessellation, +}; + +ShaderRecompiler::RecompileResult Draw(AgcDriver::VulkanDevice& device, Emulation emulation, std::span pixelCode, std::uint32_t inputs, VkCullModeFlags cullMode = VK_CULL_MODE_NONE) { Pixels.fill(Unwritten); auto target = device.Target(); target.fragmentShaderBarycentricEnabled = false; @@ -98,9 +103,14 @@ ShaderRecompiler::RecompileResult Draw(AgcDriver::VulkanDevice& device, std::spa auto pixelResult = ShaderRecompiler::Recompile(fragment); std::vector shaders{{ShaderStage::Vertex, &vertexResult, 0}}; ShaderRecompiler::RecompileResult geometry; - if (pixelResult.barycentricEmulation.active) { + ShaderRecompiler::BarycentricTessellationShaders patches; + if (pixelResult.barycentricEmulation.active && emulation == Emulation::Geometry) { geometry = ShaderRecompiler::BuildBarycentricGeometryShader(vertexResult, pixelResult, target, device.GeometryLimits()); shaders.push_back({ShaderStage::Geometry, &geometry, 0}); + } else if (pixelResult.barycentricEmulation.active) { + patches = ShaderRecompiler::BuildBarycentricTessellationShaders(vertexResult, pixelResult, target, device.MaxFragmentInputComponents()); + shaders.push_back({ShaderStage::TessellationControl, &patches.control, 0}); + shaders.push_back({ShaderStage::TessellationEvaluation, &patches.evaluation, 0}); } shaders.push_back({ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}); @@ -114,7 +124,7 @@ ShaderRecompiler::RecompileResult Draw(AgcDriver::VulkanDevice& device, std::spa state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; state.negativeOneToOne = false; state.scissor = {{0, 0}, {Width, Height}}; - state.cullMode = VK_CULL_MODE_NONE; + state.cullMode = cullMode; state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; state.blend.colorWriteMask = 15; state.blends = {state.blend}; @@ -232,31 +242,56 @@ void CheckInterpolated() { Require(checked > 200u, "v_interp_p1/p2: too few interior pixels were checked"); } +bool Drawn() { + return std::any_of(Pixels.begin(), Pixels.end(), [](float value) { return value != Unwritten; }); +} + +void CheckWinding(AgcDriver::VulkanDevice& device, Emulation emulation, const std::string& name) { + const auto perspective = PixelInputBit(PixelInput::PerspectiveCenter); + for (const auto cullMode : {VK_CULL_MODE_BACK_BIT, VK_CULL_MODE_FRONT_BIT}) { + Draw(device, emulation, InterpolatedPixelCode, perspective, cullMode); + const bool host = Drawn(); + Draw(device, emulation, BarycentricPixelCode, perspective, cullMode); + Require(Drawn() == host, name + ": culling the " + (cullMode == VK_CULL_MODE_BACK_BIT ? "back" : "front") + " face does not match the draw without emulation"); + } +} + +void RunEmulation(AgcDriver::VulkanDevice& device, Emulation emulation) { + const std::string name = emulation == Emulation::Geometry ? "geometry stage" : "tessellation stages"; + const auto perspective = PixelInputBit(PixelInput::PerspectiveCenter); + const auto linear = PixelInputBit(PixelInput::LinearCenter); + auto result = Draw(device, emulation, BarycentricPixelCode, perspective); + Require(result.barycentricEmulation.active && result.barycentricEmulation.smooth && !result.barycentricEmulation.linear, "reading PERSP_CENTER I/J did not emulate perspective barycentrics"); + const auto smooth = CheckBarycentrics(name + " PERSP_CENTER I/J", true); + result = Draw(device, emulation, BarycentricPixelCode, linear); + Require(result.barycentricEmulation.active && !result.barycentricEmulation.smooth && result.barycentricEmulation.linear, "reading LINEAR_CENTER I/J did not emulate linear barycentrics"); + const auto noPerspective = CheckBarycentrics(name + " LINEAR_CENTER I/J", false); + result = Draw(device, emulation, VertexValuePixelCode, perspective); + Require(result.barycentricEmulation.active && result.fragmentParameters.size() == 1 && result.fragmentParameters[0].perVertex, "v_interp_mov of P10 and P20 did not emulate per-vertex parameters"); + CheckVertexValues(); + result = Draw(device, emulation, InterpolatedPixelCode, perspective); + Require(!result.barycentricEmulation.active, "plain v_interp_p1/p2 needed a generated stage"); + CheckInterpolated(); + CheckWinding(device, emulation, name); + std::printf("barycentric emulation through the %s passed: %u perspective and %u linear I/J pixels, raw vertex values, host interpolation and face culling\n", name.c_str(), smooth, noPerspective); +} + } int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - if (!device->GeometryLimits()) { - std::puts("skipped, the device lacks geometryShader"); + const bool geometry = device->GeometryLimits().has_value(); + const bool tessellation = device->Target().tessellation.has_value(); + if (!geometry && !tessellation) { + std::puts("skipped, the device lacks geometryShader and tessellationShader"); return VulkanTestSkipped; } - const auto perspective = PixelInputBit(PixelInput::PerspectiveCenter); - const auto linear = PixelInputBit(PixelInput::LinearCenter); - auto result = Draw(*device, BarycentricPixelCode, perspective); - Require(result.barycentricEmulation.active && result.barycentricEmulation.smooth && !result.barycentricEmulation.linear, "reading PERSP_CENTER I/J did not emulate perspective barycentrics"); - const auto smooth = CheckBarycentrics("PERSP_CENTER I/J", true); - result = Draw(*device, BarycentricPixelCode, linear); - Require(result.barycentricEmulation.active && !result.barycentricEmulation.smooth && result.barycentricEmulation.linear, "reading LINEAR_CENTER I/J did not emulate linear barycentrics"); - const auto noPerspective = CheckBarycentrics("LINEAR_CENTER I/J", false); - result = Draw(*device, VertexValuePixelCode, perspective); - Require(result.barycentricEmulation.active && result.fragmentParameters.size() == 1 && result.fragmentParameters[0].perVertex, "v_interp_mov of P10 and P20 did not emulate per-vertex parameters"); - CheckVertexValues(); - result = Draw(*device, InterpolatedPixelCode, perspective); - Require(!result.barycentricEmulation.active, "plain v_interp_p1/p2 needed a geometry stage"); - CheckInterpolated(); - std::printf("barycentric emulation tests passed: %u perspective and %u linear I/J pixels, raw vertex values and host interpolation\n", smooth, noPerspective); + if (geometry) RunEmulation(*device, Emulation::Geometry); + else std::puts("geometry stage skipped, the device lacks geometryShader"); + if (tessellation) RunEmulation(*device, Emulation::Tessellation); + else std::puts("tessellation stages skipped, the device lacks tessellationShader"); return 0; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; From 91d5f577ea9f59fd57834a3b0e99e975c686ce22 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 18 Oct 2026 02:17:01 +0400 Subject: [PATCH 059/103] docs(shader): record the tessellation route of the barycentric emulation --- docs/dev/debt/shader.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 5a9518896e..6fc14e165a 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -84,7 +84,7 @@ - [Mesh shader grid size](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - every mesh shader loads `NumWorkgroups.x` and declares no vertices when it is 0, which no dispatch produces. The read is a workaround for RADV (Mesa 26.2.4, seen on an RX 9070 XT, GFX12): binding a mesh pipeline whose shader reads neither `NumWorkgroups` nor `DrawIndex` keeps the user SGPR count and draw-id flag of the vertex pipeline bound before it in the same command buffer (`radv_bind_pre_rast_shader` updates them only when the shader has the `AC_UD_VS_BASE_VERTEX_START_INSTANCE` slot), so `radv_emit_userdata_mesh` opens a 3-register `SET_SH_REG` and writes one value, the CP takes the following `DISPATCH_MESH_DIRECT` as register data and the GPU hangs (gfx ring timeout, MODE1 reset). Kena: Bridge of Spirits (PPSA01802) hits it on its layered volume draws. Reading `NumWorkgroups` gives every mesh shader the grid-size slot, so the count and the values match; drop the read once a fixed Mesa is the minimum - [Shader recompilation](../../../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../../../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../../../core/shader/recompiler) was written completely independently from [libs/prx](../../../core/libs/prx). - [v_interp_p1ll_f16, v_interp_p1lv_f16 and v_interp_p2_f16](../../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp) read the 16-bit halves of each vertex's parameter through `VK_KHR_fragment_shader_barycentric` and throw without it, as fixed-function interpolation cannot split a 32-bit attribute. P10 and P20 are the vertex differences rounded to f16 to nearest even, which is not confirmed on hardware. v_interp_p1ll_f16 and v_interp_p1lv_f16 round the exact `P10 * I + P0` once to f32 and v_interp_p2_f16 rounds the exact `P20 * J + S2` once to f16, as measured on RDNA2 (#697); the exact sums are computed in f64, so these instructions need the Float64 capability. They follow the shader's float mode, measured on RDNA2 (gfx1035) in all eight combinations of `IEEE_MODE` and kept/flushed f32 and f16 denormals over 768 special and 1,024 random rows. With IEEE=1 a NaN operand is returned quieted; with IEEE=0 it is returned unquieted, except that v_interp_p2_f16's f16 result is always quiet. Flushed f32 denormals flush I/J, v_interp_p2_f16's f32 S2 and the f32 result; flushed f16 denormals flush P0, P10/P20, v_interp_p1lv_f16's f16 S2 and the f16 result. Denormal modes 1 and 2 (flush only inputs or only outputs) and rounding other than to nearest even are not measured and throw. An unknown mode follows IEEE=0 with denormals kept. Lavapipe has no barycentrics, so the draw test is skipped in CI; `agc_driver_interpolation_modes` checks the arithmetic in every mode against the hardware rows on any device with Float64. -- Without `VK_KHR_fragment_shader_barycentric`, a pixel shader whose I/J VGPRs [only feed `v_interp_p1_f32`/`v_interp_p2_f32` of their own pair](../../../core/shader/recompiler/Optimization/src/HostInterpolationChecker.cpp) reads the attribute Vulkan interpolated. Any other read of them or of a raw vertex parameter takes the weights and the three vertex values from a [generated geometry stage](../../../core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp), which needs `geometryShader` and a vertex shader before it: tessellation, mesh and rect-list pipelines, primitives other than triangles, more than 32 fragment input locations and interfaces beyond the device's geometry limits throw. +- Without `VK_KHR_fragment_shader_barycentric`, a pixel shader whose I/J VGPRs [only feed `v_interp_p1_f32`/`v_interp_p2_f32` of their own pair](../../../core/shader/recompiler/Optimization/src/HostInterpolationChecker.cpp) reads the attribute Vulkan interpolated. Any other read of them or of a raw vertex parameter takes the weights and the three vertex values from a [generated geometry stage](../../../core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp), which needs `geometryShader` and a vertex shader before it. On a device with `tessellationShader` but no `geometryShader` (MoltenVK), a triangle list without primitive restart takes them from [generated tessellation stages](../../../core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp) instead: each triangle is one three-point patch at tessellation level 1, the evaluation stage writes the corner `gl_TessCoord` points at as the weights, and the vertex clip distances are forwarded. A flat parameter takes the first vertex's value there, as the geometry stage's strip gives it; the last provoking vertex throws. The patch keeps the triangle's facing with `VertexOrderCw`, checked with face culling on MoltenVK 1.4.2 (Apple M2) only. Tessellation, mesh and rect-list pipelines, primitives other than triangles, triangle strips and fans without `geometryShader`, more than 32 fragment input locations and interfaces beyond the device's geometry or tessellation limits throw. - [v_fma_f64 and v_div_fmas_f64](../../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp) with a subnormal operand or result round once, as IEEE 754 fusedMultiplyAdd does with f64 denormals kept, measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1, `v_div_fmas_f64` with VCC clear and set. The f16/f64 denormal field of `FLOAT_MODE` (bits 7:6) is not read: when it flushes, the hardware flushes the operands and the result of `v_fma_f64` (tininess before rounding) and only the result of `v_div_fmas_f64`, while these keep them. Shaders with f64 instructions declare `DenormPreserve` for 64-bit floats only when `shaderDenormPreserveFloat64` is true and Vulkan reports `denormBehaviorIndependence` as `ALL`; other independence modes are not handled by the current backend because they require coupled execution modes. See [VkShaderFloatControlsIndependence](https://docs.vulkan.org/refpages/latest/refpages/source/VkShaderFloatControlsIndependence.html). Elsewhere (including lavapipe), the driver may flush f64 subnormals, and `agc_driver_f64_fma_subnormals` is skipped. - [SINT storage images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8- and 16-bit components saturate each stored 32-bit value to the component range, as adriadam10 measured for image stores on RDNA2 (gfx1035) in #944 (448 rows match); the PS5's gfx1013 is not measured. A D16 store sign-extends each 16-bit half before that saturation, as measured on RDNA2 (gfx1035) for `image_store d16` to all eight SINT formats with DMASK 0xf, 0x7, 0x5, 0x3, 0x2, 0xa and 0x1 (1792 rows match). - [`StoreTexel`](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) writes channel 1 of a two-channel format from the data slot whose DST_SEL is Y, for SINT and UINT alike. With DST_SEL X,Y,Z,W, gfx1035 writes it from the W slot, as adriadam10 measured in #944. @@ -143,7 +143,7 @@ - [FORCE_DEGAMMA on RGBA8_UINT](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) materializes a direct image sampled only by plain biased pixel accesses as RGBA8_SRGB when every paired sampler forces degamma. Measured on AMD integrated RDNA2 gfx1036 (driver 32.0.21036.18, Windows 11, wave32, IEEE off, DX10 clamp on, denormals preserved, round to nearest even, FP16 overflow off): a native `image_sample_b` DIM 2D on a 16x16x1 RGBA8_UINT 3D T# returns integer channels without the bit, but the same floating-point decoded values as RGBA8_UNORM with it, including point min/max reduction and the title's exact sampler. The metadata and image view must both become floating-point sRGB; ignoring the sampler bit or retaining a UINT heap is incorrect. The accepted case requires one slice, mip zero, normalized point sampling and uniform FORCE_DEGAMMA across its samplers. Other image accesses, mixed sampler states, indirect descriptors and other integer formats remain refused. Wave64, other filtering modes and the PS5's gfx1013 are not measured. - The [s_swappc_b64 fetch-shader call at vertex/local/tessellation/mesh entry](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) assumes the called fetch-shader blob ends in `s_setpc_b64` to the link pair (the GNM fetch-shader shape shadPS4's `EmitFetch` copies); the blob is not part of the shader program, so its return closes the pairing by assumption and any in-program reader of the link pair throws. Attribute fetches are inlined from the request's parsed fetch-shader plan through the same `GetAttribute` path as the embedded fetch analyzer, honoring the buffer resource `dst_sel`; an attribute writing more than four registers, or an entry call without a parsed plan, throws. The vertex/instance offset accumulator inside the fetch blob is not replayed for external blobs. - The scalar aperture sources [src_shared_base, src_shared_limit, src_private_base, src_private_limit](../../../core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp) (core/shader/recompiler, scalar source codes 235 to 238) read as the 64-bit apertures the flat routing uses: a base is the aperture's high dword (`SharedApertureHigh`, `PrivateApertureHigh`) over a low dword of 0, a limit is the same high dword over 0xffffffff. A 64-bit read (`s_mov_b64`, a 64-bit VALU source, a wave64 lane mask) gets the whole value and a 32-bit read gets the low dword. This follows LLVM's `SIRegisterInfo.td` (aperture registers are 64-bit, their low 32 bits are always 0 for a base and -1 for a limit) and `getSegmentAperture` (`AMDGPULegalizerInfo.cpp`, `SIISelLowering.cpp`: a 64-bit read whose high half is the aperture), the KFD's `MAKE_LDS_APP_LIMIT` (limit = base | 0xffffffff) and the gfx1030 and gfx1035 readouts on [#1190](https://github.com/boykopovar/AnyPS5/pull/1190) and [#3803](https://github.com/boykopovar/AnyPS5/issues/3803). Not measured on a console: the PS5's high dwords are the inferred values of the flat aperture bases entry, and the wave is taken to be in 64-bit address mode (PTR32 clear) -- [EmitSetAttribute](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - a primitive with a vertex whose clip position is (0,0,0,0) is clipped away through an extra `gl_ClipDistance` element on vertex and tessellation evaluation shaders, matching what a Radeon 610M (RDNA2, RADV) and llvmpipe rasterise; NVIDIA 615.78.08 without it draws the vertex at the viewport centre. Measured for triangles (strip and list) and for a point at (0,0,0,0), which the Radeon 610M does not draw, NVIDIA draws at the viewport centre and llvmpipe at the top-left corner; lines with such a vertex are clipped by the same element and were not measured. Mesh, geometry and the generated rect-list and barycentric stages do not write it, a device without `shaderClipDistance` cannot compile vertex shaders (the compile throws), and guest clip distances are still refused by the pipeline validator. The PS5 rasteriser was not measured. +- [EmitSetAttribute](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - a primitive with a vertex whose clip position is (0,0,0,0) is clipped away through an extra `gl_ClipDistance` element on vertex and tessellation evaluation shaders, matching what a Radeon 610M (RDNA2, RADV) and llvmpipe rasterise; NVIDIA 615.78.08 without it draws the vertex at the viewport centre. Measured for triangles (strip and list) and for a point at (0,0,0,0), which the Radeon 610M does not draw, NVIDIA draws at the viewport centre and llvmpipe at the top-left corner; lines with such a vertex are clipped by the same element and were not measured. Mesh, geometry and the generated barycentric geometry stage do not write it, the generated rect-list and barycentric tessellation stages forward the vertex shader's array, a device without `shaderClipDistance` cannot compile vertex shaders (the compile throws), and guest clip distances are still refused by the pipeline validator. The PS5 rasteriser was not measured. - [PERSP_SAMPLE and LINEAR_SAMPLE](../../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) hold the pixel-centre barycentrics, like the centroid pairs. With one sample this is the sample position ([Vulkan standard sample locations](https://docs.vulkan.org/spec/latest/chapters/primsrast.html#primsrast-multisampling): (0.5, 0.5)); with more samples the console's per-sample weights are not computed and were not measured. - [LINE_STIPPLE_TEX](../../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) reads as 0: line stipple is not emulated. RADV enables the input only as the required pixel input of a shader that reads no interpolants, because it initializes the fewest VGPRs ([Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/vulkan/radv_shader.c) `radv_compute_spi_ps_input`), and never reads it - [Static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require an ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and a later `s_setpc_b64` return reading that exact pair. A pair serves one call, or several calls whose regions are disjoint, each closed by the first return reading the pair after its target. Outside every region of its calls the pair is free to use wherever no return address can reach: a dataflow over the decoded instructions follows each dword of the pair from the calls that write it until a scalar load, SALU or VALU result is known to overwrite it (a VALU mask counts as one SGPR unless the request says wave64), and an instruction there that may read a dword still holding a return address throws; SOPC compares are exempt, as they only set SCC. Regions must nest or be disjoint; recursion, other shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. From b2d9880700fef64c1d5df394f6afa80d8b8ca558 Mon Sep 17 00:00:00 2001 From: Romans Pokrovskis Date: Sun, 18 Oct 2026 00:08:31 +0300 Subject: [PATCH 060/103] feat(agc): support thick 4 KiB standard XOR volumes --- .../include/TextureSwizzleEquations.hpp | 5 ++++ .../Graphics/src/TextureDetiler.cpp | 4 ++-- .../Graphics/src/TextureTiling.cpp | 7 +++--- .../libSceAgcDriver/tests/TextureTiling.cpp | 20 +++++++++++++++- .../tests/execution/ImageXorSwizzleVolume.cpp | 5 +++- .../tests/execution/LayeredVolumeDraw.cpp | 24 ++++++++++++------- docs/dev/debt/libSceAgcDriver.md | 1 + 7 files changed, 51 insertions(+), 15 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp index b77267d50a..d408819b49 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp @@ -84,6 +84,11 @@ inline constexpr TextureSwizzleEquation kTextureSwizzleEquations[] = { {0x109u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x00004000u, 0x00000004u, 0x04000000u, 0x00008000u, 0x00000008u, 0x08000000u, 0x00010000u, 0x00000010u}}, {0x109u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000010u}}, {0x109u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000008u}}, + {0x115u, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x40004040u, 0x20020008u, 0x08010020u, 0x10008010u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x115u, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x40004020u, 0x20020004u, 0x08010010u, 0x10008008u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x115u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x20004020u, 0x10020004u, 0x04010010u, 0x08008008u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x115u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x20002020u, 0x10010004u, 0x04008010u, 0x08004008u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x115u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x20002010u, 0x10010002u, 0x04008008u, 0x08004004u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, {0x119u, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x40004040u, 0x20020008u, 0x08010020u, 0x10008010u, 0x00000010u, 0x10000000u, 0x00010000u, 0x00000020u}}, {0x119u, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x40004020u, 0x20020004u, 0x08010010u, 0x10008008u, 0x00000008u, 0x10000000u, 0x00010000u, 0x00000010u}}, {0x119u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x20004020u, 0x10020004u, 0x04010010u, 0x08008008u, 0x00000008u, 0x08000000u, 0x00010000u, 0x00000010u}}, diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index 9397681655..62bd3365ef 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -144,8 +144,8 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem // per-bit XOR equation for the equation family (2); 20-21 override the block extent in elements. std::array values{elementBytes, BlockBytesFor(tileMode), tileMode == TextureTileMode::kLinear ? 0u : 1u, retile ? 1u : 0u}; if (thick) { - const auto thickMode = tileMode == TextureTileMode::kStandard4KB ? 0x105u : tileMode == TextureTileMode::kStandard64KB ? 0x109u : tileMode == TextureTileMode::kS64KBX ? 0x119u : tileMode == TextureTileMode::kD64KBX ? 0x11au : 0u; - Require(thickMode != 0, "3D textures are only detiled thick from SW_4KB_S, SW_64KB_S, SW_64KB_S_X or SW_64KB_D_X"); + const auto thickMode = tileMode == TextureTileMode::kStandard4KB ? 0x105u : tileMode == TextureTileMode::kS4KBX ? 0x115u : tileMode == TextureTileMode::kStandard64KB ? 0x109u : tileMode == TextureTileMode::kS64KBX ? 0x119u : tileMode == TextureTileMode::kD64KBX ? 0x11au : 0u; + Require(thickMode != 0, "3D textures are only detiled thick from SW_4KB_S, SW_4KB_S_X, SW_64KB_S, SW_64KB_S_X or SW_64KB_D_X"); const auto* equation = FindTextureSwizzleEquation(thickMode, elementBytes); Require(equation != nullptr, "no thick swizzle equation for the element size"); values[2] = 2u; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp index 6eca2ae659..a2c51f1e0c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp @@ -262,11 +262,12 @@ std::array ThickBlockExtent(TextureTileMode tileMode, std::uin Require(std::has_single_bit(bytesPerElement) && bytesPerElement <= 16u, "unsupported bytes per element for thick texture geometry"); const auto index = static_cast(std::countr_zero(bytesPerElement)); switch (tileMode) { - case TextureTileMode::kStandard4KB: return {1u << thick4KB[index][0], 1u << thick4KB[index][1], 1u << thick4KB[index][2]}; + case TextureTileMode::kStandard4KB: + case TextureTileMode::kS4KBX: return {1u << thick4KB[index][0], 1u << thick4KB[index][1], 1u << thick4KB[index][2]}; case TextureTileMode::kStandard64KB: case TextureTileMode::kS64KBX: case TextureTileMode::kD64KBX: return {1u << thick64KB[index][0], 1u << thick64KB[index][1], 1u << thick64KB[index][2]}; - default: throw std::runtime_error("AGC graphics: thick 3D textures are only supported in SW_4KB_S, SW_64KB_S, SW_64KB_S_X and SW_64KB_D_X, not in swizzle mode " + std::to_string(EquationSwizzleMode(tileMode)) + " at " + std::to_string(bytesPerElement) + " bytes per element"); + default: throw std::runtime_error("AGC graphics: thick 3D textures are only supported in SW_4KB_S, SW_4KB_S_X, SW_64KB_S, SW_64KB_S_X and SW_64KB_D_X, not in swizzle mode " + std::to_string(EquationSwizzleMode(tileMode)) + " at " + std::to_string(bytesPerElement) + " bytes per element"); } } @@ -316,7 +317,7 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s auto firstTailLevel = mipCount; if (tileMode != TextureTileMode::kLinear) { block = ThickBlockExtent(tileMode, bytesPerElement); - blockBytes = tileMode == TextureTileMode::kStandard4KB ? 4096u : 65536u; + blockBytes = tileMode == TextureTileMode::kStandard4KB || tileMode == TextureTileMode::kS4KBX ? 4096u : 65536u; const auto log2BlockBytes = static_cast(std::countr_zero(blockBytes)); const auto tailWidth = log2BlockBytes % 3u == 1u ? block[0] >> 1u : block[0]; const auto tailHeight = log2BlockBytes % 3u == 0u ? block[1] >> 1u : block[1]; diff --git a/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp index c72b4f3b02..6c6acca9d2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp @@ -32,6 +32,7 @@ struct ElementAddress { std::uint32_t y; std::uint32_t z; std::uint64_t address; + std::uint32_t pipeBankXor = 0; }; std::uint64_t equationOffset(const TextureSwizzleEquation& equation, std::uint32_t x, std::uint32_t y, std::uint32_t z) { @@ -60,7 +61,7 @@ void requireThickAddresses(const GuestTextureResource& resource, std::uint32_t b const auto geometry = DescribeSurface(resource); Require(geometry.thick && geometry.mips.size() == resource.mipCount && geometry.guestBytes == guestBytes, what + ": guest size differs from addrlib"); for (std::uint32_t level = 0; level < resource.mipCount; ++level) Require(geometry.mips[level].tail == (level >= firstTailLevel), what + ": mip " + std::to_string(level) + " is on the wrong side of the mip tail"); - const auto blockBytes = resource.tileMode == TextureTileMode::kStandard4KB ? 4096u : 65536u; + const auto blockBytes = resource.tileMode == TextureTileMode::kStandard4KB || resource.tileMode == TextureTileMode::kS4KBX ? 4096u : 65536u; const auto* equation = FindTextureSwizzleEquation(0x100u | (resource.tileMode == TextureTileMode::kStandard4KB ? 5u : resource.tileMode == TextureTileMode::kStandard64KB ? 9u : XorSwizzleMode(resource.tileMode)), bytesPerElement); Require(equation != nullptr, what + ": missing thick swizzle equation"); const auto block = ThickBlockExtent(resource.tileMode, bytesPerElement); @@ -73,6 +74,7 @@ void requireThickAddresses(const GuestTextureResource& resource, std::uint32_t b } else { address += (static_cast(element.y / block[1]) * mip.blocksPerRow + element.x / block[0]) * blockBytes + equationOffset(*equation, element.x, element.y, element.z); } + address ^= element.pipeBankXor; Require(address == element.address, what + ": mip " + std::to_string(element.mip) + " element (" + std::to_string(element.x) + ", " + std::to_string(element.y) + ", " + std::to_string(element.z) + ") detiles from " + std::to_string(address) + " instead of addrlib's " + std::to_string(element.address)); } } @@ -258,6 +260,22 @@ void RunTextureTilingTests() { Require(odd.mips[0].blocksPerRow == 5 && odd.mips[0].tiledOffset == 12288 && odd.mips[0].tiledSize == 40960, "3D mip 0 of a non-power-of-two volume changed"); } + { + constexpr ElementAddress s4xTail1[] = {{0, 0, 0, 0, 0x3000, 0x0}, {0, 32, 19, 11, 0x843c, 0x0}, {0, 8, 9, 8, 0x3e08, 0x0}, {0, 15, 2, 8, 0x3163, 0x700}, {1, 0, 0, 0, 0x1000, 0x0}, {1, 15, 9, 5, 0x1acf, 0x0}, {1, 12, 4, 0, 0x1340, 0x0}, {1, 7, 2, 5, 0x17e7, 0x700}, {2, 0, 0, 0, 0x800, 0x0}, {2, 7, 4, 2, 0x953, 0x0}, {2, 2, 2, 0, 0x822, 0x0}, {2, 0, 1, 1, 0xf0c, 0x700}, {3, 0, 0, 0, 0x300, 0x0}, {3, 3, 1, 0, 0x30b, 0x0}, {3, 3, 1, 0, 0x30b, 0x0}, {3, 0, 0, 0, 0x400, 0x700}, {4, 0, 0, 0, 0x200, 0x0}, {4, 1, 0, 0, 0x201, 0x0}, {4, 0, 0, 0, 0x200, 0x0}, {4, 0, 0, 0, 0x500, 0x700}, {5, 0, 0, 0, 0x100, 0x0}, {5, 0, 0, 0, 0x100, 0x0}, {5, 0, 0, 0, 0x100, 0x0}, {5, 0, 0, 0, 0x600, 0x700}}; + requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 1, 33, 20, 12, 6), 1, 2, 36864, s4xTail1, "SW_4KB_S_X 8 bpp 33x20x12, 6 levels"); + constexpr ElementAddress s4xTail2[] = {{0, 0, 0, 0, 0x6000, 0x0}, {0, 32, 19, 11, 0xf13c, 0x0}, {0, 8, 9, 8, 0x7408, 0x0}, {0, 15, 2, 8, 0x7962, 0x700}, {1, 0, 0, 0, 0x3000, 0x0}, {1, 15, 9, 5, 0x42ce, 0x0}, {1, 12, 4, 0, 0x4b00, 0x0}, {1, 7, 2, 5, 0x35e6, 0x700}, {2, 0, 0, 0, 0x1000, 0x0}, {2, 7, 4, 2, 0x1352, 0x0}, {2, 2, 2, 0, 0x1060, 0x0}, {2, 0, 1, 1, 0x170c, 0x700}, {3, 0, 0, 0, 0x800, 0x0}, {3, 3, 1, 0, 0x84a, 0x0}, {3, 3, 1, 0, 0x84a, 0x0}, {3, 0, 0, 0, 0xf00, 0x700}, {4, 0, 0, 0, 0x300, 0x0}, {4, 1, 0, 0, 0x302, 0x0}, {4, 0, 0, 0, 0x300, 0x0}, {4, 0, 0, 0, 0x400, 0x700}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x500, 0x700}}; + requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 12, 33, 20, 12, 6), 2, 3, 65536, s4xTail2, "SW_4KB_S_X 16 bpp 33x20x12, 6 levels"); + constexpr ElementAddress s4xWide4[] = {{0, 0, 0, 0, 0x20000, 0x0}, {0, 127, 127, 7, 0x9f4fc, 0x0}, {0, 24, 93, 0, 0x73108, 0x0}, {0, 42, 22, 4, 0x35f60, 0x700}, {1, 0, 0, 0, 0x0, 0x0}, {1, 63, 63, 3, 0x1f0fc, 0x0}, {1, 28, 38, 2, 0x13da0, 0x0}, {1, 55, 52, 1, 0x1e754, 0x700}}; + requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 56, 128, 128, 8, 2), 4, 2, 655360, s4xWide4, "SW_4KB_S_X 32 bpp 128x128x8, 2 levels"); + constexpr ElementAddress s4xTail4[] = {{0, 0, 0, 0, 0x6000, 0x0}, {0, 32, 19, 11, 0x1fdb8, 0x0}, {0, 8, 9, 8, 0x17808, 0x0}, {0, 15, 2, 8, 0x17564, 0x700}, {1, 0, 0, 0, 0x3000, 0x0}, {1, 15, 9, 5, 0x465c, 0x0}, {1, 12, 4, 0, 0x4b00, 0x0}, {1, 7, 2, 5, 0x3174, 0x700}, {2, 0, 0, 0, 0x1000, 0x0}, {2, 7, 4, 2, 0x13c4, 0x0}, {2, 2, 2, 0, 0x1060, 0x0}, {2, 0, 1, 1, 0x1718, 0x700}, {3, 0, 0, 0, 0x800, 0x0}, {3, 3, 1, 0, 0x84c, 0x0}, {3, 3, 1, 0, 0x84c, 0x0}, {3, 0, 0, 0, 0xf00, 0x700}, {4, 0, 0, 0, 0x300, 0x0}, {4, 1, 0, 0, 0x304, 0x0}, {4, 0, 0, 0, 0x300, 0x0}, {4, 0, 0, 0, 0x400, 0x700}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x500, 0x700}}; + requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 56, 33, 20, 12, 6), 4, 3, 131072, s4xTail4, "SW_4KB_S_X 32 bpp 33x20x12, 6 levels"); + constexpr ElementAddress s4xTail8[] = {{0, 0, 0, 0, 0x9000, 0x0}, {0, 32, 19, 11, 0x2fab0, 0x0}, {0, 8, 9, 8, 0x27420, 0x0}, {0, 15, 2, 8, 0x22448, 0x700}, {1, 0, 0, 0, 0x3000, 0x0}, {1, 15, 9, 5, 0x7a78, 0x0}, {1, 12, 4, 0, 0x4200, 0x0}, {1, 7, 2, 5, 0x3058, 0x700}, {2, 0, 0, 0, 0x1000, 0x0}, {2, 7, 4, 2, 0x1ac8, 0x0}, {2, 2, 2, 0, 0x1140, 0x0}, {2, 0, 1, 1, 0x1730, 0x700}, {3, 0, 0, 0, 0x800, 0x0}, {3, 3, 1, 0, 0x868, 0x0}, {3, 3, 1, 0, 0x868, 0x0}, {3, 0, 0, 0, 0xf00, 0x700}, {4, 0, 0, 0, 0x300, 0x0}, {4, 1, 0, 0, 0x308, 0x0}, {4, 0, 0, 0, 0x300, 0x0}, {4, 0, 0, 0, 0x400, 0x700}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x500, 0x700}}; + requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 71, 33, 20, 12, 6), 8, 3, 196608, s4xTail8, "SW_4KB_S_X 64 bpp 33x20x12, 6 levels"); + constexpr ElementAddress s4xTail16[] = {{0, 0, 0, 0, 0x10000, 0x0}, {0, 32, 19, 11, 0x55bb0, 0x0}, {0, 8, 9, 8, 0x46820, 0x0}, {0, 15, 2, 8, 0x3e040, 0x700}, {1, 0, 0, 0, 0x6000, 0x0}, {1, 15, 9, 5, 0xee70, 0x0}, {1, 12, 4, 0, 0x9400, 0x0}, {1, 7, 2, 5, 0x7850, 0x700}, {2, 0, 0, 0, 0x3000, 0x0}, {2, 7, 4, 2, 0x42c0, 0x0}, {2, 2, 2, 0, 0x3300, 0x0}, {2, 0, 1, 1, 0x3730, 0x700}, {3, 0, 0, 0, 0x1000, 0x0}, {3, 3, 1, 0, 0x1260, 0x0}, {3, 3, 1, 0, 0x1260, 0x0}, {3, 0, 0, 0, 0x1700, 0x700}, {4, 0, 0, 0, 0x800, 0x0}, {4, 1, 0, 0, 0x840, 0x0}, {4, 0, 0, 0, 0x800, 0x0}, {4, 0, 0, 0, 0xf00, 0x700}, {5, 0, 0, 0, 0x300, 0x0}, {5, 0, 0, 0, 0x300, 0x0}, {5, 0, 0, 0, 0x300, 0x0}, {5, 0, 0, 0, 0x400, 0x700}}; + requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 77, 33, 20, 12, 6), 16, 4, 352256, s4xTail16, "SW_4KB_S_X 128 bpp 33x20x12, 6 levels"); + requireBlockVolumeLayout(TextureTileMode::kS4KBX, 169, 71, 60, 36, 20, 5, "SW_4KB_S_X BC1 60x36x20, 5 levels"); + } + { constexpr ElementAddress chainInTail[] = {{0, 0, 0, 0, 0x800}, {0, 7, 7, 7, 0xffc}, {0, 4, 2, 4, 0xe20}, {0, 2, 7, 0, 0x968}, {0, 7, 0, 7, 0xed4}, {1, 0, 0, 0, 0x300}, {1, 3, 3, 3, 0x3fc}, {1, 2, 1, 2, 0x3c8}, {1, 1, 3, 0, 0x32c}, {1, 3, 0, 3, 0x3d4}}; requireThickAddresses(thickVolume(TextureTileMode::kStandard4KB, 56, 8, 8, 8, 2), 4, 0, 4096, chainInTail, "SW_4KB_S 32 bpp 8x8x8, 2 levels"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp index 52c5728108..4c3e66ac8f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp @@ -27,6 +27,7 @@ constexpr std::uint32_t Height = 24; constexpr std::uint32_t Depth = 20; constexpr std::uint32_t Format8888UNorm = 56; constexpr std::uint32_t Type3D = 10; +constexpr std::uint32_t TileS4KBX = 0x15; constexpr std::uint32_t TileZ64KBX = 0x18; constexpr std::uint32_t TileS64KBX = 0x19; constexpr std::uint32_t TileD64KBX = 0x1a; @@ -79,6 +80,7 @@ void FillTexels(std::uint32_t tileMode) { Require(geometry.guestBytes <= Texels.size(), "volume does not fit the test allocation"); const auto* equation = FindTextureSwizzleEquation(geometry.thick ? 0x100u | XorSwizzleMode(descriptor.tileMode) : XorSwizzleMode(descriptor.tileMode), 4u); Require(equation != nullptr, "volume swizzle equation is missing"); + const auto blockBytes = ThinBlockLayout(descriptor.tileMode, 4u)[0]; const auto& mip = geometry.mips.at(0); const auto block = geometry.thick ? ThickBlockExtent(descriptor.tileMode, 4u) : std::array{ThinBlockLayout(descriptor.tileMode, 4u)[1], ThinBlockLayout(descriptor.tileMode, 4u)[2], 1u}; for (std::uint32_t z = 0; z < Depth; ++z) { @@ -86,7 +88,7 @@ void FillTexels(std::uint32_t tileMode) { for (std::uint32_t x = 0; x < Width; ++x) { auto offset = geometry.GuestLayerOffset(z) + mip.tiledOffset; if (mip.tail) offset += EquationOffset(*equation, x + mip.tailX, y + mip.tailY, z); - else offset += (static_cast(y / block[1]) * mip.blocksPerRow + x / block[0]) * 65536u + EquationOffset(*equation, x, y, z); + else offset += (static_cast(y / block[1]) * mip.blocksPerRow + x / block[0]) * blockBytes + EquationOffset(*equation, x, y, z); const auto texel = Texel(x, y, z); std::copy(texel.begin(), texel.end(), Texels.begin() + static_cast(offset)); } @@ -145,6 +147,7 @@ int main() { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; FillInput(); + Run(*device, TileS4KBX, "thick SW_4KB_S_X volume"); Run(*device, TileS64KBX, "thick SW_64KB_S_X volume"); Run(*device, TileD64KBX, "thick SW_64KB_D_X volume"); Run(*device, TileZ64KBX, "thin SW_64KB_Z_X volume"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp index 20ab53b94c..5581d46dcc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp @@ -97,9 +97,9 @@ void BuildBands() { } } -AgcDriver::Graphics::ColorTarget VolumeTarget(std::byte* volume) { +AgcDriver::Graphics::ColorTarget VolumeTarget(std::byte* volume, ColorTileMode tileMode = ColorTileMode::RenderTarget) { AgcDriver::Graphics::ColorTarget color{reinterpret_cast(volume), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, 0, 0xe4u}; - color.tileMode = ColorTileMode::RenderTarget; + color.tileMode = tileMode; color.depth = Slices; color.depthSlice = FirstSlice; color.layers = Layers; @@ -239,19 +239,23 @@ std::uint64_t BlockOffset(const AgcDriver::Graphics::TextureSwizzleEquation& equ } void Check(const std::byte* volume, const AgcDriver::Graphics::ColorTarget& color, const std::string& what, const std::array& bands) { + using namespace AgcDriver::Graphics; const auto geometry = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::SurfaceForTarget(color)); - Require(!geometry.thick, "a 64 KiB render-target volume is not stored as thin slices"); - const auto block = AgcDriver::Graphics::ThinBlockLayout(AgcDriver::Graphics::TextureTileMode::kR64KBX, 4); - Require(Width <= block[1] && Height <= block[2], "the volume slices span several tile blocks"); - const auto* equation = AgcDriver::Graphics::FindTextureSwizzleEquation(static_cast(ColorTileMode::RenderTarget), 4); - Require(equation != nullptr, "missing the 64 KiB R_X swizzle equation"); + Require(geometry.thick == (color.tileMode == ColorTileMode::S4KBX), "unexpected volume addressing family"); + const auto tileMode = ColorTextureTileMode(color.tileMode); + const auto thin = ThinBlockLayout(tileMode, 4); + const auto block = geometry.thick ? ThickBlockExtent(tileMode, 4) : std::array{thin[1], thin[2], 1u}; + const auto& mip = geometry.mips.at(0); + const auto* equation = FindTextureSwizzleEquation((geometry.thick ? 0x100u : 0u) | static_cast(color.tileMode), 4); + Require(equation != nullptr, "missing the volume's swizzle equation"); for (std::uint32_t slice = 0; slice < Slices; ++slice) { const auto* base = volume + geometry.GuestLayerOffset(slice); for (std::uint32_t y = 0; y < Height; ++y) { for (std::uint32_t x = 0; x < Width; ++x) { const bool drawn = slice >= FirstSlice && x / BandWidth == slice - FirstSlice; const auto expected = drawn ? bands[slice - FirstSlice] : Kept; - const auto* texel = base + BlockOffset(*equation, x, y, slice); + const auto offset = (static_cast(y / block[1]) * mip.blocksPerRow + x / block[0]) * thin[0] + BlockOffset(*equation, x, y, slice); + const auto* texel = base + offset; for (std::uint32_t channel = 0; channel < 4; ++channel) { const auto actual = std::to_integer(texel[channel]); Require(actual == expected || (drawn && expected == 128 && actual == 127), what + ": slice " + std::to_string(slice) + " texel (" + std::to_string(x) + ", " + std::to_string(y) + ") channel " + std::to_string(channel) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected) + (drawn ? " (the band of layer " + std::to_string(slice - FirstSlice) + ")" : slice < FirstSlice ? " (outside the color view)" : " (another layer's band)")); @@ -283,6 +287,10 @@ int main(int argc, char** argv) { const auto fieldColor = VolumeTarget(fieldVolume.Data()); Run(*device, fieldColor, false, LayerFieldPixelCode, true); Check(fieldVolume.Data(), fieldColor, "a pixel shader reading the render target index of its packed ancillary word", LayerFieldBands); + const GuestVolume thickVolume((VolumeTarget(nullptr, ColorTileMode::S4KBX).bytes + 65535u) & ~std::size_t{65535u}); + const auto thickColor = VolumeTarget(thickVolume.Data(), ColorTileMode::S4KBX); + Run(*device, thickColor, false, PixelCode, false); + Check(thickVolume.Data(), thickColor, "thick SW_4KB_S_X vertex draw and writeback", WhiteBands); if (vertexOnly) std::puts("layered volume vertex draw tests passed"); else if (target.mesh.has_value()) Run(*device, color, true, PixelCode, false); else std::puts("mesh case skipped, the device has no VK_EXT_mesh_shader"); diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index e5ade18beb..67550c6211 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -172,3 +172,4 @@ - [Registration preparation at submit](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (libSceAgcDriver) - `Driver::Submit` waits for every registration-time shader preparation queued before the call, not only those of the shaders the commands use, so a submission behind a long queue of unrelated preparations waits for all of them on the submitting thread. Waiting there instead of in the queue worker keeps the GPU side of a submission short: a title that sees a submission take seconds treats the GPU as hung (Goat Simulator 3, PPSA04159, then stops submitting and its garbage collector fails a 10 s timeout). On the console shaders are compiled ahead of time, so neither wait exists; how long a title tolerates a slow submit call is not known. - [Unclamped normalized color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - CB_COLOR_INFO targets with BLEND_CLAMP clear (bit 15) on UNORM and SNORM formats are accepted when blending is disabled (ENABLE clear or BLEND_BYPASS set). Vulkan converts unblended shader values and clamps to the normalized range automatically; blending into an unclamped normalized target remains unmodeled because Vulkan always clamps inputs prior to blending. +- [Thick SW_4KB_S_X volumes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp) use AMD addrlib's GFX10 non-RB+ 16-pipe `GFX10_SW_4K_S3_X_PATINFO` equations for 1, 2, 4, 8 and 16-byte elements, including packed mip tails. The layout follows [AMD PAL c5e80007](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/imported/addrlib/src/gfx10/gfx10addrlib.cpp); it has not been measured on console hardware. Other thick swizzle modes outside the supported families still throw. Mipmapped 3D color targets remain refused by the color-register decoder. From 9dcf40803404c50b7e965061fecb6d06b16890c0 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:59:06 -0700 Subject: [PATCH 061/103] test(agc): expect a per-lane table image store to be supported Main's #6402 runs image stores through an image table, so a T# keyed per lane keeps its table with a supported image_store and an unsupported mark for the other operations. --- core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 3fadd9c234..52c8a5f761 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -2090,7 +2090,7 @@ void verifySelectedRecordDirectOperations() { require(TableBinding(*compiled.result, DescriptorRole::ImageTable) == nullptr && compiled.result->imageTableFaults == 0u && TableCovered(compiled.regions, &heap[2], sizeof(heap[2])), (what + " through a T# whose record the CPU evaluates was not bound from its record").c_str()); const auto wholeCode = WithImageOperation(TableWholeCode, word0, false); const auto whole = GetResourcePlan(TableRequest(wholeCode, userData)); - require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == TableOperation::Unsupported, (what + " through a T# keyed per lane left the image table or lost its unsupported mark").c_str()); + require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == (word0 == 0xf0200f08u ? TableOperation::Supported : TableOperation::Unsupported), (what + " through a T# keyed per lane left the image table or has the wrong table operation").c_str()); } const auto mixedCode = WithImageOperation(TableSrtKeyCode, 0xf0200f08u, true); const auto mixed = GetResourcePlan(TableRequest(mixedCode, userData)); From 1ee2c1c51073d2fdd3f6300cff324f64ff325ff4 Mon Sep 17 00:00:00 2001 From: Cleverdee <102689759+Cleverdee@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:58:31 +0500 Subject: [PATCH 062/103] fix(recompiler): fold chained immediate addends of image-table record offsets SplitImmediateAddend peeled a single immediate operand off a T# dword's scalar offset. A title that loads a 32-byte record with two s_load_dwordx4, the second at soffset (first soffset + 16) where the first is (key << 5) + C, left IAdd32(key << 5, C) as the base of dwords 4-7, which never matched dword 0's base, and the image failed with "GetImageResource dword 0 is not a valid runtime value". Peel immediate IAdd32 operands until none is left and sum them modulo 2^32, as the scalar adds do. The chained offsets then reach the existing folded-addend path. verifyChainedTableOffset is the pointer table offset carry shader with the second offset chained off the first. Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 36 +++++++++++++++++++ .../Optimization/src/ResourceTracker.cpp | 12 ++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index fcc5051170..0ef746978e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1904,6 +1904,40 @@ void verifyPointerTableOffsetCarry() { #endif } +void verifyChainedTableOffset() { + using namespace ShaderRecompiler; + namespace Abi = ImageTableAbi; + static TableTexture textures[2]; + alignas(256) static std::array root{}; + static std::array keys{}; + static std::array output{}; + const auto keysV = TableV(keys.data(), 0u, static_cast(sizeof(keys))); + const auto outputV = TableV(output.data(), 0u, static_cast(sizeof(output))); + std::copy(keysV.begin(), keysV.end(), root.begin()); + std::copy(outputV.begin(), outputV.end(), root.begin() + 4); + const auto at = [&](std::uint32_t offset, std::array words) { + std::copy(words.begin(), words.end(), root.begin() + offset / 4u); + }; + at(0x60u + 3u * 32u, TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D)); + at(0x60u + 5u * 32u, TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D)); + const auto base = static_cast(reinterpret_cast(root.data())); + const std::array userData{static_cast(base), static_cast(base >> 32u)}; + const std::vector code{0xf4080200u, 0xfa000000u, 0xf4080700u, 0xfa000010u, 0xf4080800u, 0xfa000020u, 0x7e200500u, 0x8f108210u, 0xf4200444u, + 0x20000000u, 0x8711ff11u, 0x000000ffu, 0x8f128511u, 0x8113a012u, 0x81149013u, 0xf4080900u, 0x26000040u, 0xf4080a00u, 0x28000040u, 0xf09c0f08u, + 0x01090000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; + auto request = TableRequest(code, userData); + request.shader.codeAddress = 0x37000u; + const auto plan = GetResourcePlan(request); + require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "chained table offset: a T# whose second load offset adds to the first one's was not planned as a table"); + const auto& column = *plan->descriptorSources.at(plan->info.images[0].source).tableColumn; + require(column.address && column.stride == 32u && column.addend == 0x20u && column.offset == 0x40u && column.maxKey == 0xffu && column.foldedAddend == 0x20u && column.foldedCarry == 16u, "chained table offset: the column is wrong"); + const auto compiled = CompileTable(request); + const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0); + const auto& words = compiled.capture->snapshot.tables.words; + require(snapshot.keys == 256u && (snapshot.codes[3] & Abi::PoisonFlag) == 0u && (snapshot.codes[5] & Abi::PoisonFlag) == 0u, "chained table offset: the records were not resolved"); + require(words.at(snapshot.codes[3]).dwords[0] == root[(0x60u + 3u * 32u) / 4u] && words.at(snapshot.codes[3]).dwords[7] == root[(0x60u + 3u * 32u) / 4u + 7u] && words.at(snapshot.codes[5]).dwords[0] == root[(0x60u + 5u * 32u) / 4u] && snapshot.codes[3] != snapshot.codes[5], "chained table offset: a record was read from the wrong offset"); +} + void verifyDescriptorPhis() { using namespace ShaderRecompiler; constexpr std::uint32_t Format8888UNorm = 56; @@ -5307,6 +5341,7 @@ int main(int argc, char** argv) { verifyScalarAddressRule(); verifyPointerImageTable(); verifyPointerTableOffsetCarry(); + verifyChainedTableOffset(); std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } @@ -5338,6 +5373,7 @@ int main(int argc, char** argv) { verifyScalarAddressRule(); verifyPointerImageTable(); verifyPointerTableOffsetCarry(); + verifyChainedTableOffset(); verifyDescriptorPhis(); verifyUniformTableImage(); verifyLaneSelectedDescriptors(); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index d7d0e65d29..a871b4cb57 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -542,11 +542,15 @@ private: static IrValue* SplitImmediateAddend(IrValue* value, std::uint32_t& addend) { value = value->Resolve(); addend = 0; - if (value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) { - if (immediateU32(value->Argument(0), addend)) return value->Argument(1)->Resolve(); - if (immediateU32(value->Argument(1), addend)) return value->Argument(0)->Resolve(); + while (value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) { + std::uint32_t immediate = 0; + IrValue* rest = nullptr; + if (immediateU32(value->Argument(0), immediate)) rest = value->Argument(1)->Resolve(); + else if (immediateU32(value->Argument(1), immediate)) rest = value->Argument(0)->Resolve(); + else break; + addend += immediate; + value = rest; } - addend = 0; return value; } From 7665cb3ef50dcad7350f39d5c333b8aa9ca7a1e4 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Sun, 18 Oct 2026 12:08:33 +1300 Subject: [PATCH 063/103] test(tools): check shader comments in one fixture repository conventions_function_bodies_shard_4 timed out at 30 s on Windows in five runs on 17 Oct (main pushes 39076267931 and 39061030084, dispatches 39074123651 and 39044676931, relinker 39072918668). Shard 4 was the slowest shard in every Windows log (10-16 s on a quiet runner, 5-11 s for the others). One case dominated it: test_comment_in_a_shader_is_reported built six fixture git repositories, 36 of the suite's 390 processes. Process start-up is what costs on Windows. #7314 doubled the shard count, but modulo sharding keeps that case whole, so its shard (now shard 10) still cost 0.63 s of the suite's 3.0 s locally against 0.76 s before. The case now writes all six shaders into one fixture and runs the checker once (6 processes), still asserting each path. The slowest of the 12 shards drops to 0.39 s locally (43 processes), with no timeout raised. --- .../tests/test_check_conventions_comments.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/tests/test_check_conventions_comments.py b/tools/tests/test_check_conventions_comments.py index e6df79d55d..cbb130d784 100644 --- a/tools/tests/test_check_conventions_comments.py +++ b/tools/tests/test_check_conventions_comments.py @@ -345,7 +345,8 @@ class SilentStubSectionTests(unittest.TestCase): class TestFixtureExportTests(unittest.TestCase): - def check(self, path, text): + def check(self, path, text=None): + files = path if text is None else {path: text} checker = Path(__file__).resolve().parents[1] / "check_conventions.py" with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -367,7 +368,8 @@ class TestFixtureExportTests(unittest.TestCase): write("core/libs/prx/libSceNgs2/tests/Render.cpp", "static int APS5_VABI Process(int value) { return value; }\n") git("add", ".") git("commit", "-qm", "feat: add libraries") - write(path, text) + for relative, content in files.items(): + write(relative, content) git("add", ".") git("commit", "-qm", "feat: add source") env = {key: value for key, value in os.environ.items() @@ -396,12 +398,16 @@ class TestFixtureExportTests(unittest.TestCase): self.assertIn("sceKernelGetProcessTime is also in libkernel", result.stdout) def test_comment_in_a_shader_is_reported(self): + shaders = {} for path in ("core/libs/prx/libSceAgcDriver/Graphics/shaders/Fill.comp", "core/libs/prx/libSceAgcDriver/tests/shaders/Fill.frag", "core/libs/prx/libSceAgcDriver/Graphics/shaders/Fill.glsl"): - for line in (" // fill the target", " /* fill the target */"): - with self.subTest(path=path, line=line): - result = self.check(path, f"#version 450\nvoid main() {{\n{line}\n}}\n") - self.assertEqual(result.returncode, 1, result.stdout + result.stderr) - self.assertIn(f"{path}:3: error: [comment]", result.stdout) + for style, line in (("Line", " // fill the target"), ("Block", " /* fill the target */")): + stem, extension = path.rsplit(".", 1) + shaders[f"{stem}{style}.{extension}"] = f"#version 450\nvoid main() {{\n{line}\n}}\n" + result = self.check(shaders) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + for path in shaders: + with self.subTest(path=path): + self.assertIn(f"{path}:3: error: [comment]", result.stdout) def test_shader_without_comments_passes(self): result = self.check("core/libs/prx/libSceAgcDriver/Graphics/shaders/Fill.vert", From 39dd854c84a01485bea2e493572c8bcb62a65aa9 Mon Sep 17 00:00:00 2001 From: blu <131330772+ycbuzi@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:20:01 -0700 Subject: [PATCH 064/103] fix(relinker): keep test artifacts in the build tree The tests build their working directories with tempfile.TemporaryDirectory() and std::filesystem::temp_directory_path(), so every executable they generate lands in the user's system temp directory. On Windows, Microsoft Defender quarantines those generated binaries as Trojan:Win32/Wacatac.H!ml, and the nine tests that run one fail with WinError 5 (access denied) on a clean checkout: autorun_exit_status, optional_plt, empty_tls, tls_function_coverage, executable_segments, windows_icon, guest_needed_modules, guest_module_directories and windows_import_modules. Point TMPDIR, TEMP and TMP at a build-local directory for every test registered here, so generated executables stay inside the build tree. Test artifacts are then removed with the build directory, and a contributor who needs an antivirus exclusion can scope it to the build tree instead of the system temp directory. --- core/relinker/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index a0b72b06c9..dab82857bf 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -735,6 +735,12 @@ if(BUILD_TESTING) ) add_test(NAME reloc_table_bounds COMMAND reloc_table_bounds_tests $) endif() + + set(relinkerTestTemp ${CMAKE_CURRENT_BINARY_DIR}/test-temp) + file(MAKE_DIRECTORY ${relinkerTestTemp}) + get_property(relinkerTests DIRECTORY PROPERTY TESTS) + set_tests_properties(${relinkerTests} PROPERTIES ENVIRONMENT_MODIFICATION + "TMPDIR=set:${relinkerTestTemp};TEMP=set:${relinkerTestTemp};TMP=set:${relinkerTestTemp}") endif() if(BUILD_TESTING AND Python3_Interpreter_FOUND) From b771ce4f377e2e971d0d038888db85c383ec62b8 Mon Sep 17 00:00:00 2001 From: Cesar Rodriguez <11452991+Tamalero@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:06:56 -0700 Subject: [PATCH 065/103] fix(libkernel): name module-relative call sites and thread ids in Linux wait traces On Linux, APS5_TRACE_WAITS and APS5_TRACE_USLEEP used an image base and a thread id of 0, so "exe+0x..." printed absolute addresses and every site reported "tid 0". The call site is now resolved with dladdr1(RTLD_DL_LINKMAP) and printed relative to its module's load address ("exe+..." for the executable, the module file name otherwise), matching guest offsets in a disassembly; the thread id is gettid(). Descriptions are cached per call site, so polling titles do not pay a dladdr1 per wait. Windows output is unchanged. --- core/libs/prx/libkernel/Time/Time.cpp | 60 ++++++++++++++++++++------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/core/libs/prx/libkernel/Time/Time.cpp b/core/libs/prx/libkernel/Time/Time.cpp index 059c0236f7..12523a767b 100644 --- a/core/libs/prx/libkernel/Time/Time.cpp +++ b/core/libs/prx/libkernel/Time/Time.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,10 @@ #include #include #include +#ifdef __linux__ +#include +#include +#endif // __linux__ #endif extern "C" int* APS5_VABI __error_nid_postfix(); @@ -179,6 +184,39 @@ std::uint64_t APS5_VABI sceKernelGetProcessTimeCounterFrequency() { return 1000000000ULL; } +static std::string DescribeCallSite(const void* caller) { + const auto address = reinterpret_cast(caller); + char text[192]; +#ifdef _WIN32 + const auto image = reinterpret_cast(GetModuleHandleW(nullptr)); + std::snprintf(text, sizeof(text), "exe+0x%llx", static_cast(address - image)); +#elif defined(__linux__) + Dl_info info{}; + link_map* map = nullptr; + if (dladdr1(caller, &info, reinterpret_cast(&map), RTLD_DL_LINKMAP) == 0 || map == nullptr) { + std::snprintf(text, sizeof(text), "0x%llx", static_cast(address)); + return text; + } + const char* path = map->l_name; + const char* slash = path == nullptr ? nullptr : std::strrchr(path, '/'); + const char* module = path == nullptr || *path == '\0' ? "exe" : slash == nullptr ? path : slash + 1; + std::snprintf(text, sizeof(text), "%s+0x%llx", module, static_cast(address - map->l_addr)); +#else + std::snprintf(text, sizeof(text), "0x%llx", static_cast(address)); +#endif + return text; +} + +static unsigned long CurrentThreadIdForTrace() { +#ifdef _WIN32 + return GetCurrentThreadId(); +#elif defined(__linux__) + return static_cast(gettid()); +#else + return 0; +#endif +} + // Debug aid: APS5_TRACE_USLEEP reports every 2000 calls which guest call sites sleep, so a CPU // thread polling for GPU or I/O completion can be found (offsets are relative to the executable). static void TraceSleep(const void* caller, std::uint64_t microseconds) { @@ -192,15 +230,10 @@ static void TraceSleep(const void* caller, std::uint64_t microseconds) { ++site.first; site.second += microseconds; if (++calls % 2000 != 0) return; -#ifdef _WIN32 - const auto image = reinterpret_cast(GetModuleHandleW(nullptr)); -#else - const std::uintptr_t image = 0; -#endif std::vector>> hot(sites.begin(), sites.end()); std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.first > b.second.first; }); std::fprintf(stderr, "[usleep] %llu calls:", static_cast(calls)); - for (std::size_t i = 0; i < hot.size() && i < 6; ++i) std::fprintf(stderr, " exe+0x%llx x%llu (%llu us)", static_cast(hot[i].first - image), static_cast(hot[i].second.first), static_cast(hot[i].second.second)); + for (std::size_t i = 0; i < hot.size() && i < 6; ++i) std::fprintf(stderr, " %s x%llu (%llu us)", DescribeCallSite(reinterpret_cast(hot[i].first)).c_str(), static_cast(hot[i].second.first), static_cast(hot[i].second.second)); std::fprintf(stderr, "\n"); } @@ -211,16 +244,13 @@ void KernelTraceWait_nid_postfix(const char* kind, const void* caller, std::uint static std::mutex mutex; static std::unordered_map sites; static std::uint64_t lastReport = 0; - char key[96]; -#ifdef _WIN32 - const auto image = reinterpret_cast(GetModuleHandleW(nullptr)); - const unsigned long thread = GetCurrentThreadId(); -#else - const std::uintptr_t image = 0; - const unsigned long thread = 0; -#endif - std::snprintf(key, sizeof(key), "%s exe+0x%llx", kind, static_cast(reinterpret_cast(caller) - image)); + static std::unordered_map callSites; + char key[224]; + const unsigned long thread = CurrentThreadIdForTrace(); std::lock_guard lock(mutex); + auto callSite = callSites.find(caller); + if (callSite == callSites.end()) callSite = callSites.emplace(caller, DescribeCallSite(caller)).first; + std::snprintf(key, sizeof(key), "%s %s", kind, callSite->second.c_str()); auto& site = sites[key]; ++site.calls; site.timeouts += timedOut ? 1u : 0u; From 619257798abe5bc12a5064a8042853204c2464ab Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:36:14 +0000 Subject: [PATCH 066/103] refactor(libSceSystemService): share the console language setting SystemServiceConsoleLanguage in SystemService.hpp reads ANYPS5_LANGUAGE with the same default, range and error as sceSystemServiceParamGetInt, which now calls it. Other libraries can pick the console language without linking libSceSystemService. --- core/libs/prx/libSceSystemService/Export.cpp | 17 +--------------- .../prx/libSceSystemService/SystemService.hpp | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/core/libs/prx/libSceSystemService/Export.cpp b/core/libs/prx/libSceSystemService/Export.cpp index 6480b49dfc..0e66ec6bd9 100644 --- a/core/libs/prx/libSceSystemService/Export.cpp +++ b/core/libs/prx/libSceSystemService/Export.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -28,20 +27,6 @@ namespace { -int ConsoleLanguage() { - const char* configured = std::getenv("ANYPS5_LANGUAGE"); - if (configured == nullptr || configured[0] == '\0') return SYSTEM_SERVICE_PARAM_LANG_ENGLISH_US; - const std::string_view text(configured); - int language = -1; - const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), language); - if (error != std::errc{} || end != text.data() + text.size() || language < SYSTEM_SERVICE_PARAM_LANG_JAPANESE || language > SYSTEM_SERVICE_PARAM_LANG_UKRAINIAN) { - throw std::runtime_error("sceSystemServiceParamGetInt: ANYPS5_LANGUAGE must be a console language number from " + - std::to_string(SYSTEM_SERVICE_PARAM_LANG_JAPANESE) + " to " + std::to_string(SYSTEM_SERVICE_PARAM_LANG_UKRAINIAN) + - ", not \"" + std::string(text) + "\""); - } - return language; -} - struct ConsoleTimeZone { int minutes; bool summer; @@ -298,7 +283,7 @@ int APS5_VABI sceSystemServiceParamGetInt(int paramId, int* value) { } switch (paramId) { case SYSTEM_SERVICE_PARAM_ID_LANG: - case SYSTEM_SERVICE_PARAM_ID_400: *value = ConsoleLanguage(); break; + case SYSTEM_SERVICE_PARAM_ID_400: *value = SystemServiceConsoleLanguage("sceSystemServiceParamGetInt"); break; case SYSTEM_SERVICE_PARAM_ID_DATE_FORMAT: *value = SYSTEM_SERVICE_PARAM_DATE_FORMAT_DDMMYYYY; break; case SYSTEM_SERVICE_PARAM_ID_TIME_FORMAT: *value = SYSTEM_SERVICE_PARAM_TIME_FORMAT_24HOUR; break; case SYSTEM_SERVICE_PARAM_ID_TIME_ZONE: *value = HostTimeZone().minutes; break; diff --git a/core/libs/prx/libSceSystemService/SystemService.hpp b/core/libs/prx/libSceSystemService/SystemService.hpp index 99b73cc0b5..7b1664cc90 100644 --- a/core/libs/prx/libSceSystemService/SystemService.hpp +++ b/core/libs/prx/libSceSystemService/SystemService.hpp @@ -1,8 +1,14 @@ #ifndef CORE_LIBS_PRX_LIBSCESYSTEMSERVICE_SYSTEMSERVICE_HPP #define CORE_LIBS_PRX_LIBSCESYSTEMSERVICE_SYSTEMSERVICE_HPP +#include #include #include +#include +#include +#include +#include +#include constexpr int SYSTEM_SERVICE_OK = 0; constexpr int SYSTEM_SERVICE_ERROR_PARAMETER = -2136932349; @@ -31,4 +37,18 @@ constexpr int SYSTEM_SERVICE_PARAM_TIME_FORMAT_24HOUR = 1; constexpr int SYSTEM_SERVICE_PARAM_GAME_PARENTAL_OFF = 0; constexpr int SYSTEM_SERVICE_PARAM_ENTER_BUTTON_CROSS = 1; +inline int SystemServiceConsoleLanguage(std::string_view caller) { + const char* configured = std::getenv("ANYPS5_LANGUAGE"); + if (configured == nullptr || configured[0] == '\0') return SYSTEM_SERVICE_PARAM_LANG_ENGLISH_US; + const std::string_view text(configured); + int language = -1; + const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), language); + if (error != std::errc{} || end != text.data() + text.size() || language < SYSTEM_SERVICE_PARAM_LANG_JAPANESE || language > SYSTEM_SERVICE_PARAM_LANG_UKRAINIAN) { + throw std::runtime_error(std::string(caller) + ": ANYPS5_LANGUAGE must be a console language number from " + + std::to_string(SYSTEM_SERVICE_PARAM_LANG_JAPANESE) + " to " + std::to_string(SYSTEM_SERVICE_PARAM_LANG_UKRAINIAN) + + ", not \"" + std::string(text) + "\""); + } + return language; +} + #endif From e40191d7c91da40b070f92a7dfd48c298742bf05 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:36:15 +0000 Subject: [PATCH 067/103] fix(libSceNpTrophy2): give progress values a type and a 64-bit value A progress value is {int32 type, 4 reserved bytes, uint64 value}, as KytyPS5 lays it out, which makes the trophy details 1312 bytes and the trophy data 32 bytes. Size checks for every query structure keep the layout fixed, and the technical debt names its single public source. --- .../libSceNpTrophy2/include/NpTrophy2Constants.hpp | 3 ++- .../prx/libSceNpTrophy2/include/NpTrophy2Types.hpp | 14 +++++++++++++- docs/dev/debt/libSceNpTrophy2.md | 2 ++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceNpTrophy2/include/NpTrophy2Constants.hpp b/core/libs/prx/libSceNpTrophy2/include/NpTrophy2Constants.hpp index 8102f3785b..6a853b4c28 100644 --- a/core/libs/prx/libSceNpTrophy2/include/NpTrophy2Constants.hpp +++ b/core/libs/prx/libSceNpTrophy2/include/NpTrophy2Constants.hpp @@ -29,7 +29,8 @@ static constexpr std::uint32_t NP_TROPHY2_PROGRESS_PERCENTAGE = 0; static constexpr std::int32_t NP_TROPHY2_TROPHY_GRADE_BRONZE = 4; static constexpr std::int32_t NP_TROPHY2_PROGRESS_TYPE_NONE = 0; -static constexpr std::uint32_t NP_TROPHY2_PROGRESS_VALUE_NONE = 0; +static constexpr std::int32_t NP_TROPHY2_PROGRESS_TYPE_VALUE = 1; +static constexpr std::uint64_t NP_TROPHY2_PROGRESS_VALUE_NONE = 0; static constexpr std::size_t NP_TROPHY2_ICON_SIZE_NONE = 0; diff --git a/core/libs/prx/libSceNpTrophy2/include/NpTrophy2Types.hpp b/core/libs/prx/libSceNpTrophy2/include/NpTrophy2Types.hpp index 30fadecda7..ecfde98357 100644 --- a/core/libs/prx/libSceNpTrophy2/include/NpTrophy2Types.hpp +++ b/core/libs/prx/libSceNpTrophy2/include/NpTrophy2Types.hpp @@ -4,7 +4,11 @@ #include #include -struct NpTrophy2Progress { std::uint32_t value; }; +struct NpTrophy2Progress { + std::int32_t type; + std::uint8_t reserved[4]; + std::uint64_t value; +}; struct NpTrophy2GameDetails { std::uint32_t num_groups; @@ -67,4 +71,12 @@ struct NpTrophy2Data { std::uint64_t timestamp_tick; }; +static_assert(sizeof(NpTrophy2Progress) == 16); +static_assert(sizeof(NpTrophy2GameDetails) == 152); +static_assert(sizeof(NpTrophy2GameData) == 24); +static_assert(sizeof(NpTrophy2GroupDetails) == 152); +static_assert(sizeof(NpTrophy2GroupData) == 32); +static_assert(sizeof(NpTrophy2Details) == 1312); +static_assert(sizeof(NpTrophy2Data) == 32); + #endif diff --git a/docs/dev/debt/libSceNpTrophy2.md b/docs/dev/debt/libSceNpTrophy2.md index 142e30364e..d5e9083695 100644 --- a/docs/dev/debt/libSceNpTrophy2.md +++ b/docs/dev/debt/libSceNpTrophy2.md @@ -6,4 +6,6 @@ ### Unknown function info +- [Query structures](../../../core/libs/prx/libSceNpTrophy2/include/NpTrophy2Types.hpp): a progress value is a 16-byte `{int32 type, 4 reserved bytes, uint64 value}` and the game, group and trophy structures are 152, 24, 152, 32, 1312 and 32 bytes, as [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/9e41d8320a2774b303fbad344c2347e8d0292e83/src/libs/libTrophy.cpp) asserts; no other public source confirms the layout. Progress type 1 marks a progressive trophy's target and progress, as in KytyPS5. + - [sceNpTrophy2GetRewardIcon](../../../core/libs/prx/libSceNpTrophy2/src/Context.cpp) (libSceNpTrophy2) takes `(int context, int handle, int reward_id, void* buffer, size_t* size)`, from the call sites in the bundled `Media/Plugins/PSNCore.prx` of PPSA06888, the same shape as `sceNpTrophy2GetTrophyIcon`. The title calls it twice: first with a null buffer to read the size, then with a buffer of that size allocated with `malloc`. The meaning of the third argument is unverified, and the reward icon data is unknown, so every call throws From d374de5e3871410bae3efb94ca99c4ced7eaa25e Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:25:07 +0000 Subject: [PATCH 068/103] feat(libSceNpTrophy2): report the title's trophy data from trophyNN.ucp The title's sce_sys/trophy2/trophyNN.ucp is read on first use with a bounds-checked .ucp reader and a small JSON reader. tropconf.json gives each trophy's grade, group, hidden and reward flags, platinum and unlock condition, and tropmeta_.json its texts in the console language, then en-US, then the file's default language. Contexts remember their user and service label, so the queries find the file. The trophy, group and game queries report these names, grades, groups and counts, list every trophy and group, and give progressive trophies their target. Unlocks are not recorded yet, so every trophy reports as locked and nothing is written to the host. A missing file is ignored and a malformed one is logged once; either way the queries keep their placeholders. A JSON document that nests deeper than 64 levels or holds more than 1048576 values counts as malformed, which keeps the parsed values to about 140 MiB. --- core/libs/prx/libSceNpTrophy2/CMakeLists.txt | 15 + .../libSceNpTrophy2/include/TitleTrophies.hpp | 54 ++ .../libSceNpTrophy2/include/TrophyPackage.hpp | 96 ++++ core/libs/prx/libSceNpTrophy2/src/Context.cpp | 50 +- .../libs/prx/libSceNpTrophy2/src/GameInfo.cpp | 54 +- .../prx/libSceNpTrophy2/src/GroupInfo.cpp | 118 +++-- .../prx/libSceNpTrophy2/src/TitleTrophies.cpp | 140 +++++ .../prx/libSceNpTrophy2/src/TrophyInfo.cpp | 119 +++-- .../prx/libSceNpTrophy2/src/TrophyPackage.cpp | 481 ++++++++++++++++++ .../libSceNpTrophy2/tests/TrophyPackage.cpp | 283 +++++++++++ .../CMakeLists.txt | 12 + core/libs/tests/GuestNpTrophyData.cpp | 188 +++++++ docs/dev/debt/libSceNpTrophy2.md | 8 +- 13 files changed, 1520 insertions(+), 98 deletions(-) create mode 100644 core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp create mode 100644 core/libs/prx/libSceNpTrophy2/include/TrophyPackage.hpp create mode 100644 core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp create mode 100644 core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp create mode 100644 core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp create mode 100644 core/libs/tests/GuestNpTrophyData.cpp diff --git a/core/libs/prx/libSceNpTrophy2/CMakeLists.txt b/core/libs/prx/libSceNpTrophy2/CMakeLists.txt index 165feb0839..cd03e490cb 100644 --- a/core/libs/prx/libSceNpTrophy2/CMakeLists.txt +++ b/core/libs/prx/libSceNpTrophy2/CMakeLists.txt @@ -4,6 +4,8 @@ add_library(libSceNpTrophy2 SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/src/GameInfo.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/TrophyInfo.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GroupInfo.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/TrophyPackage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/TitleTrophies.cpp ) target_include_directories(libSceNpTrophy2 PRIVATE @@ -23,3 +25,16 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +add_test_executable(np_trophy2_package_tests EXCLUDE_FROM_ALL tests/TrophyPackage.cpp) +target_include_directories(np_trophy2_package_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(np_trophy2_package_tests PRIVATE libSceNpTrophy2 libc) +configure_windows_unwind(np_trophy2_package_tests) +if(BUILD_TESTING) + set_target_properties(np_trophy2_package_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME np_trophy2_package COMMAND np_trophy2_package_tests) + if(WIN32) + set_tests_properties(np_trophy2_package PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp b/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp new file mode 100644 index 0000000000..8caaa30647 --- /dev/null +++ b/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp @@ -0,0 +1,54 @@ +#ifndef CORE_LIBS_PRX_LIBSCENPTROPHY2_INCLUDE_TITLETROPHIES_HPP +#define CORE_LIBS_PRX_LIBSCENPTROPHY2_INCLUDE_TITLETROPHIES_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "prx/libSceNpTrophy2/include/TrophyPackage.hpp" + +namespace NpTrophy2 { + +inline constexpr std::uint64_t MaxUcpSize = std::uint64_t{256} << 20u; + +struct TitleTrophies { + std::optional package; +}; + +std::string PackagePath(std::uint32_t serviceLabel); +TitleTrophies LoadTitleTrophies(std::uint32_t serviceLabel, std::string_view language); +const TitleTrophies& Title(std::uint32_t serviceLabel); +const TrophyPackage* Package(std::uint32_t serviceLabel); +void CopyText(char* destination, std::size_t size, std::string_view text); + +struct ContextOwner { + std::int32_t userId = 0; + std::uint32_t serviceLabel = 0; +}; + +int CreateContext(std::int32_t userId, std::uint32_t serviceLabel); +void DestroyContext(int context); +std::optional FindContext(int context); + +std::optional ProgressTarget(const TrophyDefinition& trophy); +std::uint64_t TrophyProgress(const TrophyDefinition& trophy, std::optional stat, bool unlocked); + +struct TrophyCounts { + std::uint32_t trophies = 0; + std::uint32_t unlocked = 0; + std::array grades{}; + std::array unlockedGrades{}; + + std::uint32_t Percentage() const; +}; + +TrophyCounts CountTrophies(const TrophyPackage& package, std::optional group); + +} + +#endif diff --git a/core/libs/prx/libSceNpTrophy2/include/TrophyPackage.hpp b/core/libs/prx/libSceNpTrophy2/include/TrophyPackage.hpp new file mode 100644 index 0000000000..1e1e20bd4c --- /dev/null +++ b/core/libs/prx/libSceNpTrophy2/include/TrophyPackage.hpp @@ -0,0 +1,96 @@ +#ifndef CORE_LIBS_PRX_LIBSCENPTROPHY2_INCLUDE_TROPHYPACKAGE_HPP +#define CORE_LIBS_PRX_LIBSCENPTROPHY2_INCLUDE_TROPHYPACKAGE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NpTrophy2 { + +inline constexpr std::uint32_t UcpMagic = 0xB228C60Au; +inline constexpr std::uint32_t UcpVersion = 1; +inline constexpr std::uint64_t UcpHeaderSize = 0x40; +inline constexpr std::uint64_t UcpTocHeaderSize = 0x20; +inline constexpr std::uint64_t UcpEntrySize = 0x40; +inline constexpr std::size_t UcpNameSize = 0x20; +inline constexpr std::uint64_t UcpMaxEntries = 4096; + +using UcpFiles = std::map>; + +UcpFiles ReadUcp(std::string_view data); + +class Json { +public: + enum class Kind { Null, Bool, Number, String, Array, Object }; + + Kind kind = Kind::Null; + bool boolean = false; + std::string text; + std::vector items; + std::vector> members; + + const Json* Find(std::string_view key) const; + bool IsTrue() const; + std::optional Unsigned() const; + std::optional Id() const; +}; + +Json ParseJson(std::string_view text); + +inline constexpr std::int32_t GradePlatinum = 1; +inline constexpr std::int32_t GradeGold = 2; +inline constexpr std::int32_t GradeSilver = 3; +inline constexpr std::int32_t GradeBronze = 4; +inline constexpr std::int32_t BaseGroup = -1; +inline constexpr std::size_t MaxTrophies = 1000; +inline constexpr std::size_t MaxGroups = 50; + +enum class Comparator { None, GreaterEqual, Greater, LessEqual, Less }; + +struct UnlockCondition { + std::uint64_t statId = 0; + std::uint64_t target = 0; + Comparator comparator = Comparator::None; + + bool Reached(std::uint64_t value) const; +}; + +struct TrophyDefinition { + std::int32_t id = 0; + std::int32_t grade = GradeBronze; + std::int32_t groupId = BaseGroup; + bool hidden = false; + bool hasReward = false; + bool progressive = false; + std::optional platinumId; + std::optional statId; + std::optional target; + Comparator comparator = Comparator::None; + std::string name; + std::string detail; + std::string reward; + + std::optional Condition() const; +}; + +struct TrophyPackage { + std::string language; + std::string title; + std::map trophies; + std::map groups; + std::vector skipped; +}; + +TrophyPackage ParseTrophyPackage(std::string_view ucp, std::string_view language); + +std::string_view LanguageTag(int consoleLanguage); + +} + +#endif diff --git a/core/libs/prx/libSceNpTrophy2/src/Context.cpp b/core/libs/prx/libSceNpTrophy2/src/Context.cpp index 197f834f71..90887a7f4a 100644 --- a/core/libs/prx/libSceNpTrophy2/src/Context.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/Context.cpp @@ -1,26 +1,68 @@ #include #include +#include +#include #include #include #include "prx/libSceNpTrophy2/include/NpTrophy2.hpp" +#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" #include "prx/libc/include/General.hpp" +namespace NpTrophy2 { + +namespace { + +struct Contexts { + std::mutex mutex; + std::map owners; + int next = NP_TROPHY2_CONTEXT_DEFAULT; +}; + +Contexts& contexts() { + static Contexts table; + return table; +} + +} + +int CreateContext(std::int32_t userId, std::uint32_t serviceLabel) { + auto& table = contexts(); + std::lock_guard lock(table.mutex); + const int context = table.next++; + table.owners[context] = {userId, serviceLabel}; + return context; +} + +void DestroyContext(int context) { + auto& table = contexts(); + std::lock_guard lock(table.mutex); + table.owners.erase(context); +} + +std::optional FindContext(int context) { + auto& table = contexts(); + std::lock_guard lock(table.mutex); + const auto found = table.owners.find(context); + if (found == table.owners.end()) return std::nullopt; + return found->second; +} + +} + extern "C" { int APS5_VABI sceNpTrophy2CreateContext(int* context, int user_id, uint32_t service_label, uint64_t options) { if (context == nullptr) { APS5_INVALID_ARG_EX; } - (void)user_id; - (void)service_label; (void)options; - *context = NP_TROPHY2_CONTEXT_DEFAULT; + *context = NpTrophy2::CreateContext(user_id, service_label); return SCE_NP_TROPHY2_OK; } int APS5_VABI sceNpTrophy2DestroyContext(int context) { - (void)context; + NpTrophy2::DestroyContext(context); return SCE_NP_TROPHY2_OK; } diff --git a/core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp b/core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp index 0c5b320f6f..bea174aa49 100644 --- a/core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp @@ -1,9 +1,11 @@ #include #include +#include #include #include #include "prx/libSceNpTrophy2/include/NpTrophy2.hpp" +#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" #include "prx/libc/include/General.hpp" extern "C" { @@ -12,26 +14,50 @@ int APS5_VABI sceNpTrophy2GetGameInfo(int context, int handle, NpTrophy2GameDeta if (details == nullptr && data == nullptr) { APS5_INVALID_ARG_EX; } - (void)context; (void)handle; + const auto owner = NpTrophy2::FindContext(context); + const auto* package = owner ? NpTrophy2::Package(owner->serviceLabel) : nullptr; + if (package == nullptr) { + if (details != nullptr) { + std::memset(details, 0, sizeof(*details)); + details->num_groups = NP_TROPHY2_NUM_GROUPS; + details->num_trophies = NP_TROPHY2_NUM_TROPHIES; + details->num_platinum = NP_TROPHY2_NUM_PLATINUM; + details->num_gold = NP_TROPHY2_NUM_GOLD; + details->num_silver = NP_TROPHY2_NUM_SILVER; + details->num_bronze = NP_TROPHY2_NUM_BRONZE; + std::strncpy(details->title, NP_TROPHY2_GAME_TITLE, sizeof(details->title) - 1); + } + if (data != nullptr) { + std::memset(data, 0, sizeof(*data)); + data->unlocked_trophies = NP_TROPHY2_UNLOCKED_TROPHIES; + data->unlocked_platinum = NP_TROPHY2_UNLOCKED_PLATINUM; + data->unlocked_gold = NP_TROPHY2_UNLOCKED_GOLD; + data->unlocked_silver = NP_TROPHY2_UNLOCKED_SILVER; + data->unlocked_bronze = NP_TROPHY2_UNLOCKED_BRONZE; + data->progress_percentage = NP_TROPHY2_PROGRESS_PERCENTAGE; + } + return SCE_NP_TROPHY2_OK; + } + const auto counts = NpTrophy2::CountTrophies(*package, std::nullopt); if (details != nullptr) { std::memset(details, 0, sizeof(*details)); - details->num_groups = NP_TROPHY2_NUM_GROUPS; - details->num_trophies = NP_TROPHY2_NUM_TROPHIES; - details->num_platinum = NP_TROPHY2_NUM_PLATINUM; - details->num_gold = NP_TROPHY2_NUM_GOLD; - details->num_silver = NP_TROPHY2_NUM_SILVER; - details->num_bronze = NP_TROPHY2_NUM_BRONZE; - std::strncpy(details->title, NP_TROPHY2_GAME_TITLE, sizeof(details->title) - 1); + details->num_groups = static_cast(package->groups.size()); + details->num_trophies = counts.trophies; + details->num_platinum = counts.grades[NpTrophy2::GradePlatinum]; + details->num_gold = counts.grades[NpTrophy2::GradeGold]; + details->num_silver = counts.grades[NpTrophy2::GradeSilver]; + details->num_bronze = counts.grades[NpTrophy2::GradeBronze]; + NpTrophy2::CopyText(details->title, sizeof(details->title), package->title); } if (data != nullptr) { std::memset(data, 0, sizeof(*data)); - data->unlocked_trophies = NP_TROPHY2_UNLOCKED_TROPHIES; - data->unlocked_platinum = NP_TROPHY2_UNLOCKED_PLATINUM; - data->unlocked_gold = NP_TROPHY2_UNLOCKED_GOLD; - data->unlocked_silver = NP_TROPHY2_UNLOCKED_SILVER; - data->unlocked_bronze = NP_TROPHY2_UNLOCKED_BRONZE; - data->progress_percentage = NP_TROPHY2_PROGRESS_PERCENTAGE; + data->unlocked_trophies = counts.unlocked; + data->unlocked_platinum = counts.unlockedGrades[NpTrophy2::GradePlatinum]; + data->unlocked_gold = counts.unlockedGrades[NpTrophy2::GradeGold]; + data->unlocked_silver = counts.unlockedGrades[NpTrophy2::GradeSilver]; + data->unlocked_bronze = counts.unlockedGrades[NpTrophy2::GradeBronze]; + data->progress_percentage = counts.Percentage(); } return SCE_NP_TROPHY2_OK; } diff --git a/core/libs/prx/libSceNpTrophy2/src/GroupInfo.cpp b/core/libs/prx/libSceNpTrophy2/src/GroupInfo.cpp index 5e97582aa7..badd913d14 100644 --- a/core/libs/prx/libSceNpTrophy2/src/GroupInfo.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/GroupInfo.cpp @@ -1,41 +1,85 @@ +#include #include #include #include +#include #include #include #include "prx/libSceNpTrophy2/include/NpTrophy2.hpp" +#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" #include "prx/libc/include/General.hpp" +namespace { + +void fillPlaceholderDetails(std::int32_t groupId, NpTrophy2GroupDetails* details) { + std::memset(details, 0, sizeof(*details)); + details->group_id = groupId; + details->num_trophies = NP_TROPHY2_NUM_TROPHIES; + details->num_platinum = NP_TROPHY2_NUM_PLATINUM; + details->num_gold = NP_TROPHY2_NUM_GOLD; + details->num_silver = NP_TROPHY2_NUM_SILVER; + details->num_bronze = NP_TROPHY2_NUM_BRONZE; + std::strncpy(details->title, NP_TROPHY2_GROUP_TITLE, sizeof(details->title) - 1); +} + +void fillPlaceholderData(std::int32_t groupId, NpTrophy2GroupData* data) { + std::memset(data, 0, sizeof(*data)); + data->group_id = groupId; + data->unlocked_trophies = NP_TROPHY2_UNLOCKED_TROPHIES; + data->unlocked_platinum = NP_TROPHY2_UNLOCKED_PLATINUM; + data->unlocked_gold = NP_TROPHY2_UNLOCKED_GOLD; + data->unlocked_silver = NP_TROPHY2_UNLOCKED_SILVER; + data->unlocked_bronze = NP_TROPHY2_UNLOCKED_BRONZE; + data->progress_percentage = NP_TROPHY2_PROGRESS_PERCENTAGE; +} + +void fillGroup(const NpTrophy2::TrophyPackage& package, std::int32_t groupId, const std::string& title, NpTrophy2GroupDetails* details, NpTrophy2GroupData* data) { + const auto counts = NpTrophy2::CountTrophies(package, groupId); + if (details != nullptr) { + std::memset(details, 0, sizeof(*details)); + details->group_id = groupId; + details->num_trophies = counts.trophies; + details->num_platinum = counts.grades[NpTrophy2::GradePlatinum]; + details->num_gold = counts.grades[NpTrophy2::GradeGold]; + details->num_silver = counts.grades[NpTrophy2::GradeSilver]; + details->num_bronze = counts.grades[NpTrophy2::GradeBronze]; + NpTrophy2::CopyText(details->title, sizeof(details->title), title); + } + if (data != nullptr) { + std::memset(data, 0, sizeof(*data)); + data->group_id = groupId; + data->unlocked_trophies = counts.unlocked; + data->unlocked_platinum = counts.unlockedGrades[NpTrophy2::GradePlatinum]; + data->unlocked_gold = counts.unlockedGrades[NpTrophy2::GradeGold]; + data->unlocked_silver = counts.unlockedGrades[NpTrophy2::GradeSilver]; + data->unlocked_bronze = counts.unlockedGrades[NpTrophy2::GradeBronze]; + data->progress_percentage = counts.Percentage(); + } +} + +} + extern "C" { int APS5_VABI sceNpTrophy2GetGroupInfo(int context, int handle, int group_id, NpTrophy2GroupDetails* details, NpTrophy2GroupData* data) { if (details == nullptr && data == nullptr) { APS5_INVALID_ARG_EX; } - (void)context; (void)handle; - const std::int32_t normalized_group_id = (group_id < 0 ? NP_TROPHY2_GROUP_ID_BASE : group_id); - if (details != nullptr) { - std::memset(details, 0, sizeof(*details)); - details->group_id = normalized_group_id; - details->num_trophies = NP_TROPHY2_NUM_TROPHIES; - details->num_platinum = NP_TROPHY2_NUM_PLATINUM; - details->num_gold = NP_TROPHY2_NUM_GOLD; - details->num_silver = NP_TROPHY2_NUM_SILVER; - details->num_bronze = NP_TROPHY2_NUM_BRONZE; - std::strncpy(details->title, NP_TROPHY2_GROUP_TITLE, sizeof(details->title) - 1); + const auto owner = NpTrophy2::FindContext(context); + const auto* package = owner ? NpTrophy2::Package(owner->serviceLabel) : nullptr; + if (package == nullptr) { + const std::int32_t normalized_group_id = (group_id < 0 ? NP_TROPHY2_GROUP_ID_BASE : group_id); + if (details != nullptr) fillPlaceholderDetails(normalized_group_id, details); + if (data != nullptr) fillPlaceholderData(normalized_group_id, data); + return SCE_NP_TROPHY2_OK; } - if (data != nullptr) { - std::memset(data, 0, sizeof(*data)); - data->group_id = normalized_group_id; - data->unlocked_trophies = NP_TROPHY2_UNLOCKED_TROPHIES; - data->unlocked_platinum = NP_TROPHY2_UNLOCKED_PLATINUM; - data->unlocked_gold = NP_TROPHY2_UNLOCKED_GOLD; - data->unlocked_silver = NP_TROPHY2_UNLOCKED_SILVER; - data->unlocked_bronze = NP_TROPHY2_UNLOCKED_BRONZE; - data->progress_percentage = NP_TROPHY2_PROGRESS_PERCENTAGE; + const auto group = package->groups.find(group_id); + if (group == package->groups.end()) { + throw std::invalid_argument(std::string(__func__) + ": group " + std::to_string(group_id) + " is not in " + NpTrophy2::PackagePath(owner->serviceLabel)); } + fillGroup(*package, group->first, group->second, details, data); return SCE_NP_TROPHY2_OK; } @@ -43,29 +87,21 @@ int APS5_VABI sceNpTrophy2GetGroupInfoArray(int context, int handle, uint32_t of if (count == nullptr) { APS5_INVALID_ARG_EX; } - (void)context; (void)handle; - const uint32_t out_count = (offset == 0 && limit != 0 ? 1u : 0u); - *count = out_count; - if (out_count != 0 && details_array != nullptr) { - std::memset(details_array, 0, sizeof(*details_array)); - details_array->group_id = NP_TROPHY2_GROUP_ID_BASE; - details_array->num_trophies = NP_TROPHY2_NUM_TROPHIES; - details_array->num_platinum = NP_TROPHY2_NUM_PLATINUM; - details_array->num_gold = NP_TROPHY2_NUM_GOLD; - details_array->num_silver = NP_TROPHY2_NUM_SILVER; - details_array->num_bronze = NP_TROPHY2_NUM_BRONZE; - std::strncpy(details_array->title, NP_TROPHY2_GROUP_TITLE, sizeof(details_array->title) - 1); + const auto owner = NpTrophy2::FindContext(context); + const auto* package = owner ? NpTrophy2::Package(owner->serviceLabel) : nullptr; + if (package == nullptr) { + const uint32_t out_count = (offset == 0 && limit != 0 ? 1u : 0u); + *count = out_count; + if (out_count != 0 && details_array != nullptr) fillPlaceholderDetails(NP_TROPHY2_GROUP_ID_BASE, details_array); + if (out_count != 0 && data_array != nullptr) fillPlaceholderData(NP_TROPHY2_GROUP_ID_BASE, data_array); + return SCE_NP_TROPHY2_OK; } - if (out_count != 0 && data_array != nullptr) { - std::memset(data_array, 0, sizeof(*data_array)); - data_array->group_id = NP_TROPHY2_GROUP_ID_BASE; - data_array->unlocked_trophies = NP_TROPHY2_UNLOCKED_TROPHIES; - data_array->unlocked_platinum = NP_TROPHY2_UNLOCKED_PLATINUM; - data_array->unlocked_gold = NP_TROPHY2_UNLOCKED_GOLD; - data_array->unlocked_silver = NP_TROPHY2_UNLOCKED_SILVER; - data_array->unlocked_bronze = NP_TROPHY2_UNLOCKED_BRONZE; - data_array->progress_percentage = NP_TROPHY2_PROGRESS_PERCENTAGE; + *count = 0; + auto group = package->groups.begin(); + std::advance(group, std::min(offset, package->groups.size())); + for (; group != package->groups.end() && *count < limit; ++group, ++*count) { + fillGroup(*package, group->first, group->second, details_array != nullptr ? &details_array[*count] : nullptr, data_array != nullptr ? &data_array[*count] : nullptr); } return SCE_NP_TROPHY2_OK; } diff --git a/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp b/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp new file mode 100644 index 0000000000..885a0b4206 --- /dev/null +++ b/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp @@ -0,0 +1,140 @@ +#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" +#include "prx/libSceSystemService/SystemService.hpp" +#include "prx/libc/include/General.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NpTrophy2 { + +namespace { + +bool tracing() { + static const bool trace = std::getenv("APS5_TRACE_UDS") != nullptr; + return trace; +} + +std::string labelled(const char* format, std::uint32_t serviceLabel) { + char path[64]; + std::snprintf(path, sizeof(path), format, static_cast(serviceLabel)); + return path; +} + +std::optional readTitleFile(const std::string& guestPath) { + const auto path = ResolvePath_nid_no_patch(guestPath.c_str()); + std::error_code error; + const bool exists = std::filesystem::exists(path, error); + if (error) throw std::filesystem::filesystem_error("cannot check the file", path, error); + if (!exists) return std::nullopt; + const auto size = std::filesystem::file_size(path); + if (size > MaxUcpSize) throw std::runtime_error("it has " + std::to_string(size) + " bytes, more than " + std::to_string(MaxUcpSize)); + std::string bytes(static_cast(size), '\0'); + std::ifstream file(path, std::ios::binary); + if (!file.read(bytes.data(), static_cast(bytes.size()))) throw std::runtime_error("cannot read " + path.string()); + return bytes; +} + +template +std::optional loadTitleFile(const std::string& guestPath, const char* without, TParse parse) { + std::optional bytes; + try { + bytes = readTitleFile(guestPath); + } catch (const std::runtime_error& error) { + std::fprintf(stderr, "[trophy] %s is not used, so %s: %s\n", guestPath.c_str(), without, error.what()); + return std::nullopt; + } + if (!bytes) { + if (tracing()) std::fprintf(stderr, "[uds] %s does not exist, so %s\n", guestPath.c_str(), without); + return std::nullopt; + } + try { + return parse(*bytes); + } catch (const std::runtime_error& error) { + std::fprintf(stderr, "[trophy] %s is malformed, so %s: %s\n", guestPath.c_str(), without, error.what()); + return std::nullopt; + } +} + +std::string counted(std::size_t count, const char* one, const char* many) { + return std::to_string(count) + " " + (count == 1 ? one : many); +} + +} + +std::string PackagePath(std::uint32_t serviceLabel) { + return labelled("/app0/sce_sys/trophy2/trophy%02u.ucp", serviceLabel); +} + +TitleTrophies LoadTitleTrophies(std::uint32_t serviceLabel, std::string_view language) { + TitleTrophies title; + const auto packagePath = PackagePath(serviceLabel); + title.package = loadTitleFile(packagePath, "trophy queries report placeholders and stats unlock no trophies", + [language](std::string_view bytes) { return ParseTrophyPackage(bytes, language); }); + if (title.package) { + std::fprintf(stderr, "[trophy] %s: %s, %s, texts in %s\n", packagePath.c_str(), counted(title.package->trophies.size(), "trophy", "trophies").c_str(), + counted(title.package->groups.size(), "group", "groups").c_str(), title.package->language.c_str()); + for (const auto& skipped : title.package->skipped) std::fprintf(stderr, "[trophy] %s: %s\n", packagePath.c_str(), skipped.c_str()); + } + return title; +} + +const TitleTrophies& Title(std::uint32_t serviceLabel) { + static std::mutex mutex; + static std::map> titles; + std::lock_guard lock(mutex); + auto& title = titles[serviceLabel]; + if (!title) title = std::make_unique(LoadTitleTrophies(serviceLabel, LanguageTag(SystemServiceConsoleLanguage("libSceNpTrophy2")))); + return *title; +} + +const TrophyPackage* Package(std::uint32_t serviceLabel) { + const auto& title = Title(serviceLabel); + return title.package ? &*title.package : nullptr; +} + +void CopyText(char* destination, std::size_t size, std::string_view text) { + if (size == 0) throw std::invalid_argument("text cannot be copied into an empty buffer"); + auto length = std::min(text.size(), size - 1); + while (length > 0 && length < text.size() && (static_cast(text[length]) & 0xC0u) == 0x80u) --length; + std::memcpy(destination, text.data(), length); + destination[length] = '\0'; +} + +std::optional ProgressTarget(const TrophyDefinition& trophy) { + if (!trophy.progressive) return std::nullopt; + return trophy.target; +} + +std::uint64_t TrophyProgress(const TrophyDefinition& trophy, std::optional stat, bool unlocked) { + const auto target = ProgressTarget(trophy); + if (!target) throw std::invalid_argument("trophy " + std::to_string(trophy.id) + " is not progressive or has no target"); + if (unlocked) return *target; + return std::min(stat.value_or(0), *target); +} + +TrophyCounts CountTrophies(const TrophyPackage& package, std::optional group) { + TrophyCounts counts; + for (const auto& [id, trophy] : package.trophies) { + if (group && trophy.groupId != *group) continue; + ++counts.trophies; + ++counts.grades[static_cast(trophy.grade)]; + } + return counts; +} + +std::uint32_t TrophyCounts::Percentage() const { + const auto points = [](const std::array& counts) { return counts[GradeGold] * 6u + counts[GradeSilver] * 2u + counts[GradeBronze]; }; + const auto total = points(grades); + return total == 0 ? 0 : points(unlockedGrades) * 100u / total; +} + +} diff --git a/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp b/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp index 5090940b58..001ba17a97 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp @@ -1,39 +1,88 @@ +#include #include #include #include +#include #include #include #include "prx/libSceNpTrophy2/include/NpTrophy2.hpp" +#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" #include "prx/libc/include/General.hpp" +namespace { + +void fillPlaceholderDetails(int trophyId, NpTrophy2Details* details) { + std::memset(details, 0, sizeof(*details)); + details->trophy_id = trophyId; + details->trophy_grade = NP_TROPHY2_TROPHY_GRADE_BRONZE; + details->group_id = NP_TROPHY2_GROUP_ID_BASE; + details->hidden = false; + details->has_reward = false; + details->target.type = NP_TROPHY2_PROGRESS_TYPE_NONE; + details->target.value = NP_TROPHY2_PROGRESS_VALUE_NONE; + std::strncpy(details->name, NP_TROPHY2_TROPHY_NAME, sizeof(details->name) - 1); + std::strncpy(details->description, NP_TROPHY2_TROPHY_DESCRIPTION, sizeof(details->description) - 1); + std::strncpy(details->reward, NP_TROPHY2_TROPHY_REWARD, sizeof(details->reward) - 1); +} + +void fillPlaceholderData(int trophyId, NpTrophy2Data* data) { + std::memset(data, 0, sizeof(*data)); + data->trophy_id = trophyId; + data->unlocked = false; + data->progress.type = NP_TROPHY2_PROGRESS_TYPE_NONE; + data->progress.value = NP_TROPHY2_PROGRESS_VALUE_NONE; + data->timestamp_tick = 0; +} + +void fillDetails(const NpTrophy2::TrophyDefinition& trophy, NpTrophy2Details* details) { + std::memset(details, 0, sizeof(*details)); + details->trophy_id = trophy.id; + details->trophy_grade = trophy.grade; + details->group_id = trophy.groupId; + details->hidden = trophy.hidden; + details->has_reward = trophy.hasReward; + if (const auto target = NpTrophy2::ProgressTarget(trophy)) { + details->target.type = NP_TROPHY2_PROGRESS_TYPE_VALUE; + details->target.value = *target; + } + NpTrophy2::CopyText(details->name, sizeof(details->name), trophy.name); + NpTrophy2::CopyText(details->description, sizeof(details->description), trophy.detail); + NpTrophy2::CopyText(details->reward, sizeof(details->reward), trophy.reward); +} + +void fillData(const NpTrophy2::TrophyDefinition& trophy, NpTrophy2Data* data) { + std::memset(data, 0, sizeof(*data)); + data->trophy_id = trophy.id; + data->unlocked = false; + data->timestamp_tick = 0; + if (!NpTrophy2::ProgressTarget(trophy)) return; + data->progress.type = NP_TROPHY2_PROGRESS_TYPE_VALUE; + data->progress.value = NpTrophy2::TrophyProgress(trophy, std::nullopt, data->unlocked); +} + +} + extern "C" { int APS5_VABI sceNpTrophy2GetTrophyInfo(int context, int handle, int trophy_id, NpTrophy2Details* details, NpTrophy2Data* data) { if (details == nullptr && data == nullptr) { APS5_INVALID_ARG_EX; } - (void)context; (void)handle; - if (details != nullptr) { - std::memset(details, 0, sizeof(*details)); - details->trophy_id = trophy_id; - details->trophy_grade = NP_TROPHY2_TROPHY_GRADE_BRONZE; - details->group_id = NP_TROPHY2_GROUP_ID_BASE; - details->hidden = false; - details->has_reward = false; - details->target.value = NP_TROPHY2_PROGRESS_VALUE_NONE; - std::strncpy(details->name, NP_TROPHY2_TROPHY_NAME, sizeof(details->name) - 1); - std::strncpy(details->description, NP_TROPHY2_TROPHY_DESCRIPTION, sizeof(details->description) - 1); - std::strncpy(details->reward, NP_TROPHY2_TROPHY_REWARD, sizeof(details->reward) - 1); + const auto owner = NpTrophy2::FindContext(context); + const auto* package = owner ? NpTrophy2::Package(owner->serviceLabel) : nullptr; + if (package == nullptr) { + if (details != nullptr) fillPlaceholderDetails(trophy_id, details); + if (data != nullptr) fillPlaceholderData(trophy_id, data); + return SCE_NP_TROPHY2_OK; } - if (data != nullptr) { - std::memset(data, 0, sizeof(*data)); - data->trophy_id = trophy_id; - data->unlocked = false; - data->progress.value = NP_TROPHY2_PROGRESS_VALUE_NONE; - data->timestamp_tick = 0; + const auto trophy = package->trophies.find(trophy_id); + if (trophy == package->trophies.end()) { + throw std::invalid_argument(std::string(__func__) + ": trophy " + std::to_string(trophy_id) + " is not in " + NpTrophy2::PackagePath(owner->serviceLabel)); } + if (details != nullptr) fillDetails(trophy->second, details); + if (data != nullptr) fillData(trophy->second, data); return SCE_NP_TROPHY2_OK; } @@ -41,28 +90,22 @@ int APS5_VABI sceNpTrophy2GetTrophyInfoArray(int context, int handle, uint32_t o if (count == nullptr) { APS5_INVALID_ARG_EX; } - (void)context; (void)handle; - const uint32_t out_count = (offset == 0 && limit != 0 ? 1u : 0u); - *count = out_count; - if (out_count != 0 && details_array != nullptr) { - std::memset(details_array, 0, sizeof(*details_array)); - details_array->trophy_id = NP_TROPHY2_TROPHY_ID_DEFAULT; - details_array->trophy_grade = NP_TROPHY2_TROPHY_GRADE_BRONZE; - details_array->group_id = NP_TROPHY2_GROUP_ID_BASE; - details_array->hidden = false; - details_array->has_reward = false; - details_array->target.value = NP_TROPHY2_PROGRESS_VALUE_NONE; - std::strncpy(details_array->name, NP_TROPHY2_TROPHY_NAME, sizeof(details_array->name) - 1); - std::strncpy(details_array->description, NP_TROPHY2_TROPHY_DESCRIPTION, sizeof(details_array->description) - 1); - std::strncpy(details_array->reward, NP_TROPHY2_TROPHY_REWARD, sizeof(details_array->reward) - 1); + const auto owner = NpTrophy2::FindContext(context); + const auto* package = owner ? NpTrophy2::Package(owner->serviceLabel) : nullptr; + if (package == nullptr) { + const uint32_t out_count = (offset == 0 && limit != 0 ? 1u : 0u); + *count = out_count; + if (out_count != 0 && details_array != nullptr) fillPlaceholderDetails(NP_TROPHY2_TROPHY_ID_DEFAULT, details_array); + if (out_count != 0 && data_array != nullptr) fillPlaceholderData(NP_TROPHY2_TROPHY_ID_DEFAULT, data_array); + return SCE_NP_TROPHY2_OK; } - if (out_count != 0 && data_array != nullptr) { - std::memset(data_array, 0, sizeof(*data_array)); - data_array->trophy_id = NP_TROPHY2_TROPHY_ID_DEFAULT; - data_array->unlocked = false; - data_array->progress.value = NP_TROPHY2_PROGRESS_VALUE_NONE; - data_array->timestamp_tick = 0; + *count = 0; + auto trophy = package->trophies.begin(); + std::advance(trophy, std::min(offset, package->trophies.size())); + for (; trophy != package->trophies.end() && *count < limit; ++trophy, ++*count) { + if (details_array != nullptr) fillDetails(trophy->second, &details_array[*count]); + if (data_array != nullptr) fillData(trophy->second, &data_array[*count]); } return SCE_NP_TROPHY2_OK; } diff --git a/core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp b/core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp new file mode 100644 index 0000000000..01d0bc8063 --- /dev/null +++ b/core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp @@ -0,0 +1,481 @@ +#include "prx/libSceNpTrophy2/include/TrophyPackage.hpp" + +#include +#include +#include +#include +#include +#include + +namespace NpTrophy2 { + +namespace { + +constexpr int MaxJsonDepth = 64; +constexpr std::size_t MaxJsonValues = std::size_t{1} << 20u; + +std::uint64_t bigEndian(std::string_view data, std::uint64_t offset, std::size_t size) { + std::uint64_t value = 0; + for (std::size_t i = 0; i < size; ++i) value = (value << 8u) | static_cast(data[offset + i]); + return value; +} + +std::string hex(std::uint64_t value) { + char text[24]; + std::snprintf(text, sizeof(text), "0x%llx", static_cast(value)); + return text; +} + +void appendUtf8(std::string& out, std::uint32_t code) { + if (code < 0x80) { + out += static_cast(code); + } else if (code < 0x800) { + out += static_cast(0xC0 | (code >> 6)); + out += static_cast(0x80 | (code & 0x3F)); + } else if (code < 0x10000) { + out += static_cast(0xE0 | (code >> 12)); + out += static_cast(0x80 | ((code >> 6) & 0x3F)); + out += static_cast(0x80 | (code & 0x3F)); + } else { + out += static_cast(0xF0 | (code >> 18)); + out += static_cast(0x80 | ((code >> 12) & 0x3F)); + out += static_cast(0x80 | ((code >> 6) & 0x3F)); + out += static_cast(0x80 | (code & 0x3F)); + } +} + +class JsonReader { +public: + explicit JsonReader(std::string_view text) : text(text) {} + + Json Document() { + if (text.starts_with("\xEF\xBB\xBF")) position = 3; + auto document = readValue(0); + skipSpace(); + if (position != text.size()) fail("text follows the document"); + return document; + } + +private: + [[noreturn]] void fail(const std::string& problem) const { + throw std::runtime_error(problem + " at byte " + std::to_string(position)); + } + + void skipSpace() { + while (position < text.size() && (text[position] == ' ' || text[position] == '\t' || text[position] == '\n' || text[position] == '\r')) ++position; + } + + bool take(char expected) { + if (position >= text.size() || text[position] != expected) return false; + ++position; + return true; + } + + void expect(std::string_view word) { + if (text.substr(position, word.size()) != word) fail("expected " + std::string(word)); + position += word.size(); + } + + Json readValue(int depth) { + if (depth >= MaxJsonDepth) fail("values nest deeper than " + std::to_string(MaxJsonDepth) + " levels"); + if (++values > MaxJsonValues) fail("the document holds more than " + std::to_string(MaxJsonValues) + " values"); + skipSpace(); + if (position >= text.size()) fail("the document ends before a value"); + Json result; + switch (text[position]) { + case '{': + result.kind = Json::Kind::Object; + readObject(result, depth); + break; + case '[': + result.kind = Json::Kind::Array; + readArray(result, depth); + break; + case '"': + result.kind = Json::Kind::String; + result.text = readString(); + break; + case 't': + expect("true"); + result.kind = Json::Kind::Bool; + result.boolean = true; + break; + case 'f': + expect("false"); + result.kind = Json::Kind::Bool; + break; + case 'n': + expect("null"); + break; + default: + result.kind = Json::Kind::Number; + result.text = readNumber(); + break; + } + return result; + } + + void readObject(Json& result, int depth) { + ++position; + skipSpace(); + if (take('}')) return; + while (true) { + skipSpace(); + if (position >= text.size() || text[position] != '"') fail("expected a member name"); + auto key = readString(); + skipSpace(); + if (!take(':')) fail("expected ':'"); + result.members.emplace_back(std::move(key), readValue(depth + 1)); + skipSpace(); + if (take('}')) return; + if (!take(',')) fail("expected ',' or '}'"); + } + } + + void readArray(Json& result, int depth) { + ++position; + skipSpace(); + if (take(']')) return; + while (true) { + result.items.push_back(readValue(depth + 1)); + skipSpace(); + if (take(']')) return; + if (!take(',')) fail("expected ',' or ']'"); + } + } + + std::uint32_t readHexDigits() { + if (text.size() - position < 4) fail("a \\u escape is cut short"); + std::uint32_t code = 0; + const auto* end = text.data() + position + 4; + const auto [stop, error] = std::from_chars(text.data() + position, end, code, 16); + if (error != std::errc{} || stop != end) fail("a \\u escape does not have 4 hex digits"); + position += 4; + return code; + } + + std::string readString() { + ++position; + std::string out; + while (true) { + if (position >= text.size()) fail("a string is not closed"); + const char character = text[position++]; + if (character == '"') return out; + if (static_cast(character) < 0x20) fail("a string holds a control character"); + if (character != '\\') { + out += character; + continue; + } + if (position >= text.size()) fail("a string is not closed"); + switch (text[position++]) { + case '"': out += '"'; break; + case '\\': out += '\\'; break; + case '/': out += '/'; break; + case 'b': out += '\b'; break; + case 'f': out += '\f'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'u': { + auto code = readHexDigits(); + if (code >= 0xDC00 && code < 0xE000) fail("a low surrogate has no high surrogate"); + if (code >= 0xD800 && code < 0xDC00) { + if (text.substr(position, 2) != "\\u") fail("a high surrogate has no low surrogate"); + position += 2; + const auto low = readHexDigits(); + if (low < 0xDC00 || low >= 0xE000) fail("a high surrogate has no low surrogate"); + code = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); + } + appendUtf8(out, code); + break; + } + default: + fail("a string has an unknown escape"); + } + } + } + + bool skipDigits() { + const auto start = position; + while (position < text.size() && text[position] >= '0' && text[position] <= '9') ++position; + return position != start; + } + + std::string readNumber() { + const auto start = position; + take('-'); + if (!take('0') && !skipDigits()) fail("expected a value"); + if (take('.') && !skipDigits()) fail("a number has no digits after '.'"); + if (take('e') || take('E')) { + if (!take('+')) take('-'); + if (!skipDigits()) fail("a number has no exponent digits"); + } + return std::string(text.substr(start, position - start)); + } + + std::string_view text; + std::size_t position = 0; + std::size_t values = 0; +}; + +std::string describe(const Json& value) { + switch (value.kind) { + case Json::Kind::Null: return "null"; + case Json::Kind::Bool: return value.boolean ? "true" : "false"; + case Json::Kind::Number: return value.text; + case Json::Kind::String: return "\"" + value.text + "\""; + case Json::Kind::Array: return "an array"; + case Json::Kind::Object: return "an object"; + } + throw std::logic_error("a JSON value has an unknown kind"); +} + +std::string kindName(Json::Kind kind) { + switch (kind) { + case Json::Kind::Null: return "null"; + case Json::Kind::Bool: return "boolean"; + case Json::Kind::Number: return "number"; + case Json::Kind::String: return "string"; + case Json::Kind::Array: return "array"; + case Json::Kind::Object: return "object"; + } + throw std::logic_error("a JSON value has an unknown kind"); +} + +const Json& member(const Json& object, std::string_view key, Json::Kind kind, const std::string& where) { + const auto* found = object.Find(key); + if (found == nullptr || found->kind != kind) throw std::runtime_error(where + " has no " + std::string(key) + " " + kindName(kind)); + return *found; +} + +Json document(const UcpFiles& files, const std::string& name) { + const auto found = files.find(name); + if (found == files.end()) throw std::runtime_error("it has no " + name); + Json parsed; + try { + parsed = ParseJson(found->second); + } catch (const std::runtime_error& error) { + throw std::runtime_error(name + " is not JSON: " + error.what()); + } + if (parsed.kind != Json::Kind::Object) throw std::runtime_error(name + " is not a JSON object"); + return parsed; +} + +Comparator comparatorOf(const Json& value) { + if (value.kind != Json::Kind::String) return Comparator::None; + if (value.text == "ge") return Comparator::GreaterEqual; + if (value.text == "gt") return Comparator::Greater; + if (value.text == "le") return Comparator::LessEqual; + if (value.text == "lt") return Comparator::Less; + return Comparator::None; +} + +std::string join(const std::vector& parts, std::string_view separator) { + std::string out; + for (const auto& part : parts) { + if (!out.empty()) out += separator; + out += part; + } + return out; +} + +std::string alternatives(const std::vector& parts) { + std::string out; + for (std::size_t index = 0; index < parts.size(); ++index) { + if (index != 0) out += index + 1 == parts.size() ? " or " : ", "; + out += parts[index]; + } + return out; +} + +TrophyDefinition readTrophy(const Json& definition, std::size_t index, std::vector& skipped) { + const auto where = "tropconf.json trophies[" + std::to_string(index) + "]"; + if (definition.kind != Json::Kind::Object) throw std::runtime_error(where + " is not an object"); + const auto* id = definition.Find("id"); + const auto parsedId = id != nullptr ? id->Id() : std::nullopt; + if (!parsedId) throw std::runtime_error(where + " has no id from 0 to 2147483647"); + TrophyDefinition trophy; + trophy.id = *parsedId; + const auto name = "tropconf.json trophy " + std::to_string(trophy.id); + const auto* grade = definition.Find("grade"); + const auto letter = grade != nullptr && grade->kind == Json::Kind::String && grade->text.size() == 1 ? std::string_view("PGSB").find(grade->text[0]) : std::string_view::npos; + if (letter == std::string_view::npos) throw std::runtime_error(name + " has no grade P, G, S or B"); + trophy.grade = GradePlatinum + static_cast(letter); + if (const auto* group = definition.Find("groupId")) { + const auto groupId = group->Id(); + if (!groupId) throw std::runtime_error(name + " has groupId " + describe(*group) + ", not an id from 0 to 2147483647"); + trophy.groupId = *groupId; + } + if (const auto* platinum = definition.Find("platinumTrophyId")) trophy.platinumId = platinum->Id(); + const auto* hidden = definition.Find("hidden"); + trophy.hidden = hidden != nullptr && hidden->IsTrue(); + const auto* reward = definition.Find("hasReward"); + trophy.hasReward = reward != nullptr && reward->IsTrue(); + const auto* condition = definition.Find("unlockCondition"); + if (condition == nullptr || condition->kind != Json::Kind::Object) return trophy; + const auto* progressive = condition->Find("progressive"); + trophy.progressive = progressive != nullptr && progressive->IsTrue(); + if (const auto* stat = condition->Find("udsStatId")) { + trophy.statId = stat->Unsigned(); + if (!trophy.statId) throw std::runtime_error(name + " has udsStatId " + describe(*stat) + ", not an unsigned integer"); + } + std::vector problems; + if (const auto* target = condition->Find("targetValue")) { + trophy.target = target->Unsigned(); + if (!trophy.target) problems.push_back("its targetValue " + describe(*target) + " is not an unsigned integer"); + } else if (trophy.statId) { + problems.emplace_back("it has no targetValue"); + } + if (const auto* comparator = condition->Find("comparator")) { + trophy.comparator = comparatorOf(*comparator); + if (trophy.comparator == Comparator::None) problems.push_back("its comparator " + describe(*comparator) + " is not ge, gt, le or lt"); + } else if (trophy.statId) { + problems.emplace_back("it has no comparator"); + } + if (!problems.empty()) skipped.push_back(name + " never unlocks through stats: " + join(problems, ", ")); + return trophy; +} + +} + +const Json* Json::Find(std::string_view key) const { + for (auto member = members.rbegin(); member != members.rend(); ++member) { + if (member->first == key) return &member->second; + } + return nullptr; +} + +bool Json::IsTrue() const { + return kind == Kind::Bool && boolean; +} + +std::optional Json::Unsigned() const { + if (kind != Kind::Number && kind != Kind::String) return std::nullopt; + std::uint64_t value = 0; + const auto* end = text.data() + text.size(); + const auto [stop, error] = std::from_chars(text.data(), end, value); + if (error != std::errc{} || stop != end) return std::nullopt; + return value; +} + +std::optional Json::Id() const { + const auto value = Unsigned(); + if (!value || *value > static_cast(INT32_MAX)) return std::nullopt; + return static_cast(*value); +} + +Json ParseJson(std::string_view text) { + return JsonReader(text).Document(); +} + +UcpFiles ReadUcp(std::string_view data) { + if (data.size() < UcpHeaderSize) throw std::runtime_error("it has " + std::to_string(data.size()) + " bytes, fewer than the " + std::to_string(UcpHeaderSize) + "-byte header"); + const auto magic = bigEndian(data, 0, 4); + if (magic != UcpMagic) throw std::runtime_error("its magic is " + hex(magic) + ", not " + hex(UcpMagic)); + const auto version = bigEndian(data, 4, 4); + if (version != UcpVersion) throw std::runtime_error("its version is " + std::to_string(version) + ", not " + std::to_string(UcpVersion)); + const auto size = bigEndian(data, 8, 8); + if (size < UcpHeaderSize || size > data.size()) throw std::runtime_error("its size " + hex(size) + " is not from the header size " + hex(UcpHeaderSize) + " to the file size " + hex(data.size())); + const auto count = bigEndian(data, 0x10, 4); + if (count > UcpMaxEntries) throw std::runtime_error("it lists " + std::to_string(count) + " files, more than " + std::to_string(UcpMaxEntries)); + const auto toc = bigEndian(data, 0x14, 4); + if (toc > size || UcpTocHeaderSize + count * UcpEntrySize > size - toc) { + throw std::runtime_error("its table of " + std::to_string(count) + " files at " + hex(toc) + " ends past its size " + hex(size)); + } + UcpFiles files; + for (std::uint64_t index = 0; index < count; ++index) { + const auto entry = toc + UcpTocHeaderSize + index * UcpEntrySize; + const auto field = data.substr(entry, UcpNameSize); + const std::string name(field.substr(0, field.find('\0'))); + const auto offset = bigEndian(data, entry + UcpNameSize, 8); + const auto length = bigEndian(data, entry + UcpNameSize + 8, 8); + if (offset > size || length > size - offset) { + throw std::runtime_error("its file \"" + name + "\" at " + hex(offset) + " with " + hex(length) + " bytes ends past its size " + hex(size)); + } + files.try_emplace(name, data.substr(offset, length)); + } + return files; +} + +bool UnlockCondition::Reached(std::uint64_t value) const { + switch (comparator) { + case Comparator::GreaterEqual: return value >= target; + case Comparator::Greater: return value > target; + case Comparator::LessEqual: return value <= target; + case Comparator::Less: return value < target; + case Comparator::None: break; + } + throw std::logic_error("an unlock condition on stat " + std::to_string(statId) + " has no comparator"); +} + +std::optional TrophyDefinition::Condition() const { + if (grade == GradePlatinum || !statId || !target || comparator == Comparator::None) return std::nullopt; + return UnlockCondition{*statId, *target, comparator}; +} + +TrophyPackage ParseTrophyPackage(std::string_view ucp, std::string_view language) { + const auto files = ReadUcp(ucp); + const auto conf = document(files, "tropconf.json"); + const auto& trophies = member(conf, "trophies", Json::Kind::Array, "tropconf.json"); + if (trophies.items.size() > MaxTrophies) throw std::runtime_error("tropconf.json lists " + std::to_string(trophies.items.size()) + " trophies, more than " + std::to_string(MaxTrophies)); + const auto& defaultLanguage = member(conf, "defaultLanguage", Json::Kind::String, "tropconf.json"); + TrophyPackage package; + const std::array candidates{language, "en-US", defaultLanguage.text}; + std::vector tried; + for (std::size_t index = 0; index < candidates.size() && package.language.empty(); ++index) { + const auto candidate = candidates[index]; + const auto earlier = candidates.begin() + static_cast(index); + if (candidate.empty() || std::find(candidates.begin(), earlier, candidate) != earlier) continue; + if (files.contains("tropmeta_" + std::string(candidate) + ".json")) { + package.language = candidate; + } else { + tried.push_back(index + 1 == candidates.size() ? "its defaultLanguage \"" + std::string(candidate) + "\"" : std::string(candidate)); + } + } + if (package.language.empty()) throw std::runtime_error("it has no tropmeta_.json for " + alternatives(tried)); + const auto metaName = "tropmeta_" + package.language + ".json"; + const auto meta = document(files, metaName); + const auto& metadata = member(meta, "metadata", Json::Kind::Object, metaName); + const auto& titleMetadata = member(metadata, "titleMetadata", Json::Kind::Object, metaName + " metadata"); + package.title = member(titleMetadata, "name", Json::Kind::String, metaName + " titleMetadata").text; + const auto& trophyMetadata = member(metadata, "trophyMetadata", Json::Kind::Array, metaName + " metadata"); + for (std::size_t index = 0; index < trophies.items.size(); ++index) { + auto trophy = readTrophy(trophies.items[index], index, package.skipped); + package.groups.try_emplace(trophy.groupId, trophy.groupId == BaseGroup ? package.title : std::string()); + if (package.groups.size() > MaxGroups) throw std::runtime_error("tropconf.json uses more than " + std::to_string(MaxGroups) + " groups"); + const auto id = trophy.id; + if (!package.trophies.emplace(id, std::move(trophy)).second) throw std::runtime_error("tropconf.json lists trophy " + std::to_string(id) + " twice"); + } + if (const auto* groups = metadata.Find("groupMetadata"); groups != nullptr && groups->kind == Json::Kind::Array) { + for (const auto& group : groups->items) { + const auto* id = group.Find("id"); + const auto* name = group.Find("name"); + const auto groupId = id != nullptr ? id->Id() : std::nullopt; + if (!groupId || name == nullptr || name->kind != Json::Kind::String) continue; + if (const auto found = package.groups.find(*groupId); found != package.groups.end()) found->second = name->text; + } + } + for (const auto& text : trophyMetadata.items) { + const auto* id = text.Find("id"); + const auto trophyId = id != nullptr ? id->Id() : std::nullopt; + const auto found = trophyId ? package.trophies.find(*trophyId) : package.trophies.end(); + if (found == package.trophies.end()) continue; + for (const auto& [key, field] : {std::pair{"name", &found->second.name}, std::pair{"detail", &found->second.detail}, std::pair{"reward", &found->second.reward}}) { + if (const auto* value = text.Find(key); value != nullptr && value->kind == Json::Kind::String) *field = value->text; + } + } + return package; +} + +std::string_view LanguageTag(int consoleLanguage) { + static constexpr std::array tags = { + "ja-JP", "en-US", "fr-FR", "es-ES", "de-DE", "it-IT", "nl-NL", "pt-PT", "ru-RU", "ko-KR", "zh-Hant", + "zh-Hans", "fi-FI", "sv-SE", "da-DK", "no-NO", "pl-PL", "pt-BR", "en-GB", "tr-TR", "es-419", "ar-AE", + "fr-CA", "cs-CZ", "hu-HU", "el-GR", "ro-RO", "th-TH", "vi-VN", "id-ID", "uk-UA"}; + if (consoleLanguage < 0 || consoleLanguage >= static_cast(tags.size())) { + throw std::out_of_range("console language " + std::to_string(consoleLanguage) + " has no locale tag"); + } + return tags[static_cast(consoleLanguage)]; +} + +} diff --git a/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp b/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp new file mode 100644 index 0000000000..1611b0ba99 --- /dev/null +++ b/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp @@ -0,0 +1,283 @@ +#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" +#include "prx/libSceNpTrophy2/include/TrophyPackage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Files = std::vector>; + +void check(bool condition, const char* message) { + if (condition) return; + std::fprintf(stderr, "%s\n", message); + std::abort(); +} + +template +std::string failure(TAction action) { + try { + action(); + } catch (const std::runtime_error& error) { + return error.what(); + } + return {}; +} + +bool contains(const std::string& text, const char* part) { + return text.find(part) != std::string::npos; +} + +void putBigEndian(std::string& data, std::size_t offset, std::uint64_t value, std::size_t size) { + for (std::size_t i = 0; i < size; ++i) data[offset + i] = static_cast((value >> (8 * (size - 1 - i))) & 0xFF); +} + +std::string ucp(const Files& files) { + const std::size_t toc = 0x40; + std::size_t offset = toc + 0x20 + files.size() * 0x40; + std::size_t size = offset; + for (const auto& [name, contents] : files) size += contents.size(); + std::string data(size, '\0'); + putBigEndian(data, 0, 0xB228C60A, 4); + putBigEndian(data, 4, 1, 4); + putBigEndian(data, 8, size, 8); + putBigEndian(data, 0x10, files.size(), 4); + putBigEndian(data, 0x14, toc, 4); + std::size_t entry = toc + 0x20; + for (const auto& [name, contents] : files) { + data.replace(entry, name.size(), name); + putBigEndian(data, entry + 0x20, offset, 8); + putBigEndian(data, entry + 0x28, contents.size(), 8); + data.replace(offset, contents.size(), contents); + offset += contents.size(); + entry += 0x40; + } + return data; +} + +const char TropConf[] = R"({"defaultLanguage":"en-US","trophies":[ + {"id":"0","grade":"P","hidden":false}, + {"id":"1","grade":"G","platinumTrophyId":"0","unlockCondition":{"udsStatId":"10","comparator":"ge","targetValue":"10","progressive":false}}, + {"id":"2","grade":"S","platinumTrophyId":0,"unlockCondition":{"udsStatId":20,"comparator":"gt","targetValue":100}}, + {"id":3,"grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"30","comparator":"le","targetValue":"5"}}, + {"id":"4","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"30","comparator":"lt","targetValue":"3"}}, + {"id":"5","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"40","comparator":"ge","targetValue":"5","progressive":true}}, + {"id":"6","grade":"B","groupId":"1","hidden":true,"hasReward":true,"platinumTrophyId":"-1"}, + {"id":"7","grade":"B","groupId":"1","unlockCondition":{"udsStatId":"10","comparator":"eq","targetValue":"-1"}} +]})"; + +const char TropMetaEnglish[] = R"({"metadata":{"titleMetadata":{"name":"Synthetic Game"}, + "trophyMetadata":[{"id":"0","name":"All of them","detail":"Unlock every trophy"},{"id":"1","name":"Ten","detail":"Reach ten"}, + {"id":"6","name":"Secret","detail":"A hidden one","reward":"A hat"},{"id":"99","name":"Not a trophy"}], + "groupMetadata":[{"id":"1","name":"Expansion"},{"id":"9","name":"Unused"}]}})"; + +const char TropMetaFrench[] = R"({"metadata":{"titleMetadata":{"name":"Jeu synth\u00e9tique"},"trophyMetadata":[{"id":"1","name":"Dix"}]}})"; + +std::string trophyPackage() { + return ucp({{"tropconf.json", TropConf}, {"tropmeta_en-US.json", TropMetaEnglish}, {"tropmeta_fr-FR.json", TropMetaFrench}, {"trop0001.png", "not an image"}}); +} + +void testContainer() { + const auto data = ucp({{"a.json", "{}"}, {"b.bin", std::string("\0\1\2", 3)}, {"a.json", "second"}}); + const auto files = NpTrophy2::ReadUcp(data); + check(files.size() == 2 && files.at("a.json") == "{}" && files.at("b.bin") == std::string_view("\0\1\2", 3), "the container files were not read, or a repeated name replaced the first"); + check(NpTrophy2::ReadUcp(ucp({})).empty(), "an empty container was not read"); + const std::string longName(0x20, 'n'); + const auto named = NpTrophy2::ReadUcp(ucp({{longName, "x"}})); + check(named.size() == 1 && named.begin()->first == longName, "a name that fills its 32 bytes was not read whole"); + auto padded = data + "trailing"; + check(NpTrophy2::ReadUcp(padded).size() == 2, "bytes after the container size were not ignored"); + + for (std::size_t length = 0; length < data.size(); ++length) { + check(!failure([&] { NpTrophy2::ReadUcp(std::string_view(data).substr(0, length)); }).empty(), "a truncated container was read"); + } + check(contains(failure([&] { NpTrophy2::ReadUcp(std::string_view(data).substr(0, 0x3F)); }), "fewer than the 64-byte header"), "a short header was not named"); + auto corrupt = data; + corrupt[0] = 0; + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "magic is 0x28c60a"), "a wrong magic was not named"); + corrupt = data; + putBigEndian(corrupt, 4, 2, 4); + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "version is 2"), "a wrong version was not named"); + corrupt = data; + putBigEndian(corrupt, 8, 0x3F, 8); + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "its size 0x3f"), "a size inside the header was accepted"); + corrupt = data; + putBigEndian(corrupt, 8, data.size() + 1, 8); + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "is not from the header size"), "a size past the file was accepted"); + corrupt = data; + putBigEndian(corrupt, 0x10, 4097, 4); + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "more than 4096"), "too many files were accepted"); + corrupt = data; + putBigEndian(corrupt, 0x10, 40, 4); + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "table of 40 files"), "a table past the size was accepted"); + corrupt = data; + putBigEndian(corrupt, 0x14, data.size() + 1, 4); + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "ends past its size"), "a table offset past the size was accepted"); + corrupt = data; + putBigEndian(corrupt, 0x60 + 0x20, data.size() + 1, 8); + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "its file \"a.json\""), "a file offset past the size was accepted"); + corrupt = data; + putBigEndian(corrupt, 0x60 + 0x28, ~std::uint64_t{0}, 8); + check(contains(failure([&] { NpTrophy2::ReadUcp(corrupt); }), "ends past its size"), "a file length past the size was accepted"); +} + +void testJson() { + const auto value = NpTrophy2::ParseJson("\xEF\xBB\xBF {\"a\": [1, -2.5e+3, \"x\\u00e9\\ud83d\\ude00\\n\", true, false, null, {}], \"a\": 7, \"b\": \"12\"} "); + check(value.kind == NpTrophy2::Json::Kind::Object && value.members.size() == 3, "an object was not parsed"); + check(value.Find("a") != nullptr && value.Find("a")->text == "7", "a repeated key did not keep its last value"); + const auto& list = value.members[0].second.items; + check(list.size() == 7 && list[0].text == "1" && list[1].text == "-2.5e+3" && list[2].text == "x\xC3\xA9\xF0\x9F\x98\x80\n", "array values or escapes were not parsed"); + check(list[3].IsTrue() && !list[4].IsTrue() && list[5].kind == NpTrophy2::Json::Kind::Null && list[6].kind == NpTrophy2::Json::Kind::Object, "literals were not parsed"); + check(value.Find("b")->Unsigned() == std::optional(12) && list[0].Unsigned() == std::optional(1), "numbers and numeric strings are not unsigned"); + check(!list[1].Unsigned() && !list[3].Unsigned() && !NpTrophy2::ParseJson("\"\"").Unsigned() && !NpTrophy2::ParseJson("\" 1\"").Unsigned(), "a value that is not an unsigned integer was read as one"); + check(NpTrophy2::ParseJson("18446744073709551615").Unsigned() == std::optional(UINT64_MAX) && !NpTrophy2::ParseJson("18446744073709551616").Unsigned(), "the unsigned range is wrong"); + check(NpTrophy2::ParseJson("2147483647").Id() == std::optional(INT32_MAX) && !NpTrophy2::ParseJson("2147483648").Id() && !NpTrophy2::ParseJson("-1").Id(), "the id range is wrong"); + std::string deep(64, '['); + deep += std::string(64, ']'); + check(NpTrophy2::ParseJson(deep).kind == NpTrophy2::Json::Kind::Array, "64 levels of nesting were refused"); + for (const char* text : {"", "{", "[1,]", "{\"a\":1,}", "{\"a\" 1}", "{a:1}", "01", "1.", "-", "1e", "+1", "\"\\x\"", "\"\\u12g4\"", "\"\\ud800\"", "\"\\udc00\"", + "\"\\ud800\\u0041\"", "\"a\nb\"", "\"open", "tru", "nul", "{} {}", "[1] x"}) { + check(!failure([text] { NpTrophy2::ParseJson(text); }).empty(), "malformed JSON was parsed"); + } + const auto deeper = "[" + deep + "]"; + check(contains(failure([&deeper] { NpTrophy2::ParseJson(deeper); }), "deeper than 64"), "deep nesting was not refused"); + std::string most = "["; + for (std::size_t value = 2; value < 1048576; ++value) most += "0,"; + most += "0]"; + check(NpTrophy2::ParseJson(most).items.size() == 1048575, "a document of 1048576 values was refused"); + const auto more = "[0," + most.substr(1); + check(contains(failure([&more] { NpTrophy2::ParseJson(more); }), "holds more than 1048576 values"), "a document of more than 1048576 values was parsed"); +} + +void testPackage() { + const auto package = NpTrophy2::ParseTrophyPackage(trophyPackage(), "en-US"); + check(package.language == "en-US" && package.title == "Synthetic Game" && package.trophies.size() == 8, "the package was not read"); + const auto& platinum = package.trophies.at(0); + check(platinum.grade == NpTrophy2::GradePlatinum && platinum.name == "All of them" && platinum.detail == "Unlock every trophy" && !platinum.Condition(), "the platinum was not read"); + const auto& gold = package.trophies.at(1); + check(gold.grade == NpTrophy2::GradeGold && gold.name == "Ten" && gold.platinumId == std::optional(0) && gold.groupId == NpTrophy2::BaseGroup, "trophy 1 was not read"); + const auto condition = gold.Condition(); + check(condition && condition->statId == 10 && condition->target == 10 && condition->comparator == NpTrophy2::Comparator::GreaterEqual && !gold.progressive, "trophy 1's condition was not read"); + check(package.trophies.at(2).grade == NpTrophy2::GradeSilver && package.trophies.at(3).grade == NpTrophy2::GradeBronze, "grades were not read"); + check(package.trophies.at(2).Condition()->comparator == NpTrophy2::Comparator::Greater && package.trophies.at(3).Condition()->comparator == NpTrophy2::Comparator::LessEqual && + package.trophies.at(4).Condition()->comparator == NpTrophy2::Comparator::Less, + "comparators were not read"); + const auto& progressive = package.trophies.at(5); + check(progressive.progressive && NpTrophy2::ProgressTarget(progressive) == std::optional(5) && !NpTrophy2::ProgressTarget(gold), "the progressive target was not read"); + const auto& hidden = package.trophies.at(6); + check(hidden.hidden && hidden.hasReward && hidden.groupId == 1 && hidden.name == "Secret" && hidden.reward == "A hat" && !hidden.platinumId, "trophy 6 was not read"); + check(!package.trophies.at(7).Condition() && package.trophies.at(7).name.empty(), "an unusable condition was kept"); + check(package.skipped.size() == 1 && contains(package.skipped[0], "trophy 7 never unlocks through stats") && contains(package.skipped[0], "\"-1\"") && contains(package.skipped[0], "\"eq\""), + "the unusable condition was not reported"); + check(package.groups.size() == 2 && package.groups.at(NpTrophy2::BaseGroup) == "Synthetic Game" && package.groups.at(1) == "Expansion", "groups were not read"); + + const auto french = NpTrophy2::ParseTrophyPackage(trophyPackage(), "fr-FR"); + check(french.language == "fr-FR" && french.title == "Jeu synth\xC3\xA9tique" && french.trophies.at(1).name == "Dix" && french.trophies.at(6).name.empty(), "the system language was not used"); + check(NpTrophy2::ParseTrophyPackage(trophyPackage(), "de-DE").language == "en-US", "English was not the fallback"); + std::string conf = TropConf; + conf.replace(conf.find("en-US"), 5, "ja-JP"); + check(NpTrophy2::ParseTrophyPackage(ucp({{"tropconf.json", conf}, {"tropmeta_ja-JP.json", TropMetaEnglish}}), "de-DE").language == "ja-JP", "the default language was not the last fallback"); + const auto withoutTexts = ucp({{"tropconf.json", conf}}); + check(contains(failure([&withoutTexts] { NpTrophy2::ParseTrophyPackage(withoutTexts, "de-DE"); }), "no tropmeta_.json for de-DE, en-US or its defaultLanguage \"ja-JP\""), + "a package without texts was read"); + const auto missingTexts = [](const std::string& defaultLanguage, const char* language) { + std::string text = TropConf; + text.replace(text.find("en-US"), 5, defaultLanguage); + const auto data = ucp({{"tropconf.json", text}}); + return failure([&data, language] { NpTrophy2::ParseTrophyPackage(data, language); }); + }; + check(missingTexts("en-US", "en-US") == "it has no tropmeta_.json for en-US", "a missing text named en-US more than once"); + check(missingTexts("ja-JP", "en-US") == "it has no tropmeta_.json for en-US or its defaultLanguage \"ja-JP\"", "a missing text in English named English twice"); + check(missingTexts("en-US", "de-DE") == "it has no tropmeta_.json for de-DE or en-US", "a missing text with an English default named English twice"); + + const auto broken = [](const std::string& replaced, const std::string& replacement) { + std::string text = TropConf; + text.replace(text.find(replaced), replaced.size(), replacement); + const auto data = ucp({{"tropconf.json", text}, {"tropmeta_en-US.json", TropMetaEnglish}}); + return failure([&data] { NpTrophy2::ParseTrophyPackage(data, "en-US"); }); + }; + check(contains(broken(R"("grade":"G")", R"("grade":"X")"), "trophy 1 has no grade"), "a wrong grade was accepted"); + check(contains(broken(R"("grade":"G")", R"("grade":"GS")"), "trophy 1 has no grade"), "a two-letter grade was accepted"); + check(contains(broken(R"({"id":"4",)", R"({"id":"3",)"), "lists trophy 3 twice"), "a repeated trophy was accepted"); + check(contains(broken(R"({"id":"4",)", R"({"id":"-4",)"), "trophies[4] has no id"), "a negative id was accepted"); + check(contains(broken(R"("groupId":"1","hidden")", R"("groupId":"x","hidden")"), "trophy 6 has groupId \"x\""), "a wrong group was accepted"); + check(contains(broken(R"("udsStatId":"10")", R"("udsStatId":"-10")"), "udsStatId \"-10\""), "a negative stat id was accepted"); + check(contains(broken(R"("trophies":[)", R"("trophies":0,"other":[)"), "has no trophies array"), "trophies that are not an array were accepted"); + const auto withoutConf = ucp({{"tropmeta_en-US.json", TropMetaEnglish}}); + check(contains(failure([&withoutConf] { NpTrophy2::ParseTrophyPackage(withoutConf, "en-US"); }), "no tropconf.json"), "a package without tropconf.json was read"); + const auto brokenConf = ucp({{"tropconf.json", "{\"trophies\":["}, {"tropmeta_en-US.json", TropMetaEnglish}}); + check(contains(failure([&brokenConf] { NpTrophy2::ParseTrophyPackage(brokenConf, "en-US"); }), "tropconf.json is not JSON"), "a tropconf.json that is not JSON was read"); + const auto withoutTitle = ucp({{"tropconf.json", TropConf}, {"tropmeta_en-US.json", "{\"metadata\":{}}"}}); + check(contains(failure([&withoutTitle] { NpTrophy2::ParseTrophyPackage(withoutTitle, "en-US"); }), "has no titleMetadata"), "texts without a title were read"); + std::string groups = R"({"defaultLanguage":"en-US","trophies":[)"; + for (int id = 0; id <= 50; ++id) groups += (id == 0 ? "" : ",") + std::string(R"({"id":)") + std::to_string(id) + R"(,"grade":"B","groupId":)" + std::to_string(id) + "}"; + groups += "]}"; + const auto manyGroups = ucp({{"tropconf.json", groups}, {"tropmeta_en-US.json", TropMetaEnglish}}); + check(contains(failure([&manyGroups] { NpTrophy2::ParseTrophyPackage(manyGroups, "en-US"); }), "more than 50 groups"), "more than 50 groups were accepted"); + std::string padded = TropConf; + std::string padding = "\"padding\":["; + for (std::size_t value = 0; value < 1048576; ++value) padding += "0,"; + padded.insert(1, padding + "0],"); + const auto oversized = ucp({{"tropconf.json", padded}, {"tropmeta_en-US.json", TropMetaEnglish}}); + check(contains(failure([&oversized] { NpTrophy2::ParseTrophyPackage(oversized, "en-US"); }), "tropconf.json is not JSON: the document holds more than 1048576 values"), + "a tropconf.json of more than 1048576 values was read"); + check(contains(failure([] { NpTrophy2::ParseTrophyPackage("not a package", "en-US"); }), "fewer than the 64-byte header"), "a package that is not a container was read"); + check(NpTrophy2::LanguageTag(1) == "en-US" && NpTrophy2::LanguageTag(0) == "ja-JP" && NpTrophy2::LanguageTag(2) == "fr-FR" && NpTrophy2::LanguageTag(29) == "id-ID", "language tags are wrong"); + bool thrown = false; + try { + NpTrophy2::LanguageTag(31); + } catch (const std::out_of_range&) { + thrown = true; + } + check(thrown, "language 31 has a tag"); +} + +void testProgressAndCounts() { + const auto package = NpTrophy2::ParseTrophyPackage(trophyPackage(), "en-US"); + const auto& progressive = package.trophies.at(5); + check(NpTrophy2::TrophyProgress(progressive, std::nullopt, false) == 0 && NpTrophy2::TrophyProgress(progressive, 3, false) == 3 && NpTrophy2::TrophyProgress(progressive, 9, false) == 5 && + NpTrophy2::TrophyProgress(progressive, 1, true) == 5, + "progress is not the stat up to the target"); + bool thrown = false; + try { + NpTrophy2::TrophyProgress(package.trophies.at(1), 3, false); + } catch (const std::invalid_argument&) { + thrown = true; + } + check(thrown, "a trophy that is not progressive reported progress"); + + const auto all = NpTrophy2::CountTrophies(package, std::nullopt); + check(all.trophies == 8 && all.unlocked == 0 && all.grades[NpTrophy2::GradePlatinum] == 1 && all.grades[NpTrophy2::GradeGold] == 1 && all.grades[NpTrophy2::GradeSilver] == 1 && + all.grades[NpTrophy2::GradeBronze] == 5, + "the game counts are wrong"); + check(all.Percentage() == 0, "a game without unlocks has progress"); + const auto expansion = NpTrophy2::CountTrophies(package, 1); + check(expansion.trophies == 2 && expansion.unlocked == 0 && expansion.Percentage() == 0, "the group counts are wrong"); + check(NpTrophy2::CountTrophies(package, 5).trophies == 0 && NpTrophy2::CountTrophies(package, 5).Percentage() == 0, "an empty group has trophies"); +} + +} + +int main() { + const auto previous = std::filesystem::current_path(); + const auto root = std::filesystem::temp_directory_path() / ("anyps5-trophy-package-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(root); + std::filesystem::current_path(root); + testContainer(); + testJson(); + testPackage(); + testProgressAndCounts(); + std::filesystem::current_path(previous); + std::filesystem::remove_all(root); + std::puts("Trophy package tests passed"); + return 0; +} diff --git a/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt b/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt index ffa5b0803b..a7f5848c61 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt +++ b/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt @@ -13,3 +13,15 @@ target_include_directories(libSceNpUniversalDataSystem PRIVATE ) target_link_libraries(libSceNpUniversalDataSystem PRIVATE libc) configure_windows_unwind(libSceNpUniversalDataSystem) + +if(BUILD_TESTING) + add_test_executable(guest_np_trophy_data_tests ${LIBS_INCLUDE_DIR}/tests/GuestNpTrophyData.cpp) + target_include_directories(guest_np_trophy_data_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_np_trophy_data_tests PRIVATE libSceNpUniversalDataSystem libSceNpTrophy2 libc) + configure_windows_unwind(guest_np_trophy_data_tests) + add_test(NAME guest_np_trophy_data COMMAND guest_np_trophy_data_tests) + if(WIN32) + set_tests_properties(guest_np_trophy_data PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/tests/GuestNpTrophyData.cpp b/core/libs/tests/GuestNpTrophyData.cpp new file mode 100644 index 0000000000..6e07c0e769 --- /dev/null +++ b/core/libs/tests/GuestNpTrophyData.cpp @@ -0,0 +1,188 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libSceNpTrophy2/include/NpTrophy2Types.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceNpTrophy2CreateContext(int*, int, uint32_t, uint64_t); +int APS5_VABI sceNpTrophy2GetGameInfo(int, int, NpTrophy2GameDetails*, NpTrophy2GameData*); +int APS5_VABI sceNpTrophy2GetGroupInfo(int, int, int, NpTrophy2GroupDetails*, NpTrophy2GroupData*); +int APS5_VABI sceNpTrophy2GetGroupInfoArray(int, int, uint32_t, uint32_t, NpTrophy2GroupDetails*, NpTrophy2GroupData*, uint32_t*); +int APS5_VABI sceNpTrophy2GetTrophyInfo(int, int, int, NpTrophy2Details*, NpTrophy2Data*); +int APS5_VABI sceNpTrophy2GetTrophyInfoArray(int, int, uint32_t, uint32_t, NpTrophy2Details*, NpTrophy2Data*, uint32_t*); +} + +static constexpr int InitialUser = 0x10000000; + +static const char TropConf[] = R"({"defaultLanguage":"en-US","trophies":[ + {"id":"0","grade":"P"}, + {"id":"1","grade":"G","platinumTrophyId":"0","unlockCondition":{"udsStatId":"10","comparator":"ge","targetValue":"10","progressive":false}}, + {"id":"2","grade":"S","platinumTrophyId":"0","unlockCondition":{"udsStatId":"20","comparator":"gt","targetValue":"100"}}, + {"id":"3","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"30","comparator":"le","targetValue":"5"}}, + {"id":"4","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"30","comparator":"lt","targetValue":"3"}}, + {"id":"5","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"40","comparator":"ge","targetValue":"5","progressive":true}}, + {"id":"6","grade":"B","groupId":"1","hidden":true,"hasReward":true}, + {"id":"7","grade":"B","groupId":"1"} +]})"; + +static const char TropMeta[] = R"({"metadata":{"titleMetadata":{"name":"Synthetic Game"}, + "trophyMetadata":[{"id":"0","name":"All of them"},{"id":"1","name":"Ten","detail":"Reach ten"},{"id":"5","name":"Five steps"}, + {"id":"6","name":"Secret","detail":"A hidden one","reward":"A hat"}], + "groupMetadata":[{"id":"1","name":"Expansion"}]}})"; + +static void Require(bool value, const char* message) { + if (value) return; + std::fprintf(stderr, "%s\n", message); + std::abort(); +} + +template +static bool ThrowsInvalidArgument(TCall call) { + try { + call(); + } catch (const std::invalid_argument&) { + return true; + } + return false; +} + +static void PutBigEndian(std::string& data, std::size_t offset, std::uint64_t value, std::size_t size) { + for (std::size_t i = 0; i < size; ++i) data[offset + i] = static_cast((value >> (8 * (size - 1 - i))) & 0xFF); +} + +static std::string Ucp(const std::vector>& files) { + std::size_t offset = 0x60 + files.size() * 0x40; + std::size_t size = offset; + for (const auto& file : files) size += file.second.size(); + std::string data(size, '\0'); + PutBigEndian(data, 0, 0xB228C60A, 4); + PutBigEndian(data, 4, 1, 4); + PutBigEndian(data, 8, size, 8); + PutBigEndian(data, 0x10, files.size(), 4); + PutBigEndian(data, 0x14, 0x40, 4); + std::size_t entry = 0x60; + for (const auto& [name, contents] : files) { + data.replace(entry, name.size(), name); + PutBigEndian(data, entry + 0x20, offset, 8); + PutBigEndian(data, entry + 0x28, contents.size(), 8); + data.replace(offset, contents.size(), contents); + offset += contents.size(); + entry += 0x40; + } + return data; +} + +static void Write(const std::filesystem::path& path, const std::string& text) { + std::filesystem::create_directories(path.parent_path()); + std::FILE* file = std::fopen(path.string().c_str(), "wb"); + Require(file != nullptr, "a test file was not created"); + const bool written = std::fwrite(text.data(), 1, text.size(), file) == text.size(); + Require(std::fclose(file) == 0 && written, "a test file was not written"); +} + +static int TrophyContext(std::uint32_t serviceLabel) { + int context = 0; + Require(sceNpTrophy2CreateContext(&context, InitialUser, serviceLabel, 0) == 0, "trophy context was not created"); + return context; +} + +static NpTrophy2Data Trophy(int context, int trophyId, NpTrophy2Details* details = nullptr) { + NpTrophy2Data data{}; + Require(sceNpTrophy2GetTrophyInfo(context, 1, trophyId, details, &data) == 0, "trophy info was not returned"); + Require(data.trophy_id == trophyId, "trophy info is for another trophy"); + return data; +} + +static NpTrophy2GameData Game(int context, NpTrophy2GameDetails* details = nullptr) { + NpTrophy2GameData data{}; + Require(sceNpTrophy2GetGameInfo(context, 1, details, &data) == 0, "game info was not returned"); + return data; +} + +static void ReportTrophyData(int context) { + NpTrophy2GameDetails game{}; + const auto progress = Game(context, &game); + Require(std::strcmp(game.title, "Synthetic Game") == 0 && game.num_groups == 2 && game.num_trophies == 8 && game.num_platinum == 1 && game.num_gold == 1 && game.num_silver == 1 && game.num_bronze == 5, + "the game details are not the title's"); + Require(progress.unlocked_trophies == 0 && progress.progress_percentage == 0, "the game has unlocks before any event"); + + NpTrophy2Details details{}; + Trophy(context, 1, &details); + Require(std::strcmp(details.name, "Ten") == 0 && std::strcmp(details.description, "Reach ten") == 0 && details.trophy_grade == 2 && details.group_id == -1 && !details.hidden && + details.target.type == 0 && details.target.value == 0, + "trophy 1 does not report its name, grade and group"); + Trophy(context, 6, &details); + Require(std::strcmp(details.name, "Secret") == 0 && std::strcmp(details.reward, "A hat") == 0 && details.trophy_grade == 4 && details.group_id == 1 && details.hidden && details.has_reward, + "trophy 6 does not report its hidden flag, reward and group"); + Trophy(context, 0, &details); + Require(details.trophy_grade == 1 && std::strcmp(details.name, "All of them") == 0, "the platinum is not reported"); + const auto steps = Trophy(context, 5, &details); + Require(details.target.type == 1 && details.target.value == 5 && !steps.unlocked && steps.progress.type == 1 && steps.progress.value == 0, "the progressive trophy does not report its target"); + NpTrophy2Data unknown{}; + Require(ThrowsInvalidArgument([context, &unknown] { sceNpTrophy2GetTrophyInfo(context, 1, 99, nullptr, &unknown); }), "a trophy that is not in the package was reported"); + + std::vector list(10); + std::vector listed(10); + std::uint32_t count = 0; + Require(sceNpTrophy2GetTrophyInfoArray(context, 1, 0, 10, list.data(), listed.data(), &count) == 0 && count == 8, "the trophy list does not hold every trophy"); + for (std::uint32_t i = 0; i < count; ++i) Require(list[i].trophy_id == static_cast(i) && listed[i].trophy_id == static_cast(i), "the trophy list is not in id order"); + Require(std::strcmp(list[5].name, "Five steps") == 0 && listed[5].progress.type == 1, "the trophy list does not carry names and progress"); + Require(sceNpTrophy2GetTrophyInfoArray(context, 1, 6, 5, list.data(), nullptr, &count) == 0 && count == 2 && list[0].trophy_id == 6 && list[1].trophy_id == 7, + "the trophy list does not start at its offset"); + Require(sceNpTrophy2GetTrophyInfoArray(context, 1, 8, 5, nullptr, listed.data(), &count) == 0 && count == 0, "the trophy list goes past its end"); + + NpTrophy2GroupDetails group{}; + NpTrophy2GroupData groupData{}; + Require(sceNpTrophy2GetGroupInfo(context, 1, 1, &group, &groupData) == 0 && std::strcmp(group.title, "Expansion") == 0 && group.num_trophies == 2 && group.num_bronze == 2 && groupData.group_id == 1, + "group 1 is not reported"); + NpTrophy2GroupData unknownGroup{}; + Require(ThrowsInvalidArgument([context, &unknownGroup] { sceNpTrophy2GetGroupInfo(context, 1, 7, nullptr, &unknownGroup); }), "a group that is not in the package was reported"); + std::vector groups(4); + Require(sceNpTrophy2GetGroupInfoArray(context, 1, 0, 4, groups.data(), nullptr, &count) == 0 && count == 2 && groups[0].group_id == -1 && std::strcmp(groups[0].title, "Synthetic Game") == 0 && + groups[0].num_trophies == 6 && groups[1].group_id == 1, + "the group list does not hold the base game and group 1"); +} + +static void ReportPlaceholders() { + for (const std::uint32_t serviceLabel : {1u, 2u}) { + const auto context = TrophyContext(serviceLabel); + NpTrophy2GameDetails game{}; + Game(context, &game); + NpTrophy2Details details{}; + Trophy(context, 12, &details); + std::uint32_t count = 0; + NpTrophy2Data listed{}; + Require(std::strcmp(game.title, "Game") == 0 && std::strcmp(details.name, "Trophy") == 0 && + sceNpTrophy2GetTrophyInfoArray(context, 1, 0, 10, nullptr, &listed, &count) == 0 && count == 1, + "a missing or malformed trophy package does not leave the placeholders"); + } +} + +int main() { + const auto previous = std::filesystem::current_path(); + const auto root = std::filesystem::temp_directory_path() / ("anyps5-trophy-data-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(root); + std::filesystem::current_path(root); + Write("app0/sce_sys/param.json", R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}}})"); + Write("app0/sce_sys/trophy2/trophy00.ucp", Ucp({{"tropconf.json", TropConf}, {"tropmeta_en-US.json", TropMeta}})); + Write("app0/sce_sys/trophy2/trophy02.ucp", "not a package"); + + const auto context = TrophyContext(0); + ReportTrophyData(context); + ReportPlaceholders(); + + std::filesystem::current_path(previous); + std::filesystem::remove_all(root); + std::puts("Trophy data tests passed"); + return 0; +} diff --git a/docs/dev/debt/libSceNpTrophy2.md b/docs/dev/debt/libSceNpTrophy2.md index d5e9083695..a92744d72c 100644 --- a/docs/dev/debt/libSceNpTrophy2.md +++ b/docs/dev/debt/libSceNpTrophy2.md @@ -6,6 +6,12 @@ ### Unknown function info -- [Query structures](../../../core/libs/prx/libSceNpTrophy2/include/NpTrophy2Types.hpp): a progress value is a 16-byte `{int32 type, 4 reserved bytes, uint64 value}` and the game, group and trophy structures are 152, 24, 152, 32, 1312 and 32 bytes, as [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/9e41d8320a2774b303fbad344c2347e8d0292e83/src/libs/libTrophy.cpp) asserts; no other public source confirms the layout. Progress type 1 marks a progressive trophy's target and progress, as in KytyPS5. +- [Query structures](../../../core/libs/prx/libSceNpTrophy2/include/NpTrophy2Types.hpp): a progress value is a 16-byte `{int32 type, 4 reserved bytes, uint64 value}` and the game, group and trophy structures are 152, 24, 152, 32, 1312 and 32 bytes, as [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/9e41d8320a2774b303fbad344c2347e8d0292e83/src/libs/libTrophy.cpp) asserts; no other public source confirms the layout. Progress type 1 marks a progressive trophy's target and progress, as in KytyPS5. Trophies without a `groupId` belong to group -1, KytyPS5's base game group. - [sceNpTrophy2GetRewardIcon](../../../core/libs/prx/libSceNpTrophy2/src/Context.cpp) (libSceNpTrophy2) takes `(int context, int handle, int reward_id, void* buffer, size_t* size)`, from the call sites in the bundled `Media/Plugins/PSNCore.prx` of PPSA06888, the same shape as `sceNpTrophy2GetTrophyIcon`. The title calls it twice: first with a null buffer to read the size, then with a buffer of that size allocated with `malloc`. The meaning of the third argument is unverified, and the reward icon data is unknown, so every call throws + +### Functional + +- [Trophy queries](../../../core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp) report the title's [trophy data](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp) (`sce_sys/trophy2/trophyNN.ucp`). Without the file, or when it is malformed, they report the old placeholders and `sceNpTrophy2GetTrophyInfoArray` lists only trophy 0. Icons are not read. Hidden trophies report their names and descriptions; whether the console blanks them before the unlock is not known. `progress_percentage` weighs gold 6, silver 2 and bronze 1 and leaves the platinum out, as KytyPS5 does (the ratio of the published PSN trophy points); the console's formula is not known. A trophy or group that is not in the file throws. Unlocks are not recorded, so every trophy reports as locked, with no unlock time and a progress of 0, and the unlocked counts and `progress_percentage` are 0. A context that was not created or was destroyed reports placeholders and returns success; the console's errors for it are unknown. +- [Trophy data limits](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): a `trophyNN.ucp` larger than 256 MiB or listing more than 4096 files, and a JSON file in it that nests deeper than 64 levels or holds more than 1048576 values, are not used, and the title runs as without that file. The limits keep the memory the reader takes to the file and about 140 MiB of parsed JSON; the console's limits are not known. +- [Text language](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): texts come from `tropmeta_.json` for the console language, then `en-US`, then the file's `defaultLanguage`. The tags for languages 0 to 29 are KytyPS5's; `uk-UA` for 30 (Ukrainian) is an assumption. From 61e12c75fdd5290c11ccefd9116e5f8726363b41 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:26:06 +0000 Subject: [PATCH 069/103] docs: describe trophy data in USAGE --- docs/user/USAGE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 9d1882c0bb..bc96b850c1 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -157,6 +157,12 @@ Games that open the console's system font sets (`sceFontOpenFontSet`) need font Games ask for the console's language with `sceSystemServiceParamGetInt` and show their text in it when they have it. It is English (United States) unless `ANYPS5_LANGUAGE` is set to another console language number: 0 Japanese, 1 English (United States), 2 French, 3 Spanish, 4 German, 5 Italian, 6 Dutch, 7 Portuguese (Portugal), 8 Russian, 9 Korean, 10 Chinese (Traditional), 11 Chinese (Simplified), 12 Finnish, 13 Swedish, 14 Danish, 15 Norwegian, 16 Polish, 17 Portuguese (Brazil), 18 English (United Kingdom), 19 Turkish, 20 Spanish (Latin America), 21 Arabic, 22 French (Canada), 23 Czech, 24 Hungarian, 25 Greek, 26 Romanian, 27 Thai, 28 Vietnamese, 29 Indonesian, 30 Ukrainian. For example, `ANYPS5_LANGUAGE=21` selects Arabic. An empty value is the same as an unset one; any other value stops the game at its first language query with an error that names the variable. +### Trophies + +The trophy data comes from the game's own `app0/sce_sys/trophy2/trophyNN.ucp`, where `NN` is the service label (`00` for most games). The trophy queries report its names, descriptions and rewards in the console language (`ANYPS5_LANGUAGE`, see above), or in English (United States) or the data's default language when it has no text in that language, and its grades, groups, hidden flags, number of trophies and grade counts. The first trophy query prints what was read, such as `[trophy] /app0/sce_sys/trophy2/trophy00.ucp: 52 trophies, 1 group, texts in en-US`. + +Trophies are not unlocked yet: every trophy reports as locked, and nothing is written for them. A missing or malformed `trophyNN.ucp` makes the queries report placeholders, and a malformed one prints the reason once; `APS5_TRACE_UDS=1` prints a `[uds]` line for a missing one. Icons are not read: `sceNpTrophy2GetGameIcon`, `sceNpTrophy2GetGroupIcon`, `sceNpTrophy2GetTrophyIcon` and `sceNpTrophy2GetRewardIcon` stop the game with an error. No notification is shown, and callbacks registered with `sceNpTrophy2RegisterUnlockCallback` are not called. + ### PlayGo dumps without metadata When a dump has neither `sce_sys/playgo-chunk.dat` nor `sce_sys/playgo-scenario.json`, AnyPS5 cannot tell which nonzero PlayGo chunk IDs exist. Requests for those IDs stop with an error by default. Set `APS5_PLAYGO_UNLISTED_CHUNKS` to choose an answer for that dump: From 95179c1d61b472b713988e51f6a69d4386d78906 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:27:39 +0000 Subject: [PATCH 070/103] feat(libSceNpTrophy2): record trophy unlocks per title and user Unlocks are kept in _trophy/_.txt in the title's run directory, one " " line per trophy, and each new unlock logs "[trophy] unlocked trophy <id> for user <id>". GetTrophyInfo, GetTrophyInfoArray, GetGroupInfo, GetGroupInfoArray and GetGameInfo report the recorded unlocks, their times, the unlocked counts by grade and the progress percentage. An unlock also unlocks the platinum the trophy names once every trophy naming that platinum is unlocked. Unlock listeners let a later notification subscribe. Removing a listener waits for its running callback, and a listener removed during a dispatch is not called by it. --- core/libs/prx/libSceNpTrophy2/CMakeLists.txt | 14 ++ .../libSceNpTrophy2/include/TitleTrophies.hpp | 14 +- .../libSceNpTrophy2/include/TrophyStore.hpp | 74 +++++++ core/libs/prx/libSceNpTrophy2/src/Context.cpp | 2 +- .../libs/prx/libSceNpTrophy2/src/GameInfo.cpp | 5 +- .../prx/libSceNpTrophy2/src/GroupInfo.cpp | 10 +- .../prx/libSceNpTrophy2/src/TitleTrophies.cpp | 21 +- .../prx/libSceNpTrophy2/src/TrophyInfo.cpp | 23 ++- .../prx/libSceNpTrophy2/src/TrophyStore.cpp | 190 +++++++++++++++++ .../libSceNpTrophy2/tests/TrophyPackage.cpp | 32 ++- .../prx/libSceNpTrophy2/tests/TrophyStore.cpp | 195 ++++++++++++++++++ docs/dev/debt/libSceNpTrophy2.md | 5 +- 12 files changed, 548 insertions(+), 37 deletions(-) create mode 100644 core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp create mode 100644 core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp create mode 100644 core/libs/prx/libSceNpTrophy2/tests/TrophyStore.cpp diff --git a/core/libs/prx/libSceNpTrophy2/CMakeLists.txt b/core/libs/prx/libSceNpTrophy2/CMakeLists.txt index cd03e490cb..a7189fbc13 100644 --- a/core/libs/prx/libSceNpTrophy2/CMakeLists.txt +++ b/core/libs/prx/libSceNpTrophy2/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(libSceNpTrophy2 SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/src/GameInfo.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/TrophyInfo.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GroupInfo.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/TrophyStore.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/TrophyPackage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/TitleTrophies.cpp ) @@ -26,6 +27,19 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(np_trophy2_store_tests EXCLUDE_FROM_ALL tests/TrophyStore.cpp) +target_include_directories(np_trophy2_store_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(np_trophy2_store_tests PRIVATE libSceNpTrophy2 libc) +configure_windows_unwind(np_trophy2_store_tests) +if(BUILD_TESTING) + set_target_properties(np_trophy2_store_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME np_trophy2_store COMMAND np_trophy2_store_tests) + if(WIN32) + set_tests_properties(np_trophy2_store PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpTrophy2>") + endif() +endif() + add_test_executable(np_trophy2_package_tests EXCLUDE_FROM_ALL tests/TrophyPackage.cpp) target_include_directories(np_trophy2_package_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(np_trophy2_package_tests PRIVATE libSceNpTrophy2 libc) diff --git a/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp b/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp index 8caaa30647..74901778a9 100644 --- a/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp +++ b/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp @@ -4,6 +4,7 @@ #include <array> #include <cstddef> #include <cstdint> +#include <functional> #include <map> #include <optional> #include <string> @@ -11,6 +12,7 @@ #include <vector> #include "prx/libSceNpTrophy2/include/TrophyPackage.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" namespace NpTrophy2 { @@ -26,15 +28,11 @@ const TitleTrophies& Title(std::uint32_t serviceLabel); const TrophyPackage* Package(std::uint32_t serviceLabel); void CopyText(char* destination, std::size_t size, std::string_view text); -struct ContextOwner { - std::int32_t userId = 0; - std::uint32_t serviceLabel = 0; -}; +using UnlockedQuery = std::function<bool(std::int32_t)>; -int CreateContext(std::int32_t userId, std::uint32_t serviceLabel); -void DestroyContext(int context); -std::optional<ContextOwner> FindContext(int context); +UnlockedQuery UnlockedBy(std::int32_t userId, std::uint32_t serviceLabel); +std::optional<std::int32_t> PlatinumToUnlock(const TrophyPackage& package, std::int32_t trophyId, const UnlockedQuery& unlocked); std::optional<std::uint64_t> ProgressTarget(const TrophyDefinition& trophy); std::uint64_t TrophyProgress(const TrophyDefinition& trophy, std::optional<std::uint64_t> stat, bool unlocked); @@ -47,7 +45,7 @@ struct TrophyCounts { std::uint32_t Percentage() const; }; -TrophyCounts CountTrophies(const TrophyPackage& package, std::optional<std::int32_t> group); +TrophyCounts CountTrophies(const TrophyPackage& package, std::optional<std::int32_t> group, const UnlockedQuery& unlocked); } diff --git a/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp b/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp new file mode 100644 index 0000000000..245966e568 --- /dev/null +++ b/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp @@ -0,0 +1,74 @@ +#ifndef CORE_LIBS_PRX_LIBSCENPTROPHY2_INCLUDE_TROPHYSTORE_HPP +#define CORE_LIBS_PRX_LIBSCENPTROPHY2_INCLUDE_TROPHYSTORE_HPP + +#include <cstddef> +#include <cstdint> +#include <filesystem> +#include <map> +#include <mutex> +#include <optional> +#include <string> +#include <utility> + +struct NpTrophy2Unlock { + std::string titleId; + std::int32_t userId = 0; + std::uint32_t serviceLabel = 0; + std::int32_t trophyId = 0; + std::uint64_t tick = 0; +}; + +using NpTrophy2UnlockListener = void (*)(void* context, const NpTrophy2Unlock& unlock); + +namespace NpTrophy2 { + +inline constexpr char TrophyRootGuest[] = "/_trophy"; +inline constexpr std::uint64_t UnixEpochTick = 62135596800000000ull; + +std::uint64_t CurrentTick(); + +class TrophyStore { +public: + explicit TrophyStore(std::filesystem::path directory); + + bool Unlock(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId, std::uint64_t tick); + std::optional<std::uint64_t> UnlockTick(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId); + std::size_t UnlockCount(std::int32_t userId, std::uint32_t serviceLabel); + std::filesystem::path RecordPath(std::int32_t userId, std::uint32_t serviceLabel) const; + +private: + using Key = std::pair<std::int32_t, std::uint32_t>; + using Unlocks = std::map<std::int32_t, std::uint64_t>; + + Unlocks& record(const Key& key); + void save(const Key& key, const Unlocks& unlocks) const; + + std::filesystem::path directory; + std::mutex mutex; + std::map<Key, Unlocks> records; +}; + +TrophyStore& TitleStore(); + +struct ContextOwner { + std::int32_t userId = 0; + std::uint32_t serviceLabel = 0; +}; + +int CreateContext(std::int32_t userId, std::uint32_t serviceLabel); +void DestroyContext(int context); +std::optional<ContextOwner> FindContext(int context); +std::optional<std::uint64_t> ContextUnlockTick(int context, std::int32_t trophyId); +std::size_t ContextUnlockCount(int context); + +} + +extern "C" { + +bool NpTrophy2RecordUnlock_nid_no_patch(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId); +std::uint64_t NpTrophy2AddUnlockListener_nid_no_patch(NpTrophy2UnlockListener listener, void* context); +void NpTrophy2RemoveUnlockListener_nid_no_patch(std::uint64_t id); + +} + +#endif diff --git a/core/libs/prx/libSceNpTrophy2/src/Context.cpp b/core/libs/prx/libSceNpTrophy2/src/Context.cpp index 90887a7f4a..ef3a23e905 100644 --- a/core/libs/prx/libSceNpTrophy2/src/Context.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/Context.cpp @@ -6,7 +6,7 @@ #include <string> #include "prx/libSceNpTrophy2/include/NpTrophy2.hpp" -#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include "prx/libc/include/General.hpp" namespace NpTrophy2 { diff --git a/core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp b/core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp index bea174aa49..dc11f536ef 100644 --- a/core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp @@ -6,6 +6,7 @@ #include "prx/libSceNpTrophy2/include/NpTrophy2.hpp" #include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include "prx/libc/include/General.hpp" extern "C" { @@ -30,7 +31,7 @@ int APS5_VABI sceNpTrophy2GetGameInfo(int context, int handle, NpTrophy2GameDeta } if (data != nullptr) { std::memset(data, 0, sizeof(*data)); - data->unlocked_trophies = NP_TROPHY2_UNLOCKED_TROPHIES; + data->unlocked_trophies = static_cast<std::uint32_t>(NpTrophy2::ContextUnlockCount(context)); data->unlocked_platinum = NP_TROPHY2_UNLOCKED_PLATINUM; data->unlocked_gold = NP_TROPHY2_UNLOCKED_GOLD; data->unlocked_silver = NP_TROPHY2_UNLOCKED_SILVER; @@ -39,7 +40,7 @@ int APS5_VABI sceNpTrophy2GetGameInfo(int context, int handle, NpTrophy2GameDeta } return SCE_NP_TROPHY2_OK; } - const auto counts = NpTrophy2::CountTrophies(*package, std::nullopt); + const auto counts = NpTrophy2::CountTrophies(*package, std::nullopt, NpTrophy2::UnlockedBy(owner->userId, owner->serviceLabel)); if (details != nullptr) { std::memset(details, 0, sizeof(*details)); details->num_groups = static_cast<std::uint32_t>(package->groups.size()); diff --git a/core/libs/prx/libSceNpTrophy2/src/GroupInfo.cpp b/core/libs/prx/libSceNpTrophy2/src/GroupInfo.cpp index badd913d14..887423ddc8 100644 --- a/core/libs/prx/libSceNpTrophy2/src/GroupInfo.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/GroupInfo.cpp @@ -8,6 +8,7 @@ #include "prx/libSceNpTrophy2/include/NpTrophy2.hpp" #include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include "prx/libc/include/General.hpp" namespace { @@ -34,8 +35,9 @@ void fillPlaceholderData(std::int32_t groupId, NpTrophy2GroupData* data) { data->progress_percentage = NP_TROPHY2_PROGRESS_PERCENTAGE; } -void fillGroup(const NpTrophy2::TrophyPackage& package, std::int32_t groupId, const std::string& title, NpTrophy2GroupDetails* details, NpTrophy2GroupData* data) { - const auto counts = NpTrophy2::CountTrophies(package, groupId); +void fillGroup(const NpTrophy2::TrophyPackage& package, const NpTrophy2::ContextOwner& owner, std::int32_t groupId, const std::string& title, NpTrophy2GroupDetails* details, + NpTrophy2GroupData* data) { + const auto counts = NpTrophy2::CountTrophies(package, groupId, NpTrophy2::UnlockedBy(owner.userId, owner.serviceLabel)); if (details != nullptr) { std::memset(details, 0, sizeof(*details)); details->group_id = groupId; @@ -79,7 +81,7 @@ int APS5_VABI sceNpTrophy2GetGroupInfo(int context, int handle, int group_id, Np if (group == package->groups.end()) { throw std::invalid_argument(std::string(__func__) + ": group " + std::to_string(group_id) + " is not in " + NpTrophy2::PackagePath(owner->serviceLabel)); } - fillGroup(*package, group->first, group->second, details, data); + fillGroup(*package, *owner, group->first, group->second, details, data); return SCE_NP_TROPHY2_OK; } @@ -101,7 +103,7 @@ int APS5_VABI sceNpTrophy2GetGroupInfoArray(int context, int handle, uint32_t of auto group = package->groups.begin(); std::advance(group, std::min<std::size_t>(offset, package->groups.size())); for (; group != package->groups.end() && *count < limit; ++group, ++*count) { - fillGroup(*package, group->first, group->second, details_array != nullptr ? &details_array[*count] : nullptr, data_array != nullptr ? &data_array[*count] : nullptr); + fillGroup(*package, *owner, group->first, group->second, details_array != nullptr ? &details_array[*count] : nullptr, data_array != nullptr ? &data_array[*count] : nullptr); } return SCE_NP_TROPHY2_OK; } diff --git a/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp b/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp index 885a0b4206..dcd465e8ce 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp @@ -109,6 +109,22 @@ void CopyText(char* destination, std::size_t size, std::string_view text) { destination[length] = '\0'; } +UnlockedQuery UnlockedBy(std::int32_t userId, std::uint32_t serviceLabel) { + return [userId, serviceLabel](std::int32_t trophyId) { return TitleStore().UnlockTick(userId, serviceLabel, trophyId).has_value(); }; +} + +std::optional<std::int32_t> PlatinumToUnlock(const TrophyPackage& package, std::int32_t trophyId, const UnlockedQuery& unlocked) { + const auto trophy = package.trophies.find(trophyId); + if (trophy == package.trophies.end() || !trophy->second.platinumId) return std::nullopt; + const auto platinumId = *trophy->second.platinumId; + const auto platinum = package.trophies.find(platinumId); + if (platinum == package.trophies.end() || platinum->second.grade != GradePlatinum || unlocked(platinumId)) return std::nullopt; + for (const auto& [id, other] : package.trophies) { + if (id != platinumId && other.platinumId == platinumId && !unlocked(id)) return std::nullopt; + } + return platinumId; +} + std::optional<std::uint64_t> ProgressTarget(const TrophyDefinition& trophy) { if (!trophy.progressive) return std::nullopt; return trophy.target; @@ -121,12 +137,15 @@ std::uint64_t TrophyProgress(const TrophyDefinition& trophy, std::optional<std:: return std::min(stat.value_or(0), *target); } -TrophyCounts CountTrophies(const TrophyPackage& package, std::optional<std::int32_t> group) { +TrophyCounts CountTrophies(const TrophyPackage& package, std::optional<std::int32_t> group, const UnlockedQuery& unlocked) { TrophyCounts counts; for (const auto& [id, trophy] : package.trophies) { if (group && trophy.groupId != *group) continue; ++counts.trophies; ++counts.grades[static_cast<std::size_t>(trophy.grade)]; + if (!unlocked(id)) continue; + ++counts.unlocked; + ++counts.unlockedGrades[static_cast<std::size_t>(trophy.grade)]; } return counts; } diff --git a/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp b/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp index 001ba17a97..dfeba5ffbe 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp @@ -8,6 +8,7 @@ #include "prx/libSceNpTrophy2/include/NpTrophy2.hpp" #include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include "prx/libc/include/General.hpp" namespace { @@ -26,13 +27,14 @@ void fillPlaceholderDetails(int trophyId, NpTrophy2Details* details) { std::strncpy(details->reward, NP_TROPHY2_TROPHY_REWARD, sizeof(details->reward) - 1); } -void fillPlaceholderData(int trophyId, NpTrophy2Data* data) { +void fillPlaceholderData(int context, int trophyId, NpTrophy2Data* data) { std::memset(data, 0, sizeof(*data)); + const auto tick = NpTrophy2::ContextUnlockTick(context, trophyId); data->trophy_id = trophyId; - data->unlocked = false; + data->unlocked = tick.has_value(); data->progress.type = NP_TROPHY2_PROGRESS_TYPE_NONE; data->progress.value = NP_TROPHY2_PROGRESS_VALUE_NONE; - data->timestamp_tick = 0; + data->timestamp_tick = tick.value_or(0); } void fillDetails(const NpTrophy2::TrophyDefinition& trophy, NpTrophy2Details* details) { @@ -51,11 +53,12 @@ void fillDetails(const NpTrophy2::TrophyDefinition& trophy, NpTrophy2Details* de NpTrophy2::CopyText(details->reward, sizeof(details->reward), trophy.reward); } -void fillData(const NpTrophy2::TrophyDefinition& trophy, NpTrophy2Data* data) { +void fillData(const NpTrophy2::ContextOwner& owner, const NpTrophy2::TrophyDefinition& trophy, NpTrophy2Data* data) { std::memset(data, 0, sizeof(*data)); + const auto tick = NpTrophy2::TitleStore().UnlockTick(owner.userId, owner.serviceLabel, trophy.id); data->trophy_id = trophy.id; - data->unlocked = false; - data->timestamp_tick = 0; + data->unlocked = tick.has_value(); + data->timestamp_tick = tick.value_or(0); if (!NpTrophy2::ProgressTarget(trophy)) return; data->progress.type = NP_TROPHY2_PROGRESS_TYPE_VALUE; data->progress.value = NpTrophy2::TrophyProgress(trophy, std::nullopt, data->unlocked); @@ -74,7 +77,7 @@ int APS5_VABI sceNpTrophy2GetTrophyInfo(int context, int handle, int trophy_id, const auto* package = owner ? NpTrophy2::Package(owner->serviceLabel) : nullptr; if (package == nullptr) { if (details != nullptr) fillPlaceholderDetails(trophy_id, details); - if (data != nullptr) fillPlaceholderData(trophy_id, data); + if (data != nullptr) fillPlaceholderData(context, trophy_id, data); return SCE_NP_TROPHY2_OK; } const auto trophy = package->trophies.find(trophy_id); @@ -82,7 +85,7 @@ int APS5_VABI sceNpTrophy2GetTrophyInfo(int context, int handle, int trophy_id, throw std::invalid_argument(std::string(__func__) + ": trophy " + std::to_string(trophy_id) + " is not in " + NpTrophy2::PackagePath(owner->serviceLabel)); } if (details != nullptr) fillDetails(trophy->second, details); - if (data != nullptr) fillData(trophy->second, data); + if (data != nullptr) fillData(*owner, trophy->second, data); return SCE_NP_TROPHY2_OK; } @@ -97,7 +100,7 @@ int APS5_VABI sceNpTrophy2GetTrophyInfoArray(int context, int handle, uint32_t o const uint32_t out_count = (offset == 0 && limit != 0 ? 1u : 0u); *count = out_count; if (out_count != 0 && details_array != nullptr) fillPlaceholderDetails(NP_TROPHY2_TROPHY_ID_DEFAULT, details_array); - if (out_count != 0 && data_array != nullptr) fillPlaceholderData(NP_TROPHY2_TROPHY_ID_DEFAULT, data_array); + if (out_count != 0 && data_array != nullptr) fillPlaceholderData(context, NP_TROPHY2_TROPHY_ID_DEFAULT, data_array); return SCE_NP_TROPHY2_OK; } *count = 0; @@ -105,7 +108,7 @@ int APS5_VABI sceNpTrophy2GetTrophyInfoArray(int context, int handle, uint32_t o std::advance(trophy, std::min<std::size_t>(offset, package->trophies.size())); for (; trophy != package->trophies.end() && *count < limit; ++trophy, ++*count) { if (details_array != nullptr) fillDetails(trophy->second, &details_array[*count]); - if (data_array != nullptr) fillData(trophy->second, &data_array[*count]); + if (data_array != nullptr) fillData(*owner, trophy->second, &data_array[*count]); } return SCE_NP_TROPHY2_OK; } diff --git a/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp b/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp new file mode 100644 index 0000000000..501fc72223 --- /dev/null +++ b/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp @@ -0,0 +1,190 @@ +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" +#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" +#include "prx/libc/include/General.hpp" +#include "prx/libkernel/AppMetadata/include/AppMetadata.hpp" + +#include <algorithm> +#include <charconv> +#include <chrono> +#include <cstdio> +#include <fstream> +#include <stdexcept> +#include <string_view> +#include <system_error> +#include <vector> + +namespace NpTrophy2 { + +namespace { + +template <typename TValue> +bool parseNumber(std::string_view text, TValue& value) { + const auto* end = text.data() + text.size(); + const auto [stop, error] = std::from_chars(text.data(), end, value); + return error == std::errc{} && stop == end; +} + +struct Listener { + std::uint64_t id = 0; + NpTrophy2UnlockListener callback = nullptr; + void* context = nullptr; +}; + +struct Listeners { + std::mutex mutex; + std::recursive_mutex dispatch; + std::vector<Listener> entries; + std::uint64_t next = 1; +}; + +bool registered(Listeners& registry, std::uint64_t id) { + std::lock_guard lock(registry.mutex); + return std::any_of(registry.entries.begin(), registry.entries.end(), [id](const Listener& entry) { return entry.id == id; }); +} + +Listeners& listeners() { + static Listeners registry; + return registry; +} + +} + +std::uint64_t CurrentTick() { + const auto now = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count(); + return UnixEpochTick + static_cast<std::uint64_t>(now); +} + +TrophyStore::TrophyStore(std::filesystem::path directory) : directory(std::move(directory)) {} + +std::filesystem::path TrophyStore::RecordPath(std::int32_t userId, std::uint32_t serviceLabel) const { + return directory / (std::to_string(userId) + "_" + std::to_string(serviceLabel) + ".txt"); +} + +TrophyStore::Unlocks& TrophyStore::record(const Key& key) { + if (const auto found = records.find(key); found != records.end()) return found->second; + Unlocks unlocks; + const auto path = RecordPath(key.first, key.second); + std::error_code error; + const bool exists = std::filesystem::exists(path, error); + if (error) throw std::filesystem::filesystem_error("cannot check the trophy record", path, error); + if (exists) { + std::ifstream file(path, std::ios::binary); + if (!file) throw std::runtime_error("cannot read the trophy record " + path.string()); + std::string line; + std::size_t number = 0; + while (std::getline(file, line)) { + ++number; + const auto space = line.find(' '); + std::int32_t trophyId = 0; + std::uint64_t tick = 0; + const std::string_view text(line); + if (space == std::string::npos || !parseNumber(text.substr(0, space), trophyId) || trophyId < 0 || !parseNumber(text.substr(space + 1), tick) || !unlocks.emplace(trophyId, tick).second) { + throw std::runtime_error("the trophy record " + path.string() + " has a line " + std::to_string(number) + " that is not a new '<trophy id> <unlock tick>' pair"); + } + } + if (file.bad()) throw std::runtime_error("cannot read the trophy record " + path.string()); + } + return records.emplace(key, std::move(unlocks)).first->second; +} + +void TrophyStore::save(const Key& key, const Unlocks& unlocks) const { + std::error_code error; + if (std::filesystem::create_directories(directory, error)) RecordWrittenPath_nid_no_patch(directory); + if (error) throw std::filesystem::filesystem_error("cannot create the trophy directory", directory, error); + const auto path = RecordPath(key.first, key.second); + auto temporary = path; + temporary += ".tmp"; + { + std::ofstream file(temporary, std::ios::binary | std::ios::trunc); + for (const auto& [trophyId, tick] : unlocks) file << trophyId << ' ' << tick << '\n'; + file.flush(); + if (!file) throw std::runtime_error("cannot write the trophy record " + temporary.string()); + } + std::filesystem::rename(temporary, path); + RecordWrittenPath_nid_no_patch(path); +} + +bool TrophyStore::Unlock(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId, std::uint64_t tick) { + if (trophyId < 0) throw std::invalid_argument("trophy " + std::to_string(trophyId) + " cannot be unlocked: trophy ids are not negative"); + std::lock_guard lock(mutex); + const Key key{userId, serviceLabel}; + auto& unlocks = record(key); + if (unlocks.contains(trophyId)) return false; + auto updated = unlocks; + updated.emplace(trophyId, tick); + save(key, updated); + unlocks = std::move(updated); + return true; +} + +std::optional<std::uint64_t> TrophyStore::UnlockTick(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId) { + std::lock_guard lock(mutex); + const auto& unlocks = record({userId, serviceLabel}); + const auto found = unlocks.find(trophyId); + if (found == unlocks.end()) return std::nullopt; + return found->second; +} + +std::size_t TrophyStore::UnlockCount(std::int32_t userId, std::uint32_t serviceLabel) { + std::lock_guard lock(mutex); + return record({userId, serviceLabel}).size(); +} + +TrophyStore& TitleStore() { + static TrophyStore store(ResolvePath_nid_no_patch(TrophyRootGuest)); + return store; +} + +std::optional<std::uint64_t> ContextUnlockTick(int context, std::int32_t trophyId) { + const auto owner = FindContext(context); + if (!owner || trophyId < 0) return std::nullopt; + return TitleStore().UnlockTick(owner->userId, owner->serviceLabel, trophyId); +} + +std::size_t ContextUnlockCount(int context) { + const auto owner = FindContext(context); + return owner ? TitleStore().UnlockCount(owner->userId, owner->serviceLabel) : 0; +} + +} + +extern "C" bool NpTrophy2RecordUnlock_nid_no_patch(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId) { + const auto title = GetAppTitleId_nid_postfix(); + const std::string_view titleId(title.value, static_cast<std::size_t>(std::find(std::begin(title.value), std::end(title.value), '\0') - std::begin(title.value))); + const NpTrophy2Unlock unlock{std::string(titleId), userId, serviceLabel, trophyId, NpTrophy2::CurrentTick()}; + if (!NpTrophy2::TitleStore().Unlock(userId, serviceLabel, trophyId, unlock.tick)) return false; + std::fprintf(stderr, "[trophy] %s unlocked trophy %d for user %d\n", unlock.titleId.c_str(), trophyId, userId); + auto& registry = NpTrophy2::listeners(); + std::lock_guard dispatching(registry.dispatch); + std::vector<NpTrophy2::Listener> current; + { + std::lock_guard lock(registry.mutex); + current = registry.entries; + } + for (const auto& entry : current) { + if (NpTrophy2::registered(registry, entry.id)) entry.callback(entry.context, unlock); + } + const auto* package = NpTrophy2::Package(serviceLabel); + if (package == nullptr) return true; + if (const auto platinum = NpTrophy2::PlatinumToUnlock(*package, trophyId, NpTrophy2::UnlockedBy(userId, serviceLabel))) NpTrophy2RecordUnlock_nid_no_patch(userId, serviceLabel, *platinum); + return true; +} + +extern "C" std::uint64_t NpTrophy2AddUnlockListener_nid_no_patch(NpTrophy2UnlockListener listener, void* context) { + if (listener == nullptr) { + APS5_INVALID_ARG_EX; + } + auto& registry = NpTrophy2::listeners(); + std::lock_guard lock(registry.mutex); + const auto id = registry.next++; + registry.entries.push_back({id, listener, context}); + return id; +} + +extern "C" void NpTrophy2RemoveUnlockListener_nid_no_patch(std::uint64_t id) { + auto& registry = NpTrophy2::listeners(); + std::lock_guard dispatching(registry.dispatch); + std::lock_guard lock(registry.mutex); + const auto removed = std::erase_if(registry.entries, [id](const NpTrophy2::Listener& entry) { return entry.id == id; }); + if (removed == 0) throw std::invalid_argument("no trophy unlock listener has id " + std::to_string(id)); +} diff --git a/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp b/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp index 1611b0ba99..5ae534f6f6 100644 --- a/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp +++ b/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp @@ -1,5 +1,6 @@ #include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" #include "prx/libSceNpTrophy2/include/TrophyPackage.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include <cstddef> #include <cstdint> @@ -8,6 +9,7 @@ #include <filesystem> #include <optional> #include <random> +#include <set> #include <stdexcept> #include <string> #include <utility> @@ -241,8 +243,10 @@ void testPackage() { check(thrown, "language 31 has a tag"); } -void testProgressAndCounts() { +void testProgressAndPlatinum() { const auto package = NpTrophy2::ParseTrophyPackage(trophyPackage(), "en-US"); + std::set<std::int32_t> unlocked; + const NpTrophy2::UnlockedQuery isUnlocked = [&unlocked](std::int32_t id) { return unlocked.contains(id); }; const auto& progressive = package.trophies.at(5); check(NpTrophy2::TrophyProgress(progressive, std::nullopt, false) == 0 && NpTrophy2::TrophyProgress(progressive, 3, false) == 3 && NpTrophy2::TrophyProgress(progressive, 9, false) == 5 && NpTrophy2::TrophyProgress(progressive, 1, true) == 5, @@ -255,14 +259,24 @@ void testProgressAndCounts() { } check(thrown, "a trophy that is not progressive reported progress"); - const auto all = NpTrophy2::CountTrophies(package, std::nullopt); - check(all.trophies == 8 && all.unlocked == 0 && all.grades[NpTrophy2::GradePlatinum] == 1 && all.grades[NpTrophy2::GradeGold] == 1 && all.grades[NpTrophy2::GradeSilver] == 1 && - all.grades[NpTrophy2::GradeBronze] == 5, + unlocked = {1, 2, 3, 4}; + check(!NpTrophy2::PlatinumToUnlock(package, 4, isUnlocked), "the platinum unlocked before trophy 5"); + unlocked.insert(5); + check(NpTrophy2::PlatinumToUnlock(package, 5, isUnlocked) == std::optional<std::int32_t>(0), "the platinum did not unlock after the last trophy that names it"); + check(!NpTrophy2::PlatinumToUnlock(package, 6, isUnlocked) && !NpTrophy2::PlatinumToUnlock(package, 0, isUnlocked) && !NpTrophy2::PlatinumToUnlock(package, 42, isUnlocked), + "a trophy that names no platinum unlocked one"); + unlocked.insert(0); + check(!NpTrophy2::PlatinumToUnlock(package, 5, isUnlocked), "an unlocked platinum was unlocked again"); + + unlocked = {1, 6}; + const auto all = NpTrophy2::CountTrophies(package, std::nullopt, isUnlocked); + check(all.trophies == 8 && all.unlocked == 2 && all.grades[NpTrophy2::GradePlatinum] == 1 && all.grades[NpTrophy2::GradeGold] == 1 && all.grades[NpTrophy2::GradeSilver] == 1 && + all.grades[NpTrophy2::GradeBronze] == 5 && all.unlockedGrades[NpTrophy2::GradeGold] == 1 && all.unlockedGrades[NpTrophy2::GradeBronze] == 1, "the game counts are wrong"); - check(all.Percentage() == 0, "a game without unlocks has progress"); - const auto expansion = NpTrophy2::CountTrophies(package, 1); - check(expansion.trophies == 2 && expansion.unlocked == 0 && expansion.Percentage() == 0, "the group counts are wrong"); - check(NpTrophy2::CountTrophies(package, 5).trophies == 0 && NpTrophy2::CountTrophies(package, 5).Percentage() == 0, "an empty group has trophies"); + check(all.Percentage() == (6 + 1) * 100 / (6 + 2 + 5), "the progress percentage does not weigh gold 6, silver 2 and bronze 1"); + const auto expansion = NpTrophy2::CountTrophies(package, 1, isUnlocked); + check(expansion.trophies == 2 && expansion.unlocked == 1 && expansion.Percentage() == 50, "the group counts are wrong"); + check(NpTrophy2::CountTrophies(package, 5, isUnlocked).trophies == 0 && NpTrophy2::CountTrophies(package, 5, isUnlocked).Percentage() == 0, "an empty group has trophies"); } } @@ -275,7 +289,7 @@ int main() { testContainer(); testJson(); testPackage(); - testProgressAndCounts(); + testProgressAndPlatinum(); std::filesystem::current_path(previous); std::filesystem::remove_all(root); std::puts("Trophy package tests passed"); diff --git a/core/libs/prx/libSceNpTrophy2/tests/TrophyStore.cpp b/core/libs/prx/libSceNpTrophy2/tests/TrophyStore.cpp new file mode 100644 index 0000000000..6a29e7e9c8 --- /dev/null +++ b/core/libs/prx/libSceNpTrophy2/tests/TrophyStore.cpp @@ -0,0 +1,195 @@ +#include "prx/libSceNpTrophy2/include/NpTrophy2Types.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" + +#include <atomic> +#include <chrono> +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <filesystem> +#include <optional> +#include <random> +#include <stdexcept> +#include <string> +#include <thread> +#include <vector> + +extern "C" { +int APS5_VABI sceNpTrophy2CreateContext(int*, int, uint32_t, uint64_t); +int APS5_VABI sceNpTrophy2DestroyContext(int); +int APS5_VABI sceNpTrophy2GetGameInfo(int, int, NpTrophy2GameDetails*, NpTrophy2GameData*); +int APS5_VABI sceNpTrophy2GetTrophyInfo(int, int, int, NpTrophy2Details*, NpTrophy2Data*); +int APS5_VABI sceNpTrophy2GetTrophyInfoArray(int, int, uint32_t, uint32_t, NpTrophy2Details*, NpTrophy2Data*, uint32_t*); +} + +namespace { + +constexpr std::int32_t InitialUser = 0x10000000; + +void check(bool condition, const char* message) { + if (condition) return; + std::fprintf(stderr, "%s\n", message); + std::abort(); +} + +template <typename TAction> +bool throws(TAction action) { + try { + action(); + } catch (const std::exception&) { + return true; + } + return false; +} + +std::string read(const char* path) { + std::string text; + std::FILE* file = std::fopen(path, "rb"); + if (file == nullptr) return text; + char buffer[256]; + for (std::size_t count; (count = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, count); + std::fclose(file); + return text; +} + +void write(const char* path, const std::string& text) { + std::FILE* file = std::fopen(path, "wb"); + check(file != nullptr, "test file was not created"); + const bool written = std::fwrite(text.data(), 1, text.size(), file) == text.size(); + check(std::fclose(file) == 0 && written, "test file was not written"); +} + +void testPersistence() { + const std::filesystem::path directory("store"); + { + NpTrophy2::TrophyStore store(directory); + check(!store.UnlockTick(InitialUser, 0, 12) && store.UnlockCount(InitialUser, 0) == 0, "a new store has unlocks"); + check(store.Unlock(InitialUser, 0, 12, 1000), "trophy 12 was not unlocked"); + check(!store.Unlock(InitialUser, 0, 12, 2000), "trophy 12 was unlocked twice"); + check(store.Unlock(InitialUser, 0, 3, 3000), "trophy 3 was not unlocked"); + check(store.Unlock(InitialUser + 1, 0, 12, 4000) && store.Unlock(InitialUser, 1, 12, 5000), "another user or service label shares unlocks"); + check(throws([&store] { store.Unlock(InitialUser, 0, -1, 6000); }), "a negative trophy id was unlocked"); + check(store.RecordPath(InitialUser, 0) == directory / "268435456_0.txt", "the record is not named after the user and service label"); + } + check(read("store/268435456_0.txt") == "3 3000\n12 1000\n", "the record does not list the unlocks"); + NpTrophy2::TrophyStore restarted(directory); + check(restarted.UnlockTick(InitialUser, 0, 12) == std::optional<std::uint64_t>(1000) && restarted.UnlockTick(InitialUser, 0, 3) == std::optional<std::uint64_t>(3000), + "unlock times did not survive a restart"); + check(restarted.UnlockCount(InitialUser, 0) == 2 && restarted.UnlockCount(InitialUser + 1, 0) == 1 && restarted.UnlockCount(InitialUser, 1) == 1 && restarted.UnlockCount(InitialUser, 2) == 0, + "unlock counts did not survive a restart"); + check(!restarted.Unlock(InitialUser, 0, 12, 7000) && restarted.UnlockTick(InitialUser, 0, 12) == std::optional<std::uint64_t>(1000), "a restart forgot an unlock"); + check(restarted.Unlock(InitialUser, 0, 7, 8000) && read("store/268435456_0.txt") == "3 3000\n7 8000\n12 1000\n", "an unlock after a restart lost earlier unlocks"); +} + +void testDamagedRecords() { + const std::filesystem::path directory("damaged"); + for (const char* text : {"12\n", "12 x\n", "x 5\n", "-1 5\n", "12 5\n12 6\n", "12 5 7\n", "\n", "12 5\r\n"}) { + std::filesystem::remove_all(directory); + std::filesystem::create_directories(directory); + write("damaged/268435456_0.txt", text); + NpTrophy2::TrophyStore store(directory); + check(throws([&store] { store.UnlockCount(InitialUser, 0); }), "a damaged record was read"); + } +} + +struct Seen { + std::vector<NpTrophy2Unlock> unlocks; +}; + +void remember(void* context, const NpTrophy2Unlock& unlock) { + static_cast<Seen*>(context)->unlocks.push_back(unlock); +} + +NpTrophy2Data trophyData(int context, int trophyId) { + NpTrophy2Data data{}; + check(sceNpTrophy2GetTrophyInfo(context, 1, trophyId, nullptr, &data) == 0, "GetTrophyInfo failed"); + check(data.trophy_id == trophyId, "GetTrophyInfo answered for another trophy"); + return data; +} + +std::uint32_t unlockedTrophies(int context) { + NpTrophy2GameData data{}; + check(sceNpTrophy2GetGameInfo(context, 1, nullptr, &data) == 0, "GetGameInfo failed"); + return data.unlocked_trophies; +} + +struct SlowListener { + std::atomic<bool> running{false}; + std::atomic<bool> finished{false}; +}; + +void slowly(void* context, const NpTrophy2Unlock&) { + auto* state = static_cast<SlowListener*>(context); + state->running = true; + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + state->finished = true; +} + +void testListenerRemovalWaitsForItsCallback() { + SlowListener state; + const auto listener = NpTrophy2AddUnlockListener_nid_no_patch(slowly, &state); + std::thread unlocking([] { NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 2, 1); }); + while (!state.running) std::this_thread::yield(); + NpTrophy2RemoveUnlockListener_nid_no_patch(listener); + const bool finished = state.finished; + unlocking.join(); + check(finished, "a listener was removed while its callback was still running, so its context could be freed under it"); +} + +void testQueries() { + int context = 0; + check(sceNpTrophy2CreateContext(&context, InitialUser, 0, 0) == 0, "context was not created"); + int other = 0; + check(sceNpTrophy2CreateContext(&other, InitialUser + 1, 0, 0) == 0 && other != context, "second context was not created"); + const auto locked = trophyData(context, 12); + check(!locked.unlocked && locked.timestamp_tick == 0, "trophy 12 is unlocked before its event"); + check(unlockedTrophies(context) == 0, "the game has unlocks before any event"); + + Seen seen; + const auto listener = NpTrophy2AddUnlockListener_nid_no_patch(remember, &seen); + const auto before = NpTrophy2::CurrentTick(); + check(NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 0, 12), "trophy 12 was not recorded"); + check(!NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 0, 12), "trophy 12 was recorded twice"); + check(NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 0, 0), "trophy 0 was not recorded"); + const auto after = NpTrophy2::CurrentTick(); + NpTrophy2RemoveUnlockListener_nid_no_patch(listener); + check(NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 0, 5) && seen.unlocks.size() == 2, "the listener did not see each new unlock once"); + const auto& first = seen.unlocks.front(); + check(first.titleId == "PPSA00000" && first.userId == InitialUser && first.serviceLabel == 0 && first.trophyId == 12 && first.tick >= before && first.tick <= after, + "the listener did not get the unlock's title, user, trophy and time"); + check(before > NpTrophy2::UnixEpochTick, "the current tick is not counted from year 1"); + + const auto unlocked = trophyData(context, 12); + check(unlocked.unlocked && unlocked.timestamp_tick == first.tick, "trophy 12 is not reported unlocked at its time"); + check(!trophyData(context, 13).unlocked, "trophy 13 is reported unlocked"); + check(unlockedTrophies(context) == 3, "the game does not count its unlocks"); + NpTrophy2Data listed{}; + std::uint32_t count = 0; + check(sceNpTrophy2GetTrophyInfoArray(context, 1, 0, 1, nullptr, &listed, &count) == 0 && count == 1 && listed.trophy_id == 0 && listed.unlocked, + "the trophy list does not report trophy 0 unlocked"); + check(!trophyData(other, 12).unlocked && unlockedTrophies(other) == 0, "another user sees the unlocks"); + check(read("_trophy/268435456_0.txt").find("12 ") != std::string::npos, "the unlock was not stored in the title folder"); + check(throws([] { NpTrophy2AddUnlockListener_nid_no_patch(nullptr, nullptr); }), "a null listener was added"); + check(throws([listener] { NpTrophy2RemoveUnlockListener_nid_no_patch(listener); }), "a removed listener was removed again"); + check(sceNpTrophy2DestroyContext(context) == 0 && sceNpTrophy2DestroyContext(other) == 0, "contexts were not destroyed"); +} + +} + +int main() { + const auto previous = std::filesystem::current_path(); + const auto root = std::filesystem::temp_directory_path() / ("anyps5-trophy-store-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(root / "app0" / "sce_sys"); + std::filesystem::current_path(root); + write("app0/sce_sys/param.json", R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}}})"); + testPersistence(); + testDamagedRecords(); + testQueries(); + testListenerRemovalWaitsForItsCallback(); + std::filesystem::current_path(previous); + std::filesystem::remove_all(root); + std::puts("Trophy store tests passed"); + return 0; +} diff --git a/docs/dev/debt/libSceNpTrophy2.md b/docs/dev/debt/libSceNpTrophy2.md index a92744d72c..e5060029a4 100644 --- a/docs/dev/debt/libSceNpTrophy2.md +++ b/docs/dev/debt/libSceNpTrophy2.md @@ -2,7 +2,7 @@ ### Silent stubs -- [libSceNpTrophy2](../../../core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp): `sceNpTrophy2RegisterUnlockCallback` accepts a callback that is never called, as no trophy is ever unlocked, `sceNpTrophy2UnregisterUnlockCallback` only returns success, and `sceNpTrophy2ShowTrophyList` returns success without showing a list (there is no system trophy UI; seen on PPSA15552). +- [libSceNpTrophy2](../../../core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp): `sceNpTrophy2RegisterUnlockCallback` accepts a callback that is never called, as the callback's signature and when the console calls it are not known, `sceNpTrophy2UnregisterUnlockCallback` only returns success, and `sceNpTrophy2ShowTrophyList` returns success without showing a list (there is no system trophy UI; seen on PPSA15552). ### Unknown function info @@ -12,6 +12,7 @@ ### Functional -- [Trophy queries](../../../core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp) report the title's [trophy data](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp) (`sce_sys/trophy2/trophyNN.ucp`). Without the file, or when it is malformed, they report the old placeholders and `sceNpTrophy2GetTrophyInfoArray` lists only trophy 0. Icons are not read. Hidden trophies report their names and descriptions; whether the console blanks them before the unlock is not known. `progress_percentage` weighs gold 6, silver 2 and bronze 1 and leaves the platinum out, as KytyPS5 does (the ratio of the published PSN trophy points); the console's formula is not known. A trophy or group that is not in the file throws. Unlocks are not recorded, so every trophy reports as locked, with no unlock time and a progress of 0, and the unlocked counts and `progress_percentage` are 0. A context that was not created or was destroyed reports placeholders and returns success; the console's errors for it are unknown. +- [Trophy queries](../../../core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp) report the title's [trophy data](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp) (`sce_sys/trophy2/trophyNN.ucp`) and the unlocks in the [trophy store](../../../core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp). Without the file, or when it is malformed, they report the old placeholders and `sceNpTrophy2GetTrophyInfoArray` lists only trophy 0. Icons are not read. Hidden trophies report their names and descriptions; whether the console blanks them before the unlock is not known. `progress_percentage` weighs gold 6, silver 2 and bronze 1 and leaves the platinum out, as KytyPS5 does (the ratio of the published PSN trophy points); the console's formula is not known. A trophy or group that is not in the file throws. A context that was not created or was destroyed reports placeholders and no unlocks and returns success; the console's errors for it are unknown. - [Trophy data limits](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): a `trophyNN.ucp` larger than 256 MiB or listing more than 4096 files, and a JSON file in it that nests deeper than 64 levels or holds more than 1048576 values, are not used, and the title runs as without that file. The limits keep the memory the reader takes to the file and about 140 MiB of parsed JSON; the console's limits are not known. - [Text language](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): texts come from `tropmeta_<tag>.json` for the console language, then `en-US`, then the file's `defaultLanguage`. The tags for languages 0 to 29 are KytyPS5's; `uk-UA` for 30 (Ukrainian) is an assumption. +- [Platinum](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp): an unlock through any path unlocks the platinum that the trophy names in `platinumTrophyId` once every trophy naming that platinum is unlocked, as KytyPS5 does; trophies without `platinumTrophyId` are not required. Without trophy data any id of 0 or more is stored, and the platinum is not unlocked when the trophies it needs are. From 98c700ea02f40d3f73efca9b62c16817a96677b4 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:35:51 +0000 Subject: [PATCH 071/103] feat(libSceNpUniversalDataSystem): keep event properties and record posted trophy unlocks Events and property objects keep their typed properties instead of dropping them. PostEvent records an _UnlockTrophy event with an int32 _trophy_id for the user and service label of its context, logs why a trophy event is ignored, and prints every posted event with APS5_TRACE_UDS. --- core/libs/SceTypes.hpp | 4 - .../CMakeLists.txt | 31 ++- .../include/UdsEvent.hpp | 65 +++++ .../src/Context.cpp | 39 ++- .../libSceNpUniversalDataSystem/src/Event.cpp | 65 ++++- .../src/EventPropertyArray.cpp | 38 ++- .../src/EventPropertyObject.cpp | 38 ++- .../src/NpUniversalDataSystem.hpp | 46 +++ .../src/UdsEvent.cpp | 147 ++++++++++ .../tests/Events.cpp | 262 ++++++++++++++++++ core/libs/tests/GuestNpTrophyData.cpp | 59 ++++ core/libs/tests/GuestNpTrophyUnlocks.cpp | 109 ++++++++ docs/dev/debt/libSceNpTrophy2.md | 2 +- docs/dev/debt/libSceNpUniversalDataSystem.md | 9 + 14 files changed, 886 insertions(+), 28 deletions(-) create mode 100644 core/libs/prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp create mode 100644 core/libs/prx/libSceNpUniversalDataSystem/src/UdsEvent.cpp create mode 100644 core/libs/prx/libSceNpUniversalDataSystem/tests/Events.cpp create mode 100644 core/libs/tests/GuestNpTrophyUnlocks.cpp create mode 100644 docs/dev/debt/libSceNpUniversalDataSystem.md diff --git a/core/libs/SceTypes.hpp b/core/libs/SceTypes.hpp index 5d444a5034..237e9166d9 100644 --- a/core/libs/SceTypes.hpp +++ b/core/libs/SceTypes.hpp @@ -1380,10 +1380,6 @@ struct NpUniversalDataSystemMemoryStat { std::size_t current_inuse_size; }; -struct NpUniversalDataSystemEvent {}; -struct NpUniversalDataSystemEventPropertyObject {}; -struct NpUniversalDataSystemEventPropertyArray {}; - struct NpUniversalDataSystemStorageStat { std::size_t in_events; std::size_t out_events; diff --git a/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt b/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt index a7f5848c61..d7b5a34d12 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt +++ b/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt @@ -5,16 +5,45 @@ add_library(libSceNpUniversalDataSystem SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/src/Event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/EventPropertyObject.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/EventPropertyArray.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/UdsEvent.cpp ) target_include_directories(libSceNpUniversalDataSystem PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/src ) -target_link_libraries(libSceNpUniversalDataSystem PRIVATE libc) +target_link_libraries(libSceNpUniversalDataSystem PRIVATE libSceNpTrophy2 libc) configure_windows_unwind(libSceNpUniversalDataSystem) +add_test_executable(np_uds_events_tests EXCLUDE_FROM_ALL + tests/Events.cpp + src/Context.cpp + src/Event.cpp + src/EventPropertyArray.cpp + src/EventPropertyObject.cpp + src/UdsEvent.cpp +) +target_include_directories(np_uds_events_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/src) +target_link_libraries(np_uds_events_tests PRIVATE libSceNpTrophy2 libc) +configure_windows_unwind(np_uds_events_tests) if(BUILD_TESTING) + set_target_properties(np_uds_events_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME np_uds_events COMMAND np_uds_events_tests) + if(WIN32) + set_tests_properties(np_uds_events PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpTrophy2>") + endif() + + add_test_executable(guest_np_trophy_unlocks_tests ${LIBS_INCLUDE_DIR}/tests/GuestNpTrophyUnlocks.cpp) + target_include_directories(guest_np_trophy_unlocks_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_np_trophy_unlocks_tests PRIVATE libSceNpUniversalDataSystem libSceNpTrophy2 libc) + configure_windows_unwind(guest_np_trophy_unlocks_tests) + add_test(NAME guest_np_trophy_unlocks COMMAND guest_np_trophy_unlocks_tests) + if(WIN32) + set_tests_properties(guest_np_trophy_unlocks PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpUniversalDataSystem>") + endif() + add_test_executable(guest_np_trophy_data_tests ${LIBS_INCLUDE_DIR}/tests/GuestNpTrophyData.cpp) target_include_directories(guest_np_trophy_data_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_np_trophy_data_tests PRIVATE libSceNpUniversalDataSystem libSceNpTrophy2 libc) diff --git a/core/libs/prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp b/core/libs/prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp new file mode 100644 index 0000000000..d3cae5efb2 --- /dev/null +++ b/core/libs/prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp @@ -0,0 +1,65 @@ +#ifndef CORE_LIBS_PRX_LIBSCENPUNIVERSALDATASYSTEM_INCLUDE_UDSEVENT_HPP +#define CORE_LIBS_PRX_LIBSCENPUNIVERSALDATASYSTEM_INCLUDE_UDSEVENT_HPP + +#include <cstdint> +#include <memory> +#include <string> +#include <string_view> +#include <utility> +#include <variant> +#include <vector> + +struct NpUniversalDataSystemEventPropertyObject; +struct NpUniversalDataSystemEventPropertyArray; + +namespace Uds { + +using Binary = std::vector<std::uint8_t>; +using Value = std::variant<std::int32_t, std::uint32_t, std::int64_t, std::uint64_t, float, double, bool, std::string, Binary, + std::shared_ptr<NpUniversalDataSystemEventPropertyObject>, std::shared_ptr<NpUniversalDataSystemEventPropertyArray>>; + +Value Clone(const Value& value); + +} + +struct NpUniversalDataSystemEventPropertyObject { + std::vector<std::pair<std::string, Uds::Value>> properties; + + Uds::Value& Set(std::string_view key, Uds::Value value); + const Uds::Value* Find(std::string_view key) const; + NpUniversalDataSystemEventPropertyObject Clone() const; +}; + +struct NpUniversalDataSystemEventPropertyArray { + std::vector<Uds::Value> values; + + Uds::Value& Add(Uds::Value value); + NpUniversalDataSystemEventPropertyArray Clone() const; +}; + +struct NpUniversalDataSystemEvent { + std::string name; + NpUniversalDataSystemEventPropertyObject properties; +}; + +namespace Uds { + +inline constexpr std::string_view UnlockTrophyEvent = "_UnlockTrophy"; +inline constexpr std::string_view UpdateTrophyProgressEvent = "_UpdateTrophyProgress"; +inline constexpr std::string_view TrophyIdProperty = "_trophy_id"; + +enum class TrophyEventKind { None, Unlock, Progress, Malformed }; + +struct TrophyEvent { + TrophyEventKind kind = TrophyEventKind::None; + std::int32_t trophyId = -1; + std::string problem; +}; + +TrophyEvent RecognizeTrophyEvent(const NpUniversalDataSystemEvent& event); +std::string Describe(const NpUniversalDataSystemEventPropertyObject& object); +std::string Describe(const NpUniversalDataSystemEvent& event); + +} + +#endif diff --git a/core/libs/prx/libSceNpUniversalDataSystem/src/Context.cpp b/core/libs/prx/libSceNpUniversalDataSystem/src/Context.cpp index da087ae06f..f73c4a2cb2 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/src/Context.cpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/src/Context.cpp @@ -1,21 +1,58 @@ #include <cstdint> +#include <map> +#include <mutex> +#include <optional> #include <stdexcept> +#include <string> #include "NpUniversalDataSystem.hpp" #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace Uds { + +namespace { + +struct Contexts { + std::mutex mutex; + std::map<int, ContextOwner> owners; + int next = NP_UNIVERSAL_DATA_SYSTEM_CONTEXT_DEFAULT; +}; + +Contexts& contexts() { + static Contexts table; + return table; +} + +} + +std::optional<ContextOwner> FindContext(int context) { + auto& table = contexts(); + std::lock_guard lock(table.mutex); + const auto found = table.owners.find(context); + if (found == table.owners.end()) return std::nullopt; + return found->second; +} + +} + extern "C" { int APS5_VABI sceNpUniversalDataSystemCreateContext(int* context, int user_id, uint32_t service_label, uint64_t options) { if (context == nullptr) { throw std::invalid_argument(std::string(__func__) + ": invalid argument context"); } - *context = NP_UNIVERSAL_DATA_SYSTEM_CONTEXT_DEFAULT; + auto& table = Uds::contexts(); + std::lock_guard lock(table.mutex); + *context = table.next++; + table.owners[*context] = {user_id, service_label}; return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } int APS5_VABI sceNpUniversalDataSystemDestroyContext(int context) { + auto& table = Uds::contexts(); + std::lock_guard lock(table.mutex); + table.owners.erase(context); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } diff --git a/core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp b/core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp index ae5ea0f668..0ead101091 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp @@ -1,13 +1,59 @@ #include <cstddef> #include <cstdint> +#include <cstdio> +#include <cstdlib> #include <cstring> +#include <memory> +#include <mutex> +#include <set> #include <stdexcept> #include <string> #include "NpUniversalDataSystem.hpp" #include "SceTypes.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include "prx/libc/include/General.hpp" +namespace { + +Uds::Owned<NpUniversalDataSystemEvent>& events() { + static Uds::Owned<NpUniversalDataSystemEvent> registry; + return registry; +} + +bool firstProgress(std::int32_t trophyId) { + static std::mutex mutex; + static std::set<std::int32_t> logged; + std::lock_guard lock(mutex); + return logged.insert(trophyId).second; +} + +void ignore(const NpUniversalDataSystemEvent& event, int context, const std::string& reason) { + std::fprintf(stderr, "[trophy] ignored %s posted to context %d: %s\n", Uds::Describe(event).c_str(), context, reason.c_str()); +} + +void post(const NpUniversalDataSystemEvent& event, int context) { + static const bool trace = std::getenv("APS5_TRACE_UDS") != nullptr; + if (trace) std::fprintf(stderr, "[uds] context %d posted %s\n", context, Uds::Describe(event).c_str()); + const auto trophy = Uds::RecognizeTrophyEvent(event); + switch (trophy.kind) { + case Uds::TrophyEventKind::None: + return; + case Uds::TrophyEventKind::Malformed: + ignore(event, context, trophy.problem); + return; + case Uds::TrophyEventKind::Progress: + if (firstProgress(trophy.trophyId)) ignore(event, context, "trophy progress is not recorded"); + return; + case Uds::TrophyEventKind::Unlock: + if (const auto owner = Uds::FindContext(context)) NpTrophy2RecordUnlock_nid_no_patch(owner->userId, owner->serviceLabel, trophy.trophyId); + else ignore(event, context, "the context does not exist"); + return; + } +} + +} + extern "C" { int APS5_VABI sceNpUniversalDataSystemCreateEvent( @@ -19,21 +65,30 @@ int APS5_VABI sceNpUniversalDataSystemCreateEvent( if (event_name == nullptr || new_event == nullptr) { APS5_INVALID_ARG_EX; } - *new_event = new NpUniversalDataSystemEvent; + auto event = std::make_unique<NpUniversalDataSystemEvent>(); + event->name = event_name; + if (prop != nullptr) event->properties = prop->Clone(); + *new_event = events().Adopt(std::move(event)); if (prop_ptr != nullptr) { - *prop_ptr = (prop != nullptr - ? const_cast<NpUniversalDataSystemEventPropertyObject*>(prop) - : new NpUniversalDataSystemEventPropertyObject); + *prop_ptr = &(*new_event)->properties; } return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } int APS5_VABI sceNpUniversalDataSystemDestroyEvent(NpUniversalDataSystemEvent* event) { - delete event; + events().Release(event); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } int APS5_VABI sceNpUniversalDataSystemPostEvent(int context, int handle, const void* event, uint64_t options) { + if (event == nullptr) { + APS5_INVALID_ARG_EX; + } + const auto* posted = static_cast<const NpUniversalDataSystemEvent*>(event); + if (!events().Contains(posted)) { + throw std::invalid_argument(std::string(__func__) + ": the event was not created by sceNpUniversalDataSystemCreateEvent or was destroyed"); + } + post(*posted, context); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } diff --git a/core/libs/prx/libSceNpUniversalDataSystem/src/EventPropertyArray.cpp b/core/libs/prx/libSceNpUniversalDataSystem/src/EventPropertyArray.cpp index fd1c988b32..57e985ba8f 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/src/EventPropertyArray.cpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/src/EventPropertyArray.cpp @@ -1,21 +1,33 @@ #include <cstddef> #include <cstdint> +#include <memory> +#include <string> +#include <utility> #include "NpUniversalDataSystem.hpp" #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +Uds::Owned<NpUniversalDataSystemEventPropertyArray>& arrays() { + static Uds::Owned<NpUniversalDataSystemEventPropertyArray> registry; + return registry; +} + +} + extern "C" { int APS5_VABI sceNpUniversalDataSystemCreateEventPropertyArray(NpUniversalDataSystemEventPropertyArray** new_array) { if (new_array == nullptr) { APS5_INVALID_ARG_EX; } - *new_array = new NpUniversalDataSystemEventPropertyArray; + *new_array = arrays().Adopt(std::make_unique<NpUniversalDataSystemEventPropertyArray>()); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } int APS5_VABI sceNpUniversalDataSystemDestroyEventPropertyArray(NpUniversalDataSystemEventPropertyArray* array) { - delete array; + arrays().Release(array); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -25,6 +37,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetString( if (array == nullptr || value == nullptr) { APS5_INVALID_ARG_EX; } + array->Add(std::string(value)); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -34,6 +47,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetInt32( if (array == nullptr) { APS5_INVALID_ARG_EX; } + array->Add(value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -43,6 +57,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetUInt32( if (array == nullptr) { APS5_INVALID_ARG_EX; } + array->Add(value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -52,6 +67,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetInt64( if (array == nullptr) { APS5_INVALID_ARG_EX; } + array->Add(value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -61,6 +77,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetUInt64( if (array == nullptr) { APS5_INVALID_ARG_EX; } + array->Add(value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -70,6 +87,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetFloat32( if (array == nullptr) { APS5_INVALID_ARG_EX; } + array->Add(value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -79,6 +97,7 @@ NpUniversalDataSystemEventPropertyArray* array, double value) if (array == nullptr) { APS5_INVALID_ARG_EX; } + array->Add(value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -88,6 +107,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetBool( if (array == nullptr) { APS5_INVALID_ARG_EX; } + array->Add(value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -97,6 +117,8 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetBinary( if (array == nullptr || value == nullptr) { APS5_INVALID_ARG_EX; } + const auto* bytes = static_cast<const std::uint8_t*>(value); + array->Add(Uds::Binary(bytes, bytes + value_size)); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -108,11 +130,11 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetObject( if (array == nullptr) { APS5_INVALID_ARG_EX; } + auto child = std::make_shared<NpUniversalDataSystemEventPropertyObject>(value != nullptr ? value->Clone() : NpUniversalDataSystemEventPropertyObject{}); if (value_ptr != nullptr) { - *value_ptr = (value != nullptr - ? const_cast<NpUniversalDataSystemEventPropertyObject*>(value) - : new NpUniversalDataSystemEventPropertyObject); + *value_ptr = child.get(); } + array->Add(std::move(child)); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -124,11 +146,11 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetArray( if (array == nullptr) { APS5_INVALID_ARG_EX; } + auto child = std::make_shared<NpUniversalDataSystemEventPropertyArray>(value != nullptr ? value->Clone() : NpUniversalDataSystemEventPropertyArray{}); if (value_ptr != nullptr) { - *value_ptr = (value != nullptr - ? const_cast<NpUniversalDataSystemEventPropertyArray*>(value) - : new NpUniversalDataSystemEventPropertyArray); + *value_ptr = child.get(); } + array->Add(std::move(child)); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } diff --git a/core/libs/prx/libSceNpUniversalDataSystem/src/EventPropertyObject.cpp b/core/libs/prx/libSceNpUniversalDataSystem/src/EventPropertyObject.cpp index 48fc34544c..d316c1ea30 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/src/EventPropertyObject.cpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/src/EventPropertyObject.cpp @@ -1,21 +1,33 @@ #include <cstddef> #include <cstdint> +#include <memory> +#include <string> +#include <utility> #include "NpUniversalDataSystem.hpp" #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +Uds::Owned<NpUniversalDataSystemEventPropertyObject>& objects() { + static Uds::Owned<NpUniversalDataSystemEventPropertyObject> registry; + return registry; +} + +} + extern "C" { int APS5_VABI sceNpUniversalDataSystemCreateEventPropertyObject(NpUniversalDataSystemEventPropertyObject** new_object) { if (new_object == nullptr) { APS5_INVALID_ARG_EX; } - *new_object = new NpUniversalDataSystemEventPropertyObject; + *new_object = objects().Adopt(std::make_unique<NpUniversalDataSystemEventPropertyObject>()); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } int APS5_VABI sceNpUniversalDataSystemDestroyEventPropertyObject(NpUniversalDataSystemEventPropertyObject* object) { - delete object; + objects().Release(object); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -25,6 +37,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetString( if (object == nullptr || key == nullptr || value == nullptr) { APS5_INVALID_ARG_EX; } + object->Set(key, std::string(value)); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -34,6 +47,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetInt32( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + object->Set(key, value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -43,6 +57,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetUInt32( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + object->Set(key, value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -52,6 +67,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetInt64( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + object->Set(key, value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -61,6 +77,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetUInt64( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + object->Set(key, value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -70,6 +87,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetFloat32( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + object->Set(key, value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -79,6 +97,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetFloat64( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + object->Set(key, value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -88,6 +107,7 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetBool( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + object->Set(key, value); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -97,6 +117,8 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetBinary( if (object == nullptr || key == nullptr || value == nullptr) { APS5_INVALID_ARG_EX; } + const auto* bytes = static_cast<const std::uint8_t*>(value); + object->Set(key, Uds::Binary(bytes, bytes + value_size)); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -109,11 +131,11 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetObject( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + auto child = std::make_shared<NpUniversalDataSystemEventPropertyObject>(value != nullptr ? value->Clone() : NpUniversalDataSystemEventPropertyObject{}); if (value_ptr != nullptr) { - *value_ptr = (value != nullptr - ? const_cast<NpUniversalDataSystemEventPropertyObject*>(value) - : new NpUniversalDataSystemEventPropertyObject); + *value_ptr = child.get(); } + object->Set(key, std::move(child)); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } @@ -126,11 +148,11 @@ int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetArray( if (object == nullptr || key == nullptr) { APS5_INVALID_ARG_EX; } + auto child = std::make_shared<NpUniversalDataSystemEventPropertyArray>(value != nullptr ? value->Clone() : NpUniversalDataSystemEventPropertyArray{}); if (value_ptr != nullptr) { - *value_ptr = (value != nullptr - ? const_cast<NpUniversalDataSystemEventPropertyArray*>(value) - : new NpUniversalDataSystemEventPropertyArray); + *value_ptr = child.get(); } + object->Set(key, std::move(child)); return SCE_NP_UNIVERSAL_DATA_SYSTEM_OK; } diff --git a/core/libs/prx/libSceNpUniversalDataSystem/src/NpUniversalDataSystem.hpp b/core/libs/prx/libSceNpUniversalDataSystem/src/NpUniversalDataSystem.hpp index 4a02cb68e0..e89083d5bd 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/src/NpUniversalDataSystem.hpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/src/NpUniversalDataSystem.hpp @@ -3,8 +3,14 @@ #include <cstddef> #include <cstdint> +#include <map> +#include <memory> +#include <mutex> +#include <optional> +#include <utility> #include "SceTypes.hpp" +#include "prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp" static constexpr int SCE_NP_UNIVERSAL_DATA_SYSTEM_OK = 0; static constexpr int SCE_NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT = -2141900542; @@ -15,4 +21,44 @@ static constexpr int NP_UNIVERSAL_DATA_SYSTEM_HANDLE_DEFAULT = 1; static constexpr const char* NP_UNIVERSAL_DATA_SYSTEM_EMPTY_JSON = "{}"; static constexpr size_t NP_UNIVERSAL_DATA_SYSTEM_EMPTY_EVENT_SIZE = 3; +namespace Uds { + +template <typename TValue> +class Owned { +public: + TValue* Adopt(std::unique_ptr<TValue> value) { + std::lock_guard lock(mutex); + auto* pointer = value.get(); + values.emplace(pointer, std::move(value)); + return pointer; + } + + void Release(const TValue* value) { + std::unique_ptr<TValue> released; + std::lock_guard lock(mutex); + const auto found = values.find(value); + if (found == values.end()) return; + released = std::move(found->second); + values.erase(found); + } + + bool Contains(const TValue* value) { + std::lock_guard lock(mutex); + return values.contains(value); + } + +private: + std::mutex mutex; + std::map<const TValue*, std::unique_ptr<TValue>> values; +}; + +struct ContextOwner { + std::int32_t userId = 0; + std::uint32_t serviceLabel = 0; +}; + +std::optional<ContextOwner> FindContext(int context); + +} + #endif diff --git a/core/libs/prx/libSceNpUniversalDataSystem/src/UdsEvent.cpp b/core/libs/prx/libSceNpUniversalDataSystem/src/UdsEvent.cpp new file mode 100644 index 0000000000..96b8672b36 --- /dev/null +++ b/core/libs/prx/libSceNpUniversalDataSystem/src/UdsEvent.cpp @@ -0,0 +1,147 @@ +#include "prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp" + +#include <cstdio> +#include <type_traits> + +namespace Uds { + +Value Clone(const Value& value) { + if (const auto* object = std::get_if<std::shared_ptr<NpUniversalDataSystemEventPropertyObject>>(&value)) return std::make_shared<NpUniversalDataSystemEventPropertyObject>((*object)->Clone()); + if (const auto* array = std::get_if<std::shared_ptr<NpUniversalDataSystemEventPropertyArray>>(&value)) return std::make_shared<NpUniversalDataSystemEventPropertyArray>((*array)->Clone()); + return value; +} + +namespace { + +void appendQuoted(std::string& out, std::string_view text) { + out += '"'; + for (const char character : text) { + if (character == '"' || character == '\\') { + out += '\\'; + out += character; + } else if (static_cast<unsigned char>(character) < 0x20) { + char escaped[8]; + std::snprintf(escaped, sizeof(escaped), "\\u%04x", static_cast<unsigned>(static_cast<unsigned char>(character))); + out += escaped; + } else { + out += character; + } + } + out += '"'; +} + +void appendValue(std::string& out, const Value& value); + +void appendObject(std::string& out, const NpUniversalDataSystemEventPropertyObject& object) { + out += '{'; + bool first = true; + for (const auto& [key, value] : object.properties) { + if (!first) out += ','; + first = false; + appendQuoted(out, key); + out += ':'; + appendValue(out, value); + } + out += '}'; +} + +void appendValue(std::string& out, const Value& value) { + std::visit([&](const auto& held) { + using Held = std::decay_t<decltype(held)>; + char number[40]; + if constexpr (std::is_same_v<Held, bool>) { + out += held ? "true" : "false"; + } else if constexpr (std::is_same_v<Held, std::int32_t> || std::is_same_v<Held, std::int64_t>) { + std::snprintf(number, sizeof(number), "%lld", static_cast<long long>(held)); + out += number; + } else if constexpr (std::is_same_v<Held, std::uint32_t> || std::is_same_v<Held, std::uint64_t>) { + std::snprintf(number, sizeof(number), "%llu", static_cast<unsigned long long>(held)); + out += number; + } else if constexpr (std::is_same_v<Held, float>) { + std::snprintf(number, sizeof(number), "%.9g", static_cast<double>(held)); + out += number; + } else if constexpr (std::is_same_v<Held, double>) { + std::snprintf(number, sizeof(number), "%.17g", held); + out += number; + } else if constexpr (std::is_same_v<Held, std::string>) { + appendQuoted(out, held); + } else if constexpr (std::is_same_v<Held, Binary>) { + std::snprintf(number, sizeof(number), "<%zu bytes>", held.size()); + out += number; + } else if constexpr (std::is_same_v<Held, std::shared_ptr<NpUniversalDataSystemEventPropertyObject>>) { + appendObject(out, *held); + } else { + out += '['; + for (std::size_t i = 0; i < held->values.size(); ++i) { + if (i != 0) out += ','; + appendValue(out, held->values[i]); + } + out += ']'; + } + }, value); +} + +} + +TrophyEvent RecognizeTrophyEvent(const NpUniversalDataSystemEvent& event) { + TrophyEvent result; + if (event.name == UnlockTrophyEvent) result.kind = TrophyEventKind::Unlock; + else if (event.name == UpdateTrophyProgressEvent) result.kind = TrophyEventKind::Progress; + else return result; + const auto* id = event.properties.Find(TrophyIdProperty); + const auto* value = id != nullptr ? std::get_if<std::int32_t>(id) : nullptr; + if (value != nullptr && *value >= 0) { + result.trophyId = *value; + } else if (result.kind == TrophyEventKind::Unlock) { + result.kind = TrophyEventKind::Malformed; + result.problem = id == nullptr ? "it has no _trophy_id" : value == nullptr ? "its _trophy_id is not an int32" : "its _trophy_id is negative"; + } + return result; +} + +std::string Describe(const NpUniversalDataSystemEventPropertyObject& object) { + std::string out; + appendObject(out, object); + return out; +} + +std::string Describe(const NpUniversalDataSystemEvent& event) { + return event.name + " " + Describe(event.properties); +} + +} + +Uds::Value& NpUniversalDataSystemEventPropertyObject::Set(std::string_view key, Uds::Value value) { + for (auto& [name, stored] : properties) { + if (name == key) { + stored = std::move(value); + return stored; + } + } + return properties.emplace_back(std::string(key), std::move(value)).second; +} + +const Uds::Value* NpUniversalDataSystemEventPropertyObject::Find(std::string_view key) const { + for (const auto& [name, stored] : properties) { + if (name == key) return &stored; + } + return nullptr; +} + +NpUniversalDataSystemEventPropertyObject NpUniversalDataSystemEventPropertyObject::Clone() const { + NpUniversalDataSystemEventPropertyObject copy; + copy.properties.reserve(properties.size()); + for (const auto& [name, stored] : properties) copy.properties.emplace_back(name, Uds::Clone(stored)); + return copy; +} + +Uds::Value& NpUniversalDataSystemEventPropertyArray::Add(Uds::Value value) { + return values.emplace_back(std::move(value)); +} + +NpUniversalDataSystemEventPropertyArray NpUniversalDataSystemEventPropertyArray::Clone() const { + NpUniversalDataSystemEventPropertyArray copy; + copy.values.reserve(values.size()); + for (const auto& stored : values) copy.values.push_back(Uds::Clone(stored)); + return copy; +} diff --git a/core/libs/prx/libSceNpUniversalDataSystem/tests/Events.cpp b/core/libs/prx/libSceNpUniversalDataSystem/tests/Events.cpp new file mode 100644 index 0000000000..92274b6f56 --- /dev/null +++ b/core/libs/prx/libSceNpUniversalDataSystem/tests/Events.cpp @@ -0,0 +1,262 @@ +#include "NpUniversalDataSystem.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" + +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <optional> +#include <stdexcept> +#include <string> + +extern "C" { +int APS5_VABI sceNpUniversalDataSystemCreateContext(int*, int, uint32_t, uint64_t); +int APS5_VABI sceNpUniversalDataSystemDestroyContext(int); +int APS5_VABI sceNpUniversalDataSystemCreateEvent(const char*, const NpUniversalDataSystemEventPropertyObject*, NpUniversalDataSystemEvent**, NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemDestroyEvent(NpUniversalDataSystemEvent*); +int APS5_VABI sceNpUniversalDataSystemPostEvent(int, int, const void*, uint64_t); +int APS5_VABI sceNpUniversalDataSystemCreateEventPropertyObject(NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemDestroyEventPropertyObject(NpUniversalDataSystemEventPropertyObject*); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetString(NpUniversalDataSystemEventPropertyObject*, const char*, const char*); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetInt32(NpUniversalDataSystemEventPropertyObject*, const char*, int32_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetUInt32(NpUniversalDataSystemEventPropertyObject*, const char*, uint32_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetInt64(NpUniversalDataSystemEventPropertyObject*, const char*, int64_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetUInt64(NpUniversalDataSystemEventPropertyObject*, const char*, uint64_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetFloat32(NpUniversalDataSystemEventPropertyObject*, const char*, float); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetFloat64(NpUniversalDataSystemEventPropertyObject*, const char*, double); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetBool(NpUniversalDataSystemEventPropertyObject*, const char*, bool); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetBinary(NpUniversalDataSystemEventPropertyObject*, const char*, const void*, size_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetObject(NpUniversalDataSystemEventPropertyObject*, const char*, const NpUniversalDataSystemEventPropertyObject*, NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetArray(NpUniversalDataSystemEventPropertyObject*, const char*, const NpUniversalDataSystemEventPropertyArray*, NpUniversalDataSystemEventPropertyArray**); +int APS5_VABI sceNpUniversalDataSystemCreateEventPropertyArray(NpUniversalDataSystemEventPropertyArray**); +int APS5_VABI sceNpUniversalDataSystemDestroyEventPropertyArray(NpUniversalDataSystemEventPropertyArray*); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetString(NpUniversalDataSystemEventPropertyArray*, const char*); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetInt32(NpUniversalDataSystemEventPropertyArray*, int32_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetUInt32(NpUniversalDataSystemEventPropertyArray*, uint32_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetInt64(NpUniversalDataSystemEventPropertyArray*, int64_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetUInt64(NpUniversalDataSystemEventPropertyArray*, uint64_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetFloat32(NpUniversalDataSystemEventPropertyArray*, float); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetFloat64(NpUniversalDataSystemEventPropertyArray*, double); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetBool(NpUniversalDataSystemEventPropertyArray*, bool); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetBinary(NpUniversalDataSystemEventPropertyArray*, const void*, size_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetObject(NpUniversalDataSystemEventPropertyArray*, const NpUniversalDataSystemEventPropertyObject*, NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemEventPropertyArraySetArray(NpUniversalDataSystemEventPropertyArray*, const NpUniversalDataSystemEventPropertyArray*, NpUniversalDataSystemEventPropertyArray**); +} + +namespace { + +constexpr int InitialUser = 0x10000000; + +void check(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +void call(int result, const char* message) { + check(result == 0, message); +} + +template <typename TAction> +bool throwsInvalidArgument(TAction action) { + try { + action(); + } catch (const std::invalid_argument&) { + return true; + } + return false; +} + +NpUniversalDataSystemEventPropertyObject* createObject() { + NpUniversalDataSystemEventPropertyObject* object = nullptr; + call(sceNpUniversalDataSystemCreateEventPropertyObject(&object), "property object was not created"); + check(object != nullptr, "property object is null"); + return object; +} + +void testObjectSetters() { + auto* object = createObject(); + const std::uint8_t bytes[] = {1, 2, 3}; + call(sceNpUniversalDataSystemEventPropertyObjectSetInt32(object, "int32", -5), "SetInt32 failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(object, "uint32", 4000000000u), "SetUInt32 failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetInt64(object, "int64", -9000000000), "SetInt64 failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetUInt64(object, "uint64", 18000000000000000000ull), "SetUInt64 failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetFloat32(object, "float32", 0.5f), "SetFloat32 failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetFloat64(object, "float64", 0.25), "SetFloat64 failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetBool(object, "bool", true), "SetBool failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetString(object, "string", "a\"b\\c\n"), "SetString failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetBinary(object, "binary", bytes, sizeof(bytes)), "SetBinary failed"); + check(Uds::Describe(*object) == R"({"int32":-5,"uint32":4000000000,"int64":-9000000000,"uint64":18000000000000000000,"float32":0.5,"float64":0.25,"bool":true,"string":"a\"b\\c\u000a","binary":<3 bytes>})", + "properties were not kept with their types"); + const auto* binary = std::get_if<Uds::Binary>(object->Find("binary")); + check(binary != nullptr && *binary == Uds::Binary{1, 2, 3}, "binary bytes were not copied"); + call(sceNpUniversalDataSystemEventPropertyObjectSetString(object, "int32", "replaced"), "SetString over int32 failed"); + const auto* replaced = std::get_if<std::string>(object->Find("int32")); + check(object->properties.size() == 9 && replaced != nullptr && *replaced == "replaced", "setting a key again did not replace its value"); + check(object->Find("missing") == nullptr, "a missing key was found"); + check(throwsInvalidArgument([] { sceNpUniversalDataSystemEventPropertyObjectSetInt32(nullptr, "key", 1); }), "a null object was accepted"); + check(throwsInvalidArgument([object] { sceNpUniversalDataSystemEventPropertyObjectSetInt32(object, nullptr, 1); }), "a null key was accepted"); + call(sceNpUniversalDataSystemDestroyEventPropertyObject(object), "property object was not destroyed"); +} + +void testNestedValues() { + auto* source = createObject(); + auto* parent = createObject(); + call(sceNpUniversalDataSystemEventPropertyObjectSetInt32(source, "x", 1), "SetInt32 on the source failed"); + NpUniversalDataSystemEventPropertyObject* child = nullptr; + call(sceNpUniversalDataSystemEventPropertyObjectSetObject(parent, "child", source, &child), "SetObject failed"); + check(child != nullptr && child != source, "SetObject did not return its own copy"); + call(sceNpUniversalDataSystemEventPropertyObjectSetInt32(source, "x", 2), "SetInt32 after SetObject failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetBool(child, "y", true), "SetBool on the child failed"); + call(sceNpUniversalDataSystemDestroyEventPropertyObject(source), "source object was not destroyed"); + NpUniversalDataSystemEventPropertyArray* list = nullptr; + call(sceNpUniversalDataSystemEventPropertyObjectSetArray(parent, "list", nullptr, &list), "SetArray failed"); + check(list != nullptr, "SetArray did not return the stored array"); + const std::uint8_t bytes[] = {4, 5}; + call(sceNpUniversalDataSystemEventPropertyArraySetInt32(list, 1), "array SetInt32 failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetUInt32(list, 2), "array SetUInt32 failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetInt64(list, -3), "array SetInt64 failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetUInt64(list, 4), "array SetUInt64 failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetFloat32(list, 1.5f), "array SetFloat32 failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetFloat64(list, 2.5), "array SetFloat64 failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetBool(list, true), "array SetBool failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetString(list, "s"), "array SetString failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetBinary(list, bytes, sizeof(bytes)), "array SetBinary failed"); + NpUniversalDataSystemEventPropertyObject* element = nullptr; + call(sceNpUniversalDataSystemEventPropertyArraySetObject(list, nullptr, &element), "array SetObject failed"); + call(sceNpUniversalDataSystemEventPropertyObjectSetUInt64(element, "z", 3), "SetUInt64 on the array element failed"); + NpUniversalDataSystemEventPropertyArray* inner = nullptr; + call(sceNpUniversalDataSystemEventPropertyArraySetArray(list, nullptr, &inner), "array SetArray failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetBool(inner, false), "SetBool on the inner array failed"); + const char* expected = R"({"child":{"x":1,"y":true},"list":[1,2,-3,4,1.5,2.5,true,"s",<2 bytes>,{"z":3},[false]]})"; + check(Uds::Describe(*parent) == expected, "nested properties were not kept"); + const auto copy = parent->Clone(); + call(sceNpUniversalDataSystemEventPropertyObjectSetInt32(child, "x", 5), "SetInt32 on the child failed"); + call(sceNpUniversalDataSystemEventPropertyArraySetBool(inner, true), "SetBool on the inner array failed"); + check(Uds::Describe(copy) == expected, "a clone shares nested values with its source"); + call(sceNpUniversalDataSystemDestroyEventPropertyObject(child), "destroying a child failed"); + call(sceNpUniversalDataSystemDestroyEventPropertyArray(list), "destroying a child array failed"); + check(Uds::Describe(*parent) == R"({"child":{"x":5,"y":true},"list":[1,2,-3,4,1.5,2.5,true,"s",<2 bytes>,{"z":3},[false,true]]})", + "destroying a child took it from its parent"); + NpUniversalDataSystemEventPropertyArray* array = nullptr; + call(sceNpUniversalDataSystemCreateEventPropertyArray(&array), "property array was not created"); + call(sceNpUniversalDataSystemEventPropertyArraySetInt32(array, 9), "SetInt32 on the created array failed"); + NpUniversalDataSystemEventPropertyArray* stored = nullptr; + call(sceNpUniversalDataSystemEventPropertyObjectSetArray(parent, "list", array, &stored), "SetArray over a key failed"); + call(sceNpUniversalDataSystemDestroyEventPropertyArray(array), "property array was not destroyed"); + check(stored != array && Uds::Describe(*parent) == R"({"child":{"x":5,"y":true},"list":[9]})", "SetArray did not store a copy under its key"); + check(throwsInvalidArgument([] { sceNpUniversalDataSystemEventPropertyArraySetInt32(nullptr, 1); }), "a null array was accepted"); + call(sceNpUniversalDataSystemDestroyEventPropertyObject(parent), "parent object was not destroyed"); +} + +void testEvents() { + auto* properties = createObject(); + call(sceNpUniversalDataSystemEventPropertyObjectSetInt32(properties, "_trophy_id", 12), "SetInt32 failed"); + NpUniversalDataSystemEvent* event = nullptr; + NpUniversalDataSystemEventPropertyObject* eventProperties = nullptr; + call(sceNpUniversalDataSystemCreateEvent("_UnlockTrophy", properties, &event, &eventProperties), "event was not created"); + check(event != nullptr && eventProperties == &event->properties, "CreateEvent did not return the event's property object"); + call(sceNpUniversalDataSystemEventPropertyObjectSetInt32(properties, "_trophy_id", 13), "SetInt32 after CreateEvent failed"); + call(sceNpUniversalDataSystemDestroyEventPropertyObject(properties), "property object was not destroyed"); + check(Uds::Describe(*event) == R"(_UnlockTrophy {"_trophy_id":12})", "the event did not copy its properties"); + call(sceNpUniversalDataSystemEventPropertyObjectSetString(eventProperties, "detail", "boss"), "SetString on the event failed"); + call(sceNpUniversalDataSystemDestroyEventPropertyObject(eventProperties), "destroying the event's property object failed"); + check(Uds::Describe(*event) == R"(_UnlockTrophy {"_trophy_id":12,"detail":"boss"})", "properties set through the returned pointer were lost"); + call(sceNpUniversalDataSystemDestroyEvent(event), "event was not destroyed"); + NpUniversalDataSystemEvent* empty = nullptr; + call(sceNpUniversalDataSystemCreateEvent("_Empty", nullptr, &empty, nullptr), "event without properties was not created"); + check(Uds::Describe(*empty) == "_Empty {}", "an event without properties was not empty"); + call(sceNpUniversalDataSystemDestroyEvent(empty), "event without properties was not destroyed"); + check(throwsInvalidArgument([] { + NpUniversalDataSystemEvent* unnamed = nullptr; + sceNpUniversalDataSystemCreateEvent(nullptr, nullptr, &unnamed, nullptr); + }), "an event without a name was created"); +} + +Uds::TrophyEvent recognize(const char* name, std::optional<Uds::Value> id) { + NpUniversalDataSystemEvent event; + event.name = name; + if (id) event.properties.Set(Uds::TrophyIdProperty, *id); + return Uds::RecognizeTrophyEvent(event); +} + +void testRecognition() { + const auto unlock = recognize("_UnlockTrophy", std::int32_t{12}); + check(unlock.kind == Uds::TrophyEventKind::Unlock && unlock.trophyId == 12, "an unlock event was not recognised"); + const auto first = recognize("_UnlockTrophy", std::int32_t{0}); + check(first.kind == Uds::TrophyEventKind::Unlock && first.trophyId == 0, "an unlock of trophy 0 was not recognised"); + const auto progress = recognize("_UpdateTrophyProgress", std::int32_t{3}); + check(progress.kind == Uds::TrophyEventKind::Progress && progress.trophyId == 3, "a progress event was not recognised"); + const auto anonymous = recognize("_UpdateTrophyProgress", std::nullopt); + check(anonymous.kind == Uds::TrophyEventKind::Progress && anonymous.trophyId == -1, "a progress event without a trophy id was not recognised"); + const auto missing = recognize("_UnlockTrophy", std::nullopt); + check(missing.kind == Uds::TrophyEventKind::Malformed && missing.problem == "it has no _trophy_id", "an unlock without a trophy id was accepted"); + const auto widened = recognize("_UnlockTrophy", std::uint32_t{12}); + check(widened.kind == Uds::TrophyEventKind::Malformed && widened.problem == "its _trophy_id is not an int32", "an unlock with an unsigned trophy id was accepted"); + const auto negative = recognize("_UnlockTrophy", std::int32_t{-1}); + check(negative.kind == Uds::TrophyEventKind::Malformed && negative.problem == "its _trophy_id is negative", "an unlock with a negative trophy id was accepted"); + for (const char* name : {"_unlockTrophy", "UnlockTrophy", "_UnlockTrophy2", "_UnlockTrophy ", "_StartActivity", ""}) { + check(recognize(name, std::int32_t{12}).kind == Uds::TrophyEventKind::None, "an unknown event was taken for a trophy event"); + } +} + +struct Unlocks { + int count = 0; +}; + +void countUnlock(void* context, const NpTrophy2Unlock&) { + ++static_cast<Unlocks*>(context)->count; +} + +void post(int context, const char* name, std::optional<Uds::Value> id) { + NpUniversalDataSystemEvent* event = nullptr; + NpUniversalDataSystemEventPropertyObject* properties = nullptr; + call(sceNpUniversalDataSystemCreateEvent(name, nullptr, &event, &properties), "event was not created"); + if (id) properties->Set(Uds::TrophyIdProperty, *id); + call(sceNpUniversalDataSystemPostEvent(context, 1, event, 0), "event was not posted"); + call(sceNpUniversalDataSystemDestroyEvent(event), "event was not destroyed"); +} + +void testPostingIgnoredEvents() { + Unlocks unlocks; + const auto listener = NpTrophy2AddUnlockListener_nid_no_patch(countUnlock, &unlocks); + int context = 0; + call(sceNpUniversalDataSystemCreateContext(&context, InitialUser, 0, 0), "context was not created"); + int second = 0; + call(sceNpUniversalDataSystemCreateContext(&second, InitialUser, 0, 0), "second context was not created"); + check(second != context, "two contexts share an id"); + post(context, "_StartActivity", std::int32_t{1}); + post(context, "_UnlockTrophy", std::string("1")); + post(context, "_UnlockTrophy", std::nullopt); + post(context, "_UpdateTrophyProgress", std::int32_t{2}); + post(context, "_UpdateTrophyProgress", std::int32_t{2}); + post(second + 1, "_UnlockTrophy", std::int32_t{4}); + call(sceNpUniversalDataSystemDestroyContext(context), "context was not destroyed"); + post(context, "_UnlockTrophy", std::int32_t{4}); + call(sceNpUniversalDataSystemDestroyContext(second), "second context was not destroyed"); + NpTrophy2RemoveUnlockListener_nid_no_patch(listener); + check(unlocks.count == 0, "an event that is not a valid unlock reached the trophy store"); + check(throwsInvalidArgument([] { sceNpUniversalDataSystemPostEvent(1, 1, nullptr, 0); }), "a null event was posted"); + NpUniversalDataSystemEvent stray; + stray.name = "_UnlockTrophy"; + check(throwsInvalidArgument([&stray] { sceNpUniversalDataSystemPostEvent(1, 1, &stray, 0); }), "an event that CreateEvent did not create was posted"); + NpUniversalDataSystemEvent* destroyed = nullptr; + call(sceNpUniversalDataSystemCreateEvent("_UnlockTrophy", nullptr, &destroyed, nullptr), "event was not created"); + call(sceNpUniversalDataSystemDestroyEvent(destroyed), "event was not destroyed"); + check(throwsInvalidArgument([destroyed] { sceNpUniversalDataSystemPostEvent(1, 1, destroyed, 0); }), "a destroyed event was posted"); +} + +} + +int main() { + try { + testObjectSetters(); + testNestedValues(); + testEvents(); + testRecognition(); + testPostingIgnoredEvents(); + std::puts("UDS event tests passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/tests/GuestNpTrophyData.cpp b/core/libs/tests/GuestNpTrophyData.cpp index 6e07c0e769..eabbe68eae 100644 --- a/core/libs/tests/GuestNpTrophyData.cpp +++ b/core/libs/tests/GuestNpTrophyData.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libSceNpTrophy2/include/NpTrophy2Types.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include <cstddef> #include <cstdint> @@ -13,7 +14,19 @@ #include <utility> #include <vector> +struct NpUniversalDataSystemEvent; +struct NpUniversalDataSystemEventPropertyObject; + extern "C" { +int APS5_VABI sceNpUniversalDataSystemCreateContext(int*, int, uint32_t, uint64_t); +int APS5_VABI sceNpUniversalDataSystemRegisterContext(int, int, uint64_t); +int APS5_VABI sceNpUniversalDataSystemCreateHandle(int*); +int APS5_VABI sceNpUniversalDataSystemCreateEvent(const char*, const NpUniversalDataSystemEventPropertyObject*, NpUniversalDataSystemEvent**, NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemDestroyEvent(NpUniversalDataSystemEvent*); +int APS5_VABI sceNpUniversalDataSystemPostEvent(int, int, const void*, uint64_t); +int APS5_VABI sceNpUniversalDataSystemCreateEventPropertyObject(NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemDestroyEventPropertyObject(NpUniversalDataSystemEventPropertyObject*); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetInt32(NpUniversalDataSystemEventPropertyObject*, const char*, int32_t); int APS5_VABI sceNpTrophy2CreateContext(int*, int, uint32_t, uint64_t); int APS5_VABI sceNpTrophy2GetGameInfo(int, int, NpTrophy2GameDetails*, NpTrophy2GameData*); int APS5_VABI sceNpTrophy2GetGroupInfo(int, int, int, NpTrophy2GroupDetails*, NpTrophy2GroupData*); @@ -90,6 +103,24 @@ static void Write(const std::filesystem::path& path, const std::string& text) { Require(std::fclose(file) == 0 && written, "a test file was not written"); } +static void PostUnlock(int trophyId) { + static int context = 0; + static int handle = 0; + if (context == 0) { + Require(sceNpUniversalDataSystemCreateContext(&context, InitialUser, 0, 0) == 0, "UDS context was not created"); + Require(sceNpUniversalDataSystemCreateHandle(&handle) == 0, "UDS handle was not created"); + Require(sceNpUniversalDataSystemRegisterContext(context, handle, 0) == 0, "UDS context was not registered"); + } + NpUniversalDataSystemEventPropertyObject* properties = nullptr; + Require(sceNpUniversalDataSystemCreateEventPropertyObject(&properties) == 0, "property object was not created"); + Require(sceNpUniversalDataSystemEventPropertyObjectSetInt32(properties, "_trophy_id", trophyId) == 0, "trophy id was not set"); + NpUniversalDataSystemEvent* event = nullptr; + Require(sceNpUniversalDataSystemCreateEvent("_UnlockTrophy", properties, &event, nullptr) == 0, "event was not created"); + Require(sceNpUniversalDataSystemPostEvent(context, handle, event, 0) == 0, "event was not posted"); + Require(sceNpUniversalDataSystemDestroyEvent(event) == 0, "event was not destroyed"); + Require(sceNpUniversalDataSystemDestroyEventPropertyObject(properties) == 0, "property object was not destroyed"); +} + static int TrophyContext(std::uint32_t serviceLabel) { int context = 0; Require(sceNpTrophy2CreateContext(&context, InitialUser, serviceLabel, 0) == 0, "trophy context was not created"); @@ -109,6 +140,15 @@ static NpTrophy2GameData Game(int context, NpTrophy2GameDetails* details = nullp return data; } +struct Unlocks { + std::vector<std::int32_t> trophies; +}; + +static void Remember(void* context, const NpTrophy2Unlock& unlock) { + Require(unlock.titleId == "PPSA00000" && unlock.userId == InitialUser && unlock.serviceLabel == 0 && unlock.tick != 0, "an unlock does not carry its title, user, label and time"); + static_cast<Unlocks*>(context)->trophies.push_back(unlock.trophyId); +} + static void ReportTrophyData(int context) { NpTrophy2GameDetails game{}; const auto progress = Game(context, &game); @@ -153,6 +193,24 @@ static void ReportTrophyData(int context) { "the group list does not hold the base game and group 1"); } +static void UnlockPlatinum(int context) { + Unlocks unlocks; + const auto listener = NpTrophy2AddUnlockListener_nid_no_patch(Remember, &unlocks); + for (const int trophyId : {1, 2, 3, 6}) PostUnlock(trophyId); + const auto partway = Game(context); + Require(!Trophy(context, 0).unlocked && partway.unlocked_trophies == 4 && partway.unlocked_gold == 1 && partway.unlocked_silver == 1 && partway.unlocked_bronze == 2 && + partway.progress_percentage == (6 + 2 + 2) * 100 / (6 + 2 + 5), + "the game does not count the unlocks by grade"); + PostUnlock(4); + Require(!Trophy(context, 0).unlocked, "the platinum unlocked before every trophy that names it"); + PostUnlock(5); + const auto progressive = Trophy(context, 5); + Require(progressive.unlocked && progressive.progress.value == 5, "an unlocked progressive trophy does not report its target as progress"); + Require(Trophy(context, 0).unlocked && Game(context).unlocked_platinum == 1, "the platinum did not unlock with the last trophy that names it"); + NpTrophy2RemoveUnlockListener_nid_no_patch(listener); + Require(unlocks.trophies == (std::vector<std::int32_t>{1, 2, 3, 6, 4, 5, 0}), "the listener did not see each unlock and the platinum once, in order"); +} + static void ReportPlaceholders() { for (const std::uint32_t serviceLabel : {1u, 2u}) { const auto context = TrophyContext(serviceLabel); @@ -179,6 +237,7 @@ int main() { const auto context = TrophyContext(0); ReportTrophyData(context); + UnlockPlatinum(context); ReportPlaceholders(); std::filesystem::current_path(previous); diff --git a/core/libs/tests/GuestNpTrophyUnlocks.cpp b/core/libs/tests/GuestNpTrophyUnlocks.cpp new file mode 100644 index 0000000000..be3d6e2859 --- /dev/null +++ b/core/libs/tests/GuestNpTrophyUnlocks.cpp @@ -0,0 +1,109 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libSceNpTrophy2/include/NpTrophy2Types.hpp" + +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <filesystem> +#include <random> +#include <string> + +struct NpUniversalDataSystemEvent; +struct NpUniversalDataSystemEventPropertyObject; + +extern "C" { +int APS5_VABI sceNpUniversalDataSystemCreateContext(int*, int, uint32_t, uint64_t); +int APS5_VABI sceNpUniversalDataSystemRegisterContext(int, int, uint64_t); +int APS5_VABI sceNpUniversalDataSystemCreateHandle(int*); +int APS5_VABI sceNpUniversalDataSystemCreateEvent(const char*, const NpUniversalDataSystemEventPropertyObject*, NpUniversalDataSystemEvent**, NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemDestroyEvent(NpUniversalDataSystemEvent*); +int APS5_VABI sceNpUniversalDataSystemPostEvent(int, int, const void*, uint64_t); +int APS5_VABI sceNpUniversalDataSystemCreateEventPropertyObject(NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemDestroyEventPropertyObject(NpUniversalDataSystemEventPropertyObject*); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetInt32(NpUniversalDataSystemEventPropertyObject*, const char*, int32_t); +int APS5_VABI sceNpTrophy2CreateContext(int*, int, uint32_t, uint64_t); +int APS5_VABI sceNpTrophy2GetGameInfo(int, int, NpTrophy2GameDetails*, NpTrophy2GameData*); +int APS5_VABI sceNpTrophy2GetTrophyInfo(int, int, int, NpTrophy2Details*, NpTrophy2Data*); +} + +static constexpr int InitialUser = 0x10000000; + +static void Require(bool value, const char* message) { + if (value) return; + std::fprintf(stderr, "%s\n", message); + std::abort(); +} + +static std::string Read(const char* path) { + std::string text; + std::FILE* file = std::fopen(path, "rb"); + if (file == nullptr) return text; + char buffer[256]; + for (std::size_t count; (count = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, count); + std::fclose(file); + return text; +} + +static void Post(int context, int handle, const char* name, int trophyId) { + NpUniversalDataSystemEventPropertyObject* properties = nullptr; + Require(sceNpUniversalDataSystemCreateEventPropertyObject(&properties) == 0, "property object was not created"); + Require(sceNpUniversalDataSystemEventPropertyObjectSetInt32(properties, "_trophy_id", trophyId) == 0, "trophy id was not set"); + NpUniversalDataSystemEvent* event = nullptr; + Require(sceNpUniversalDataSystemCreateEvent(name, properties, &event, nullptr) == 0, "event was not created"); + Require(sceNpUniversalDataSystemPostEvent(context, handle, event, 0) == 0, "event was not posted"); + Require(sceNpUniversalDataSystemDestroyEvent(event) == 0, "event was not destroyed"); + Require(sceNpUniversalDataSystemDestroyEventPropertyObject(properties) == 0, "property object was not destroyed"); +} + +static NpTrophy2Data Trophy(int context, int handle, int trophyId) { + NpTrophy2Data data{}; + Require(sceNpTrophy2GetTrophyInfo(context, handle, trophyId, nullptr, &data) == 0, "trophy info was not returned"); + Require(data.trophy_id == trophyId, "trophy info is for another trophy"); + return data; +} + +static std::uint32_t Unlocked(int context, int handle) { + NpTrophy2GameData data{}; + Require(sceNpTrophy2GetGameInfo(context, handle, nullptr, &data) == 0, "game info was not returned"); + return data.unlocked_trophies; +} + +static void UnlockThroughEvents() { + std::filesystem::create_directories("app0/sce_sys"); + std::FILE* param = std::fopen("app0/sce_sys/param.json", "wb"); + Require(param != nullptr, "param.json was not created"); + std::fputs(R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}}})", param); + std::fclose(param); + int udsContext = 0; + int handle = 0; + Require(sceNpUniversalDataSystemCreateContext(&udsContext, InitialUser, 0, 0) == 0, "UDS context was not created"); + Require(sceNpUniversalDataSystemCreateHandle(&handle) == 0, "UDS handle was not created"); + Require(sceNpUniversalDataSystemRegisterContext(udsContext, handle, 0) == 0, "UDS context was not registered"); + int trophyContext = 0; + Require(sceNpTrophy2CreateContext(&trophyContext, InitialUser, 0, 0) == 0, "trophy context was not created"); + + Require(!Trophy(trophyContext, handle, 12).unlocked && Unlocked(trophyContext, handle) == 0, "trophy 12 is unlocked before its event"); + Post(udsContext, handle, "_UnlockTrophy", 12); + const auto unlocked = Trophy(trophyContext, handle, 12); + Require(unlocked.unlocked && unlocked.timestamp_tick != 0, "the posted unlock is not reported with its time"); + Require(Unlocked(trophyContext, handle) == 1, "the game does not count the posted unlock"); + + Post(udsContext, handle, "_ChallengeComplete", 13); + Require(!Trophy(trophyContext, handle, 13).unlocked, "an unknown event unlocked a trophy"); + Post(udsContext, handle, "_UnlockTrophy", 12); + Require(Trophy(trophyContext, handle, 12).timestamp_tick == unlocked.timestamp_tick && Unlocked(trophyContext, handle) == 1, "a repeated unlock changed the record"); + + Require(Read("_trophy/268435456_0.txt") == "12 " + std::to_string(unlocked.timestamp_tick) + "\n", "the unlock was not stored in the title folder"); +} + +int main() { + const auto previous = std::filesystem::current_path(); + const auto root = std::filesystem::temp_directory_path() / ("anyps5-trophy-unlocks-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(root); + std::filesystem::current_path(root); + UnlockThroughEvents(); + std::filesystem::current_path(previous); + std::filesystem::remove_all(root); + return 0; +} diff --git a/docs/dev/debt/libSceNpTrophy2.md b/docs/dev/debt/libSceNpTrophy2.md index e5060029a4..b7701b97c1 100644 --- a/docs/dev/debt/libSceNpTrophy2.md +++ b/docs/dev/debt/libSceNpTrophy2.md @@ -15,4 +15,4 @@ - [Trophy queries](../../../core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp) report the title's [trophy data](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp) (`sce_sys/trophy2/trophyNN.ucp`) and the unlocks in the [trophy store](../../../core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp). Without the file, or when it is malformed, they report the old placeholders and `sceNpTrophy2GetTrophyInfoArray` lists only trophy 0. Icons are not read. Hidden trophies report their names and descriptions; whether the console blanks them before the unlock is not known. `progress_percentage` weighs gold 6, silver 2 and bronze 1 and leaves the platinum out, as KytyPS5 does (the ratio of the published PSN trophy points); the console's formula is not known. A trophy or group that is not in the file throws. A context that was not created or was destroyed reports placeholders and no unlocks and returns success; the console's errors for it are unknown. - [Trophy data limits](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): a `trophyNN.ucp` larger than 256 MiB or listing more than 4096 files, and a JSON file in it that nests deeper than 64 levels or holds more than 1048576 values, are not used, and the title runs as without that file. The limits keep the memory the reader takes to the file and about 140 MiB of parsed JSON; the console's limits are not known. - [Text language](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): texts come from `tropmeta_<tag>.json` for the console language, then `en-US`, then the file's `defaultLanguage`. The tags for languages 0 to 29 are KytyPS5's; `uk-UA` for 30 (Ukrainian) is an assumption. -- [Platinum](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp): an unlock through any path unlocks the platinum that the trophy names in `platinumTrophyId` once every trophy naming that platinum is unlocked, as KytyPS5 does; trophies without `platinumTrophyId` are not required. Without trophy data any id of 0 or more is stored, and the platinum is not unlocked when the trophies it needs are. +- [Platinum](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp): an unlock through any path unlocks the platinum that the trophy names in `platinumTrophyId` once every trophy naming that platinum is unlocked, as KytyPS5 does; trophies without `platinumTrophyId` are not required. A platinum or a progressive trophy can still be unlocked directly with `_UnlockTrophy`, which KytyPS5 refuses; the console's behaviour is not known. Without trophy data any id of 0 or more is stored, and the platinum is not unlocked when the trophies it needs are. diff --git a/docs/dev/debt/libSceNpUniversalDataSystem.md b/docs/dev/debt/libSceNpUniversalDataSystem.md new file mode 100644 index 0000000000..3fcbc2508f --- /dev/null +++ b/docs/dev/debt/libSceNpUniversalDataSystem.md @@ -0,0 +1,9 @@ +# libSceNpUniversalDataSystem technical debt + +### Unknown function info + +- [Events](../../../core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp): `sceNpUniversalDataSystemCreateEvent` copies `prop` and returns the event's own property object through `prop_ptr`, as [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/9e41d8320a2774b303fbad344c2347e8d0292e83/src/libs/libTrophy.cpp) does; `sceNpUniversalDataSystemEventPropertyObjectSetObject`, `SetArray` and the array setters for objects and arrays store a copy of `value` and return the stored copy through `value_ptr`. Destroying a property object or array that its Create function did not return (an event's property object or a stored copy) does nothing, as its owner frees it. These ownership rules are not verified on a console, nor are the output of `sceNpUniversalDataSystemEventToString` (still `{}`) and `sceNpUniversalDataSystemEventEstimateSize` (still 3). + +### Functional + +- [Trophy events](../../../core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp): `sceNpUniversalDataSystemPostEvent` stores an `_UnlockTrophy` event with an int32 `_trophy_id` of 0 or more as an unlock for the user and service label of its context, as KytyPS5 does. Unlocks that a title drives through statistics (`statsExtractionRuleArray` in `sce_sys/uds/udsNN.ucp` and `unlockCondition` in `sce_sys/trophy2/trophyNN.ucp`) are not evaluated, and `_UpdateTrophyProgress`, named only by a third-party unlocker, is logged once per trophy and not interpreted. A context need not be registered, and every post of an event that `sceNpUniversalDataSystemCreateEvent` returned and that is not yet destroyed returns success (a null or any other event pointer stops the game): KytyPS5's errors for an unknown or unregistered context and for a malformed unlock are not verified. The storage statistics stay zero. From 42ddbdf89e33287ede1d35441e1f01c7181a78bc Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:28:50 +0000 Subject: [PATCH 072/103] docs: describe trophy unlocks in USAGE --- docs/user/USAGE.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index bc96b850c1..c2f142dfe1 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -159,9 +159,13 @@ Games ask for the console's language with `sceSystemServiceParamGetInt` and show ### Trophies -The trophy data comes from the game's own `app0/sce_sys/trophy2/trophyNN.ucp`, where `NN` is the service label (`00` for most games). The trophy queries report its names, descriptions and rewards in the console language (`ANYPS5_LANGUAGE`, see above), or in English (United States) or the data's default language when it has no text in that language, and its grades, groups, hidden flags, number of trophies and grade counts. The first trophy query prints what was read, such as `[trophy] /app0/sce_sys/trophy2/trophy00.ucp: 52 trophies, 1 group, texts in en-US`. +A game unlocks a trophy by posting a `_UnlockTrophy` event with an int32 `_trophy_id` property to `libSceNpUniversalDataSystem`. The unlock is stored in `_trophy/<user id>_<service label>.txt` in the directory the game runs in (the one that holds `app0/`), one `<trophy id> <unlock time>` line per trophy; the time is an RTC tick, microseconds since 0001-01-01 UTC. Delete the file before starting the game to lock its trophies again. Each new unlock prints one line, such as `[trophy] PPSA01556 unlocked trophy 12 for user 268435456`; unlocking a trophy again prints nothing. `sceNpTrophy2GetTrophyInfo`, `sceNpTrophy2GetTrophyInfoArray` and `sceNpTrophy2GetGameInfo` report the stored unlocks and their times to the game. Every posted `_trophy_id` of 0 or more is stored, even one the trophy data has no trophy for or makes the platinum or a progressive trophy. -Trophies are not unlocked yet: every trophy reports as locked, and nothing is written for them. A missing or malformed `trophyNN.ucp` makes the queries report placeholders, and a malformed one prints the reason once; `APS5_TRACE_UDS=1` prints a `[uds]` line for a missing one. Icons are not read: `sceNpTrophy2GetGameIcon`, `sceNpTrophy2GetGroupIcon`, `sceNpTrophy2GetTrophyIcon` and `sceNpTrophy2GetRewardIcon` stop the game with an error. No notification is shown, and callbacks registered with `sceNpTrophy2RegisterUnlockCallback` are not called. +The trophy data comes from the game's own `app0/sce_sys/trophy2/trophyNN.ucp`, where `NN` is the service label (`00` for most games). The trophy queries report its names, descriptions and rewards in the console language (`ANYPS5_LANGUAGE`, see above), or in English (United States) or the data's default language when it has no text in that language, and its grades, groups, hidden flags, number of trophies and grade counts. A platinum trophy unlocks as soon as every trophy that names it in `platinumTrophyId` is unlocked. The first trophy query or event prints what was read, such as `[trophy] /app0/sce_sys/trophy2/trophy00.ucp: 52 trophies, 1 group, texts in en-US`. + +`APS5_TRACE_UDS=1` prints every posted event, such as `[uds] context 1 posted _UnlockTrophy {"_trophy_id":12}`, and a `[uds]` line for a missing `trophyNN.ucp`. An event that is not used prints the reason in a `[trophy] ignored` line: an `_UnlockTrophy` event without an int32 `_trophy_id` of 0 or more, or posted to a context that does not exist, and the first `_UpdateTrophyProgress` event of each trophy, because trophy progress is not recorded. + +A missing or malformed `trophyNN.ucp` makes the queries report placeholders, and a malformed one prints the reason once. Icons are not read: `sceNpTrophy2GetGameIcon`, `sceNpTrophy2GetGroupIcon`, `sceNpTrophy2GetTrophyIcon` and `sceNpTrophy2GetRewardIcon` stop the game with an error. So does an unlock file in `_trophy/` that is damaged (a line that is not a new `<id> <value>` pair, such as a line an edit left blank or ended with a carriage return) or cannot be read or written. No notification is shown, and callbacks registered with `sceNpTrophy2RegisterUnlockCallback` are not called. ### PlayGo dumps without metadata From 4362803457e4a9322982dd3280577879b2aec0e0 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:57:31 +0000 Subject: [PATCH 073/103] feat(libSceNpUniversalDataSystem): unlock trophies through the title's statistics Events other than the trophy events go through the title's sce_sys/uds/udsNN.ucp: stats_definition.json gives the statistics that keep their latest uint32 or uint64 value, and stats_extraction.json the event property ($.name, dotted for nested objects) that sets each one. Statistics are kept per user and service label in _trophy/<user id>_<service label>.stats.txt next to the unlocks. After each event that sets a statistic, every trophy's unlockCondition (ge, gt, le or lt against its targetValue) is checked on the values that event set and the stored values of the other statistics, so a later event setting the same statistic cannot hide what this one reached. Reached trophies unlock through the same path as _UnlockTrophy. Progressive trophies report their statistic as progress, and APS5_TRACE_UDS prints each statistic an event sets. A missing udsNN.ucp is ignored and a malformed one is logged once. --- .../libSceNpTrophy2/include/TitleTrophies.hpp | 20 ++ .../libSceNpTrophy2/include/TrophyPackage.hpp | 21 ++ .../libSceNpTrophy2/include/TrophyStore.hpp | 35 ++ .../prx/libSceNpTrophy2/src/TitleTrophies.cpp | 73 +++++ .../prx/libSceNpTrophy2/src/TrophyInfo.cpp | 3 +- .../prx/libSceNpTrophy2/src/TrophyPackage.cpp | 111 +++++++ .../prx/libSceNpTrophy2/src/TrophyStore.cpp | 134 +++++--- .../libSceNpTrophy2/tests/TrophyPackage.cpp | 137 ++++++++ .../CMakeLists.txt | 10 + .../include/UdsEvent.hpp | 3 + .../libSceNpUniversalDataSystem/src/Event.cpp | 1 + .../src/UdsEvent.cpp | 40 +++ .../tests/Events.cpp | 34 ++ core/libs/tests/GuestNpTrophyStats.cpp | 306 ++++++++++++++++++ docs/dev/debt/libSceNpTrophy2.md | 2 +- docs/dev/debt/libSceNpUniversalDataSystem.md | 3 +- 16 files changed, 893 insertions(+), 40 deletions(-) create mode 100644 core/libs/tests/GuestNpTrophyStats.cpp diff --git a/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp b/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp index 74901778a9..cfea54f627 100644 --- a/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp +++ b/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp @@ -20,9 +20,11 @@ inline constexpr std::uint64_t MaxUcpSize = std::uint64_t{256} << 20u; struct TitleTrophies { std::optional<TrophyPackage> package; + std::optional<StatRules> rules; }; std::string PackagePath(std::uint32_t serviceLabel); +std::string StatRulesPath(std::uint32_t serviceLabel); TitleTrophies LoadTitleTrophies(std::uint32_t serviceLabel, std::string_view language); const TitleTrophies& Title(std::uint32_t serviceLabel); const TrophyPackage* Package(std::uint32_t serviceLabel); @@ -32,6 +34,24 @@ using UnlockedQuery = std::function<bool(std::int32_t)>; UnlockedQuery UnlockedBy(std::int32_t userId, std::uint32_t serviceLabel); +struct StatUpdate { + std::uint64_t statId = 0; + std::uint64_t value = 0; + std::string input; +}; + +struct StatProblem { + std::string rule; + std::string reason; +}; + +struct StatChanges { + std::vector<StatUpdate> updates; + std::vector<StatProblem> problems; +}; + +StatChanges ApplyStatRules(const StatRules& rules, const NpTrophy2Event& event, StatStore& stats, std::int32_t userId, std::uint32_t serviceLabel); +std::vector<std::int32_t> ReachedTrophies(const TrophyPackage& package, const std::map<std::uint64_t, std::uint64_t>& stats, const UnlockedQuery& unlocked); std::optional<std::int32_t> PlatinumToUnlock(const TrophyPackage& package, std::int32_t trophyId, const UnlockedQuery& unlocked); std::optional<std::uint64_t> ProgressTarget(const TrophyDefinition& trophy); std::uint64_t TrophyProgress(const TrophyDefinition& trophy, std::optional<std::uint64_t> stat, bool unlocked); diff --git a/core/libs/prx/libSceNpTrophy2/include/TrophyPackage.hpp b/core/libs/prx/libSceNpTrophy2/include/TrophyPackage.hpp index 1e1e20bd4c..4be4238929 100644 --- a/core/libs/prx/libSceNpTrophy2/include/TrophyPackage.hpp +++ b/core/libs/prx/libSceNpTrophy2/include/TrophyPackage.hpp @@ -89,6 +89,27 @@ struct TrophyPackage { TrophyPackage ParseTrophyPackage(std::string_view ucp, std::string_view language); +enum class StatType { UInt32, UInt64 }; + +struct StatDefinition { + StatType type = StatType::UInt64; + std::uint64_t minimum = 0; + std::uint64_t maximum = UINT64_MAX; +}; + +struct StatRule { + std::string input; + std::uint64_t statId = 0; +}; + +struct StatRules { + std::map<std::uint64_t, StatDefinition> stats; + std::map<std::string, std::vector<StatRule>, std::less<>> events; + std::vector<std::string> skipped; +}; + +StatRules ParseStatRules(std::string_view ucp); + std::string_view LanguageTag(int consoleLanguage); } diff --git a/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp b/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp index 245966e568..1418d70b58 100644 --- a/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp +++ b/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp @@ -9,6 +9,7 @@ #include <optional> #include <string> #include <utility> +#include <vector> struct NpTrophy2Unlock { std::string titleId; @@ -20,6 +21,17 @@ struct NpTrophy2Unlock { using NpTrophy2UnlockListener = void (*)(void* context, const NpTrophy2Unlock& unlock); +struct NpTrophy2EventProperty { + std::string path; + std::optional<std::uint64_t> value; + std::string text; +}; + +struct NpTrophy2Event { + std::string name; + std::vector<NpTrophy2EventProperty> properties; +}; + namespace NpTrophy2 { inline constexpr char TrophyRootGuest[] = "/_trophy"; @@ -50,6 +62,28 @@ private: TrophyStore& TitleStore(); +class StatStore { +public: + explicit StatStore(std::filesystem::path directory); + + void Set(std::int32_t userId, std::uint32_t serviceLabel, std::uint64_t statId, std::uint64_t value); + std::optional<std::uint64_t> Value(std::int32_t userId, std::uint32_t serviceLabel, std::uint64_t statId); + std::map<std::uint64_t, std::uint64_t> Values(std::int32_t userId, std::uint32_t serviceLabel); + std::filesystem::path RecordPath(std::int32_t userId, std::uint32_t serviceLabel) const; + +private: + using Key = std::pair<std::int32_t, std::uint32_t>; + using Stats = std::map<std::uint64_t, std::uint64_t>; + + Stats& record(const Key& key); + + std::filesystem::path directory; + std::mutex mutex; + std::map<Key, Stats> records; +}; + +StatStore& TitleStats(); + struct ContextOwner { std::int32_t userId = 0; std::uint32_t serviceLabel = 0; @@ -68,6 +102,7 @@ extern "C" { bool NpTrophy2RecordUnlock_nid_no_patch(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId); std::uint64_t NpTrophy2AddUnlockListener_nid_no_patch(NpTrophy2UnlockListener listener, void* context); void NpTrophy2RemoveUnlockListener_nid_no_patch(std::uint64_t id); +void NpTrophy2ApplyEvent_nid_no_patch(std::int32_t userId, std::uint32_t serviceLabel, const NpTrophy2Event& event); } diff --git a/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp b/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp index dcd465e8ce..342f6ea8bb 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp @@ -10,6 +10,7 @@ #include <fstream> #include <memory> #include <mutex> +#include <set> #include <stdexcept> #include <system_error> #include <utility> @@ -68,12 +69,23 @@ std::string counted(std::size_t count, const char* one, const char* many) { return std::to_string(count) + " " + (count == 1 ? one : many); } +bool firstProblem(const std::string& rule) { + static std::mutex mutex; + static std::set<std::string> reported; + std::lock_guard lock(mutex); + return reported.insert(rule).second; +} + } std::string PackagePath(std::uint32_t serviceLabel) { return labelled("/app0/sce_sys/trophy2/trophy%02u.ucp", serviceLabel); } +std::string StatRulesPath(std::uint32_t serviceLabel) { + return labelled("/app0/sce_sys/uds/uds%02u.ucp", serviceLabel); +} + TitleTrophies LoadTitleTrophies(std::uint32_t serviceLabel, std::string_view language) { TitleTrophies title; const auto packagePath = PackagePath(serviceLabel); @@ -84,6 +96,15 @@ TitleTrophies LoadTitleTrophies(std::uint32_t serviceLabel, std::string_view lan counted(title.package->groups.size(), "group", "groups").c_str(), title.package->language.c_str()); for (const auto& skipped : title.package->skipped) std::fprintf(stderr, "[trophy] %s: %s\n", packagePath.c_str(), skipped.c_str()); } + const auto rulesPath = StatRulesPath(serviceLabel); + title.rules = loadTitleFile<StatRules>(rulesPath, "posted events update no stats", [](std::string_view bytes) { return ParseStatRules(bytes); }); + if (title.rules) { + std::size_t count = 0; + for (const auto& [name, rules] : title.rules->events) count += rules.size(); + std::fprintf(stderr, "[trophy] %s: %s, %s for %s\n", rulesPath.c_str(), counted(title.rules->stats.size(), "stat", "stats").c_str(), counted(count, "rule", "rules").c_str(), + counted(title.rules->events.size(), "event", "events").c_str()); + for (const auto& skipped : title.rules->skipped) std::fprintf(stderr, "[trophy] %s: %s\n", rulesPath.c_str(), skipped.c_str()); + } return title; } @@ -113,6 +134,39 @@ UnlockedQuery UnlockedBy(std::int32_t userId, std::uint32_t serviceLabel) { return [userId, serviceLabel](std::int32_t trophyId) { return TitleStore().UnlockTick(userId, serviceLabel, trophyId).has_value(); }; } +StatChanges ApplyStatRules(const StatRules& rules, const NpTrophy2Event& event, StatStore& stats, std::int32_t userId, std::uint32_t serviceLabel) { + StatChanges changes; + const auto matching = rules.events.find(event.name); + if (matching == rules.events.end()) return changes; + for (const auto& rule : matching->second) { + const auto& definition = rules.stats.at(rule.statId); + const auto property = std::find_if(event.properties.begin(), event.properties.end(), [&rule](const NpTrophy2EventProperty& candidate) { return candidate.path == rule.input; }); + const auto where = event.name + " $." + rule.input + " for stat " + std::to_string(rule.statId); + if (property == event.properties.end()) { + changes.problems.push_back({where, "the event has no such property"}); + } else if (!property->value) { + changes.problems.push_back({where, property->text + " is not an unsigned integer"}); + } else if (*property->value < definition.minimum || *property->value > definition.maximum) { + changes.problems.push_back({where, property->text + " is not from " + std::to_string(definition.minimum) + " to " + std::to_string(definition.maximum)}); + } else { + stats.Set(userId, serviceLabel, rule.statId, *property->value); + changes.updates.push_back({rule.statId, *property->value, rule.input}); + } + } + return changes; +} + +std::vector<std::int32_t> ReachedTrophies(const TrophyPackage& package, const std::map<std::uint64_t, std::uint64_t>& stats, const UnlockedQuery& unlocked) { + std::vector<std::int32_t> reached; + for (const auto& [id, trophy] : package.trophies) { + const auto condition = trophy.Condition(); + if (!condition) continue; + const auto stat = stats.find(condition->statId); + if (stat != stats.end() && condition->Reached(stat->second) && !unlocked(id)) reached.push_back(id); + } + return reached; +} + std::optional<std::int32_t> PlatinumToUnlock(const TrophyPackage& package, std::int32_t trophyId, const UnlockedQuery& unlocked) { const auto trophy = package.trophies.find(trophyId); if (trophy == package.trophies.end() || !trophy->second.platinumId) return std::nullopt; @@ -157,3 +211,22 @@ std::uint32_t TrophyCounts::Percentage() const { } } + +extern "C" void NpTrophy2ApplyEvent_nid_no_patch(std::int32_t userId, std::uint32_t serviceLabel, const NpTrophy2Event& event) { + const auto& title = NpTrophy2::Title(serviceLabel); + if (!title.rules) return; + const auto changes = NpTrophy2::ApplyStatRules(*title.rules, event, NpTrophy2::TitleStats(), userId, serviceLabel); + for (const auto& problem : changes.problems) { + if (NpTrophy2::firstProblem(problem.rule)) std::fprintf(stderr, "[trophy] ignored %s: %s\n", problem.rule.c_str(), problem.reason.c_str()); + } + if (NpTrophy2::tracing()) { + for (const auto& update : changes.updates) { + std::fprintf(stderr, "[uds] stat %llu of user %d is %llu (%s $.%s)\n", static_cast<unsigned long long>(update.statId), userId, + static_cast<unsigned long long>(update.value), event.name.c_str(), update.input.c_str()); + } + } + if (changes.updates.empty() || !title.package) return; + auto stats = NpTrophy2::TitleStats().Values(userId, serviceLabel); + for (const auto& update : changes.updates) stats[update.statId] = update.value; + for (const auto trophyId : NpTrophy2::ReachedTrophies(*title.package, stats, NpTrophy2::UnlockedBy(userId, serviceLabel))) NpTrophy2RecordUnlock_nid_no_patch(userId, serviceLabel, trophyId); +} diff --git a/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp b/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp index dfeba5ffbe..ce29da1513 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp @@ -60,8 +60,9 @@ void fillData(const NpTrophy2::ContextOwner& owner, const NpTrophy2::TrophyDefin data->unlocked = tick.has_value(); data->timestamp_tick = tick.value_or(0); if (!NpTrophy2::ProgressTarget(trophy)) return; + const auto stat = trophy.statId ? NpTrophy2::TitleStats().Value(owner.userId, owner.serviceLabel, *trophy.statId) : std::nullopt; data->progress.type = NP_TROPHY2_PROGRESS_TYPE_VALUE; - data->progress.value = NpTrophy2::TrophyProgress(trophy, std::nullopt, data->unlocked); + data->progress.value = NpTrophy2::TrophyProgress(trophy, stat, data->unlocked); } } diff --git a/core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp b/core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp index 01d0bc8063..5da39d1cb4 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp @@ -336,6 +336,105 @@ TrophyDefinition readTrophy(const Json& definition, std::size_t index, std::vect return trophy; } +std::optional<std::string> propertyPath(std::string_view input) { + if (!input.starts_with("$.")) return std::nullopt; + const auto path = input.substr(2); + bool segmentStart = true; + for (const char character : path) { + if (character == '.') { + if (segmentStart) return std::nullopt; + segmentStart = true; + continue; + } + const bool word = (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || (character >= '0' && character <= '9') || character == '_'; + if (!word) return std::nullopt; + segmentStart = false; + } + if (segmentStart) return std::nullopt; + return std::string(path); +} + +void readStat(const Json& definition, std::size_t index, StatRules& rules) { + const auto* id = definition.Find("statId"); + const auto statId = id != nullptr ? id->Unsigned() : std::nullopt; + if (!statId) { + rules.skipped.push_back("stats_definition.json statDefinitionArray[" + std::to_string(index) + "] has no unsigned statId"); + return; + } + const auto name = "stat " + std::to_string(*statId); + const auto* aggregation = definition.Find("aggregation"); + if (aggregation == nullptr || aggregation->kind != Json::Kind::String || aggregation->text != "latest") { + rules.skipped.push_back(name + " has aggregation " + (aggregation != nullptr ? describe(*aggregation) : std::string("none")) + "; only \"latest\" is supported"); + return; + } + const auto* type = definition.Find("dataType"); + StatDefinition stat; + if (type != nullptr && type->kind == Json::Kind::String && type->text == "uint32") { + stat.type = StatType::UInt32; + stat.maximum = UINT32_MAX; + } else if (type != nullptr && type->kind == Json::Kind::String && type->text == "uint64") { + stat.type = StatType::UInt64; + stat.maximum = UINT64_MAX; + } else { + rules.skipped.push_back(name + " has dataType " + (type != nullptr ? describe(*type) : std::string("none")) + "; only \"uint32\" and \"uint64\" are supported"); + return; + } + if (const auto* minimum = definition.Find("minValue")) { + const auto value = minimum->Unsigned(); + if (!value || *value > stat.maximum) { + rules.skipped.push_back(name + " has minValue " + describe(*minimum) + ", which its " + type->text + " type cannot hold"); + return; + } + stat.minimum = *value; + } + if (const auto* maximum = definition.Find("maxValue")) { + const auto value = maximum->Unsigned(); + if (!value || *value > stat.maximum || *value < stat.minimum) { + rules.skipped.push_back(name + " has maxValue " + describe(*maximum) + ", which is not from its minValue to its " + type->text + " limit"); + return; + } + stat.maximum = *value; + } + if (!rules.stats.emplace(*statId, stat).second) rules.skipped.push_back(name + " is defined twice; the first definition is used"); +} + +void readRule(const Json& rule, std::size_t index, StatRules& rules) { + const auto where = "stats_extraction.json statsExtractionRuleArray[" + std::to_string(index) + "]"; + const auto* condition = rule.Find("condition"); + const auto* action = rule.Find("action"); + if (condition == nullptr || condition->kind != Json::Kind::Object || action == nullptr || action->kind != Json::Kind::Object) { + rules.skipped.push_back(where + " has no condition and action objects"); + return; + } + const auto* event = condition->Find("eventName"); + if (event == nullptr || event->kind != Json::Kind::String) { + rules.skipped.push_back(where + " has no eventName"); + return; + } + if (condition->members.size() != 1) { + rules.skipped.push_back(where + " filters " + event->text + " by more than its eventName; only eventName conditions are supported"); + return; + } + const auto* output = action->Find("output"); + const auto* id = output != nullptr ? output->Find("statId") : nullptr; + const auto statId = id != nullptr ? id->Unsigned() : std::nullopt; + if (!statId) { + rules.skipped.push_back(where + " for " + event->text + " has no unsigned output statId"); + return; + } + if (!rules.stats.contains(*statId)) { + rules.skipped.push_back(where + " for " + event->text + " writes stat " + std::to_string(*statId) + ", which has no supported definition"); + return; + } + const auto* input = action->Find("input"); + const auto path = input != nullptr && input->kind == Json::Kind::String ? propertyPath(input->text) : std::nullopt; + if (!path) { + rules.skipped.push_back(where + " for " + event->text + " reads " + (input != nullptr ? describe(*input) : std::string("no input")) + "; only \"$.name\" property paths are supported"); + return; + } + rules.events[event->text].push_back({*path, *statId}); +} + } const Json* Json::Find(std::string_view key) const { @@ -467,6 +566,18 @@ TrophyPackage ParseTrophyPackage(std::string_view ucp, std::string_view language return package; } +StatRules ParseStatRules(std::string_view ucp) { + const auto files = ReadUcp(ucp); + const auto definitions = document(files, "stats_definition.json"); + const auto extraction = document(files, "stats_extraction.json"); + const auto& stats = member(definitions, "statDefinitionArray", Json::Kind::Array, "stats_definition.json"); + const auto& extractionRules = member(extraction, "statsExtractionRuleArray", Json::Kind::Array, "stats_extraction.json"); + StatRules rules; + for (std::size_t index = 0; index < stats.items.size(); ++index) readStat(stats.items[index], index, rules); + for (std::size_t index = 0; index < extractionRules.items.size(); ++index) readRule(extractionRules.items[index], index, rules); + return rules; +} + std::string_view LanguageTag(int consoleLanguage) { static constexpr std::array<std::string_view, 31> tags = { "ja-JP", "en-US", "fr-FR", "es-ES", "de-DE", "it-IT", "nl-NL", "pt-PT", "ru-RU", "ko-KR", "zh-Hant", diff --git a/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp b/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp index 501fc72223..27f753283c 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp @@ -11,6 +11,7 @@ #include <stdexcept> #include <string_view> #include <system_error> +#include <type_traits> #include <vector> namespace NpTrophy2 { @@ -24,6 +25,58 @@ bool parseNumber(std::string_view text, TValue& value) { return error == std::errc{} && stop == end; } +template <typename TKey> +bool negative(TKey key) { + if constexpr (std::is_signed_v<TKey>) return key < 0; + return false; +} + +template <typename TKey> +std::map<TKey, std::uint64_t> readRecord(const std::filesystem::path& path, const std::string& what, const char* pair) { + std::map<TKey, std::uint64_t> values; + std::error_code error; + const bool exists = std::filesystem::exists(path, error); + if (error) throw std::filesystem::filesystem_error("cannot check the " + what, path, error); + if (!exists) return values; + std::ifstream file(path, std::ios::binary); + if (!file) throw std::runtime_error("cannot read the " + what + " " + path.string()); + std::string line; + std::size_t number = 0; + while (std::getline(file, line)) { + ++number; + const auto space = line.find(' '); + TKey key{}; + std::uint64_t value = 0; + const std::string_view text(line); + if (space == std::string::npos || !parseNumber(text.substr(0, space), key) || negative(key) || !parseNumber(text.substr(space + 1), value) || !values.emplace(key, value).second) { + throw std::runtime_error("the " + what + " " + path.string() + " has a line " + std::to_string(number) + " that is not a new '" + pair + "' pair"); + } + } + if (file.bad()) throw std::runtime_error("cannot read the " + what + " " + path.string()); + return values; +} + +template <typename TKey> +void writeRecord(const std::filesystem::path& directory, const std::filesystem::path& path, const std::map<TKey, std::uint64_t>& values, const std::string& what) { + std::error_code error; + if (std::filesystem::create_directories(directory, error)) RecordWrittenPath_nid_no_patch(directory); + if (error) throw std::filesystem::filesystem_error("cannot create the trophy directory", directory, error); + auto temporary = path; + temporary += ".tmp"; + { + std::ofstream file(temporary, std::ios::binary | std::ios::trunc); + for (const auto& [key, value] : values) file << key << ' ' << value << '\n'; + file.flush(); + if (!file) throw std::runtime_error("cannot write the " + what + " " + temporary.string()); + } + std::filesystem::rename(temporary, path); + RecordWrittenPath_nid_no_patch(path); +} + +std::string recordName(std::int32_t userId, std::uint32_t serviceLabel) { + return std::to_string(userId) + "_" + std::to_string(serviceLabel); +} + struct Listener { std::uint64_t id = 0; NpTrophy2UnlockListener callback = nullptr; @@ -57,51 +110,17 @@ std::uint64_t CurrentTick() { TrophyStore::TrophyStore(std::filesystem::path directory) : directory(std::move(directory)) {} std::filesystem::path TrophyStore::RecordPath(std::int32_t userId, std::uint32_t serviceLabel) const { - return directory / (std::to_string(userId) + "_" + std::to_string(serviceLabel) + ".txt"); + return directory / (recordName(userId, serviceLabel) + ".txt"); } TrophyStore::Unlocks& TrophyStore::record(const Key& key) { if (const auto found = records.find(key); found != records.end()) return found->second; - Unlocks unlocks; - const auto path = RecordPath(key.first, key.second); - std::error_code error; - const bool exists = std::filesystem::exists(path, error); - if (error) throw std::filesystem::filesystem_error("cannot check the trophy record", path, error); - if (exists) { - std::ifstream file(path, std::ios::binary); - if (!file) throw std::runtime_error("cannot read the trophy record " + path.string()); - std::string line; - std::size_t number = 0; - while (std::getline(file, line)) { - ++number; - const auto space = line.find(' '); - std::int32_t trophyId = 0; - std::uint64_t tick = 0; - const std::string_view text(line); - if (space == std::string::npos || !parseNumber(text.substr(0, space), trophyId) || trophyId < 0 || !parseNumber(text.substr(space + 1), tick) || !unlocks.emplace(trophyId, tick).second) { - throw std::runtime_error("the trophy record " + path.string() + " has a line " + std::to_string(number) + " that is not a new '<trophy id> <unlock tick>' pair"); - } - } - if (file.bad()) throw std::runtime_error("cannot read the trophy record " + path.string()); - } + auto unlocks = readRecord<std::int32_t>(RecordPath(key.first, key.second), "trophy record", "<trophy id> <unlock tick>"); return records.emplace(key, std::move(unlocks)).first->second; } void TrophyStore::save(const Key& key, const Unlocks& unlocks) const { - std::error_code error; - if (std::filesystem::create_directories(directory, error)) RecordWrittenPath_nid_no_patch(directory); - if (error) throw std::filesystem::filesystem_error("cannot create the trophy directory", directory, error); - const auto path = RecordPath(key.first, key.second); - auto temporary = path; - temporary += ".tmp"; - { - std::ofstream file(temporary, std::ios::binary | std::ios::trunc); - for (const auto& [trophyId, tick] : unlocks) file << trophyId << ' ' << tick << '\n'; - file.flush(); - if (!file) throw std::runtime_error("cannot write the trophy record " + temporary.string()); - } - std::filesystem::rename(temporary, path); - RecordWrittenPath_nid_no_patch(path); + writeRecord(directory, RecordPath(key.first, key.second), unlocks, "trophy record"); } bool TrophyStore::Unlock(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId, std::uint64_t tick) { @@ -135,6 +154,47 @@ TrophyStore& TitleStore() { return store; } +StatStore::StatStore(std::filesystem::path directory) : directory(std::move(directory)) {} + +std::filesystem::path StatStore::RecordPath(std::int32_t userId, std::uint32_t serviceLabel) const { + return directory / (recordName(userId, serviceLabel) + ".stats.txt"); +} + +StatStore::Stats& StatStore::record(const Key& key) { + if (const auto found = records.find(key); found != records.end()) return found->second; + auto stats = readRecord<std::uint64_t>(RecordPath(key.first, key.second), "stat record", "<stat id> <value>"); + return records.emplace(key, std::move(stats)).first->second; +} + +void StatStore::Set(std::int32_t userId, std::uint32_t serviceLabel, std::uint64_t statId, std::uint64_t value) { + std::lock_guard lock(mutex); + const Key key{userId, serviceLabel}; + auto& stats = record(key); + if (const auto found = stats.find(statId); found != stats.end() && found->second == value) return; + auto updated = stats; + updated[statId] = value; + writeRecord(directory, RecordPath(userId, serviceLabel), updated, "stat record"); + stats = std::move(updated); +} + +std::optional<std::uint64_t> StatStore::Value(std::int32_t userId, std::uint32_t serviceLabel, std::uint64_t statId) { + std::lock_guard lock(mutex); + const auto& stats = record({userId, serviceLabel}); + const auto found = stats.find(statId); + if (found == stats.end()) return std::nullopt; + return found->second; +} + +std::map<std::uint64_t, std::uint64_t> StatStore::Values(std::int32_t userId, std::uint32_t serviceLabel) { + std::lock_guard lock(mutex); + return record({userId, serviceLabel}); +} + +StatStore& TitleStats() { + static StatStore store(ResolvePath_nid_no_patch(TrophyRootGuest)); + return store; +} + std::optional<std::uint64_t> ContextUnlockTick(int context, std::int32_t trophyId) { const auto owner = FindContext(context); if (!owner || trophyId < 0) return std::nullopt; diff --git a/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp b/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp index 5ae534f6f6..d1dcdba734 100644 --- a/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp +++ b/core/libs/prx/libSceNpTrophy2/tests/TrophyPackage.cpp @@ -19,6 +19,8 @@ namespace { using Files = std::vector<std::pair<std::string, std::string>>; +constexpr std::int32_t InitialUser = 0x10000000; + void check(bool condition, const char* message) { if (condition) return; std::fprintf(stderr, "%s\n", message); @@ -84,10 +86,37 @@ const char TropMetaEnglish[] = R"({"metadata":{"titleMetadata":{"name":"Syntheti const char TropMetaFrench[] = R"({"metadata":{"titleMetadata":{"name":"Jeu synth\u00e9tique"},"trophyMetadata":[{"id":"1","name":"Dix"}]}})"; +const char StatsDefinition[] = R"({"statDefinitionArray":[ + {"statId":10,"dataType":"uint64","aggregation":"latest","minValue":"0"}, + {"statId":20,"dataType":"uint64","aggregation":"latest"}, + {"statId":30,"dataType":"uint32","aggregation":"latest"}, + {"statId":"40","dataType":"uint32","aggregation":"latest","maxValue":"50"}, + {"statId":50,"dataType":"uint64","aggregation":"sum"}, + {"statId":60,"dataType":"int32","aggregation":"latest"}, + {"statId":70,"dataType":"uint32","aggregation":"latest","maxValue":"5000000000"}, + {"dataType":"uint32","aggregation":"latest"} +]})"; + +const char StatsExtraction[] = R"({"statsExtractionRuleArray":[ + {"condition":{"eventName":"COLLECT"},"action":{"input":"$.counter","output":{"statId":10}}}, + {"condition":{"eventName":"COLLECT"},"action":{"input":"$.total","output":{"statId":20}}}, + {"condition":{"eventName":"TIME"},"action":{"input":"$.seconds","output":{"statId":30}}}, + {"condition":{"eventName":"STEPS"},"action":{"input":"$.player.steps","output":{"statId":"40"}}}, + {"condition":{"eventName":"SUM"},"action":{"input":"$.counter","output":{"statId":50}}}, + {"condition":{"eventName":"FILTERED","property":{"path":"$.level"}},"action":{"input":"$.counter","output":{"statId":10}}}, + {"condition":{"eventName":"COUNT"},"action":{"output":{"statId":10}}}, + {"condition":{"eventName":"BRACKETS"},"action":{"input":"$['counter']","output":{"statId":10}}}, + {"bad":1} +]})"; + std::string trophyPackage() { return ucp({{"tropconf.json", TropConf}, {"tropmeta_en-US.json", TropMetaEnglish}, {"tropmeta_fr-FR.json", TropMetaFrench}, {"trop0001.png", "not an image"}}); } +std::string statPackage() { + return ucp({{"stats_definition.json", StatsDefinition}, {"stats_extraction.json", StatsExtraction}}); +} + void testContainer() { const auto data = ucp({{"a.json", "{}"}, {"b.bin", std::string("\0\1\2", 3)}, {"a.json", "second"}}); const auto files = NpTrophy2::ReadUcp(data); @@ -279,6 +308,110 @@ void testProgressAndPlatinum() { check(NpTrophy2::CountTrophies(package, 5, isUnlocked).trophies == 0 && NpTrophy2::CountTrophies(package, 5, isUnlocked).Percentage() == 0, "an empty group has trophies"); } +void testStatRules() { + const auto rules = NpTrophy2::ParseStatRules(statPackage()); + check(rules.stats.size() == 4 && rules.stats.at(10).type == NpTrophy2::StatType::UInt64 && rules.stats.at(30).type == NpTrophy2::StatType::UInt32 && rules.stats.at(30).maximum == UINT32_MAX && + rules.stats.at(40).maximum == 50 && rules.stats.at(10).minimum == 0, + "stat definitions were not read"); + check(rules.events.size() == 3 && rules.events.at("COLLECT").size() == 2 && rules.events.at("COLLECT")[1].input == "total" && rules.events.at("COLLECT")[1].statId == 20 && + rules.events.at("STEPS")[0].input == "player.steps" && rules.events.at("TIME")[0].statId == 30, + "extraction rules were not read"); + std::set<std::string> skipped(rules.skipped.begin(), rules.skipped.end()); + const auto reported = [&skipped](const char* part) { + for (const auto& line : skipped) { + if (contains(line, part)) return true; + } + return false; + }; + check(rules.skipped.size() == 9, "unsupported definitions and rules were not all reported"); + check(reported("stat 50 has aggregation \"sum\"") && reported("stat 60 has dataType \"int32\"") && reported("stat 70 has maxValue \"5000000000\"") && reported("statDefinitionArray[7] has no unsigned statId"), + "unsupported definitions were not reported"); + check(reported("for SUM writes stat 50") && reported("filters FILTERED") && reported("for COUNT reads no input") && reported("for BRACKETS reads \"$['counter']\"") && reported("[8] has no condition"), + "unsupported rules were not reported"); + for (const char* input : {"$", "$.", "$..a", "$.a.", "$.a-b", "counter", "$.a b"}) { + const std::string extraction = std::string(R"({"statsExtractionRuleArray":[{"condition":{"eventName":"E"},"action":{"input":")") + input + R"(","output":{"statId":10}}}]})"; + check(NpTrophy2::ParseStatRules(ucp({{"stats_definition.json", StatsDefinition}, {"stats_extraction.json", extraction}})).events.empty(), "an unsupported input path was accepted"); + } + const auto withoutDefinitions = ucp({{"stats_extraction.json", StatsExtraction}}); + check(contains(failure([&withoutDefinitions] { NpTrophy2::ParseStatRules(withoutDefinitions); }), "no stats_definition.json"), "rules without definitions were read"); + const auto withoutRules = ucp({{"stats_definition.json", StatsDefinition}, {"stats_extraction.json", "{}"}}); + check(contains(failure([&withoutRules] { NpTrophy2::ParseStatRules(withoutRules); }), "has no statsExtractionRuleArray"), "rules without an extraction array were read"); +} + +void testConditions() { + const auto package = NpTrophy2::ParseTrophyPackage(trophyPackage(), "en-US"); + std::set<std::int32_t> unlocked; + const NpTrophy2::UnlockedQuery isUnlocked = [&unlocked](std::int32_t id) { return unlocked.contains(id); }; + const auto reached = [&](std::uint64_t stat, std::uint64_t value) { return NpTrophy2::ReachedTrophies(package, {{stat, value}}, isUnlocked); }; + check(reached(10, 9).empty() && reached(10, 10) == std::vector<std::int32_t>{1} && reached(10, 11) == std::vector<std::int32_t>{1}, "ge did not unlock at its target"); + check(reached(20, 100).empty() && reached(20, 101) == std::vector<std::int32_t>{2}, "gt did not unlock past its target"); + check(reached(30, 6).empty() && reached(30, 5) == std::vector<std::int32_t>{3} && reached(30, 3) == std::vector<std::int32_t>{3} && reached(30, 2) == (std::vector<std::int32_t>{3, 4}), + "le and lt did not unlock at and below their targets"); + check(reached(40, 4).empty() && reached(40, 5) == std::vector<std::int32_t>{5}, "the progressive trophy did not unlock at its target"); + check(reached(99, 1000).empty() && NpTrophy2::ReachedTrophies(package, {}, isUnlocked).empty(), "a stat without a trophy, or no stat, unlocked a trophy"); + unlocked.insert(1); + check(reached(10, 10).empty(), "an unlocked trophy was reached again"); + check(NpTrophy2::ReachedTrophies(package, {{10, 10}, {20, 101}, {30, 0}, {40, 9}}, isUnlocked) == (std::vector<std::int32_t>{2, 3, 4, 5}), "one set of stats did not reach every trophy"); +} + +void testStatStore() { + const std::filesystem::path directory("stats"); + { + NpTrophy2::StatStore stats(directory); + check(!stats.Value(InitialUser, 0, 10) && stats.Values(InitialUser, 0).empty(), "a new store has stats"); + stats.Set(InitialUser, 0, 10, 7); + stats.Set(InitialUser, 0, 18446744073709551615ull, 1); + stats.Set(InitialUser, 0, 10, 9); + stats.Set(InitialUser + 1, 0, 10, 3); + stats.Set(InitialUser, 1, 10, 4); + check(stats.RecordPath(InitialUser, 0) == directory / "268435456_0.stats.txt", "the stats are not stored next to the unlocks"); + } + NpTrophy2::StatStore restarted(directory); + check(restarted.Value(InitialUser, 0, 10) == std::optional<std::uint64_t>(9) && restarted.Value(InitialUser + 1, 0, 10) == std::optional<std::uint64_t>(3) && + restarted.Value(InitialUser, 1, 10) == std::optional<std::uint64_t>(4) && restarted.Values(InitialUser, 0).size() == 2, + "stats did not survive a restart"); + for (const char* text : {"10\n", "10 x\n", "x 5\n", "-1 5\n", "10 5\n10 6\n", "10 5 7\n", "\n", "10 -5\n"}) { + std::filesystem::remove_all("damaged"); + std::filesystem::create_directories("damaged"); + std::FILE* file = std::fopen("damaged/268435456_0.stats.txt", "wb"); + check(file != nullptr, "a damaged stat record was not created"); + std::fputs(text, file); + std::fclose(file); + NpTrophy2::StatStore damaged("damaged"); + check(contains(failure([&damaged] { damaged.Values(InitialUser, 0); }), "stat record"), "a damaged stat record was read"); + } +} + +NpTrophy2Event event(const char* name, std::vector<NpTrophy2EventProperty> properties) { + return {name, std::move(properties)}; +} + +void testApplyStatRules() { + const auto rules = NpTrophy2::ParseStatRules(statPackage()); + NpTrophy2::StatStore stats("applied"); + const auto none = NpTrophy2::ApplyStatRules(rules, event("UNKNOWN", {{"counter", 5, "5"}}), stats, InitialUser, 0); + check(none.updates.empty() && none.problems.empty() && stats.Values(InitialUser, 0).empty(), "an unknown event changed a stat"); + const auto collect = NpTrophy2::ApplyStatRules(rules, event("COLLECT", {{"counter", 12, "12"}, {"total", 300, "300"}, {"other", 1, "1"}}), stats, InitialUser, 0); + check(collect.problems.empty() && collect.updates.size() == 2 && collect.updates[0].statId == 10 && collect.updates[0].value == 12 && collect.updates[0].input == "counter" && + collect.updates[1].statId == 20 && collect.updates[1].value == 300, + "an event did not update every stat it feeds"); + check(stats.Value(InitialUser, 0, 10) == std::optional<std::uint64_t>(12) && stats.Value(InitialUser, 0, 20) == std::optional<std::uint64_t>(300), "the updated stats were not stored"); + const auto missing = NpTrophy2::ApplyStatRules(rules, event("COLLECT", {{"counter", 13, "13"}}), stats, InitialUser, 0); + check(missing.updates.size() == 1 && missing.problems.size() == 1 && missing.problems[0].rule == "COLLECT $.total for stat 20" && missing.problems[0].reason == "the event has no such property" && + stats.Value(InitialUser, 0, 20) == std::optional<std::uint64_t>(300), + "a missing property changed its stat"); + const auto wrong = NpTrophy2::ApplyStatRules(rules, event("TIME", {{"seconds", std::nullopt, "-3"}}), stats, InitialUser, 0); + check(wrong.updates.empty() && wrong.problems.size() == 1 && wrong.problems[0].reason == "-3 is not an unsigned integer" && !stats.Value(InitialUser, 0, 30), "a negative value changed a stat"); + const auto wide = NpTrophy2::ApplyStatRules(rules, event("TIME", {{"seconds", 4294967296ull, "4294967296"}}), stats, InitialUser, 0); + check(wide.updates.empty() && wide.problems.size() == 1 && wide.problems[0].reason == "4294967296 is not from 0 to 4294967295", "a uint32 stat took a value it cannot hold"); + const auto bounded = NpTrophy2::ApplyStatRules(rules, event("STEPS", {{"player.steps", 51, "51"}}), stats, InitialUser, 0); + check(bounded.updates.empty() && bounded.problems[0].reason == "51 is not from 0 to 50", "a value above maxValue changed a stat"); + const auto nested = NpTrophy2::ApplyStatRules(rules, event("STEPS", {{"steps", 2, "2"}, {"player.steps", 50, "50"}}), stats, InitialUser, 0); + check(nested.updates.size() == 1 && nested.updates[0].statId == 40 && stats.Value(InitialUser, 0, 40) == std::optional<std::uint64_t>(50), "a nested property was not read"); + const auto latest = NpTrophy2::ApplyStatRules(rules, event("COLLECT", {{"counter", 2, "2"}, {"total", 1, "1"}}), stats, InitialUser, 0); + check(latest.updates.size() == 2 && stats.Value(InitialUser, 0, 10) == std::optional<std::uint64_t>(2), "a stat did not keep the latest value"); +} + } int main() { @@ -290,6 +423,10 @@ int main() { testJson(); testPackage(); testProgressAndPlatinum(); + testStatRules(); + testConditions(); + testStatStore(); + testApplyStatRules(); std::filesystem::current_path(previous); std::filesystem::remove_all(root); std::puts("Trophy package tests passed"); diff --git a/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt b/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt index d7b5a34d12..d9163d1dc7 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt +++ b/core/libs/prx/libSceNpUniversalDataSystem/CMakeLists.txt @@ -53,4 +53,14 @@ if(BUILD_TESTING) set_tests_properties(guest_np_trophy_data PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpUniversalDataSystem>") endif() + + add_test_executable(guest_np_trophy_stats_tests ${LIBS_INCLUDE_DIR}/tests/GuestNpTrophyStats.cpp) + target_include_directories(guest_np_trophy_stats_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_np_trophy_stats_tests PRIVATE libSceNpUniversalDataSystem libSceNpTrophy2 libc) + configure_windows_unwind(guest_np_trophy_stats_tests) + add_test(NAME guest_np_trophy_stats COMMAND guest_np_trophy_stats_tests) + if(WIN32) + set_tests_properties(guest_np_trophy_stats PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpUniversalDataSystem>") + endif() endif() diff --git a/core/libs/prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp b/core/libs/prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp index d3cae5efb2..35c8196824 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp @@ -11,6 +11,7 @@ struct NpUniversalDataSystemEventPropertyObject; struct NpUniversalDataSystemEventPropertyArray; +struct NpTrophy2Event; namespace Uds { @@ -57,8 +58,10 @@ struct TrophyEvent { }; TrophyEvent RecognizeTrophyEvent(const NpUniversalDataSystemEvent& event); +std::string Describe(const Value& value); std::string Describe(const NpUniversalDataSystemEventPropertyObject& object); std::string Describe(const NpUniversalDataSystemEvent& event); +NpTrophy2Event StatEvent(const NpUniversalDataSystemEvent& event); } diff --git a/core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp b/core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp index 0ead101091..be7df0ae46 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp @@ -38,6 +38,7 @@ void post(const NpUniversalDataSystemEvent& event, int context) { const auto trophy = Uds::RecognizeTrophyEvent(event); switch (trophy.kind) { case Uds::TrophyEventKind::None: + if (const auto owner = Uds::FindContext(context)) NpTrophy2ApplyEvent_nid_no_patch(owner->userId, owner->serviceLabel, Uds::StatEvent(event)); return; case Uds::TrophyEventKind::Malformed: ignore(event, context, trophy.problem); diff --git a/core/libs/prx/libSceNpUniversalDataSystem/src/UdsEvent.cpp b/core/libs/prx/libSceNpUniversalDataSystem/src/UdsEvent.cpp index 96b8672b36..201178ddbf 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/src/UdsEvent.cpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/src/UdsEvent.cpp @@ -1,6 +1,8 @@ #include "prx/libSceNpUniversalDataSystem/include/UdsEvent.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include <cstdio> +#include <optional> #include <type_traits> namespace Uds { @@ -81,6 +83,31 @@ void appendValue(std::string& out, const Value& value) { }, value); } +std::optional<std::uint64_t> unsignedValue(const Value& value) { + return std::visit([](const auto& held) -> std::optional<std::uint64_t> { + using Held = std::decay_t<decltype(held)>; + if constexpr (std::is_same_v<Held, std::uint32_t> || std::is_same_v<Held, std::uint64_t>) { + return held; + } else if constexpr (std::is_same_v<Held, std::int32_t> || std::is_same_v<Held, std::int64_t>) { + if (held < 0) return std::nullopt; + return static_cast<std::uint64_t>(held); + } else { + return std::nullopt; + } + }, value); +} + +void flatten(const NpUniversalDataSystemEventPropertyObject& object, const std::string& prefix, std::vector<NpTrophy2EventProperty>& properties) { + for (const auto& [key, value] : object.properties) { + auto path = prefix.empty() ? key : prefix + "." + key; + if (const auto* child = std::get_if<std::shared_ptr<NpUniversalDataSystemEventPropertyObject>>(&value)) { + flatten(**child, path, properties); + continue; + } + properties.push_back({std::move(path), unsignedValue(value), Describe(value)}); + } +} + } TrophyEvent RecognizeTrophyEvent(const NpUniversalDataSystemEvent& event) { @@ -99,6 +126,12 @@ TrophyEvent RecognizeTrophyEvent(const NpUniversalDataSystemEvent& event) { return result; } +std::string Describe(const Value& value) { + std::string out; + appendValue(out, value); + return out; +} + std::string Describe(const NpUniversalDataSystemEventPropertyObject& object) { std::string out; appendObject(out, object); @@ -109,6 +142,13 @@ std::string Describe(const NpUniversalDataSystemEvent& event) { return event.name + " " + Describe(event.properties); } +NpTrophy2Event StatEvent(const NpUniversalDataSystemEvent& event) { + NpTrophy2Event converted; + converted.name = event.name; + flatten(event.properties, {}, converted.properties); + return converted; +} + } Uds::Value& NpUniversalDataSystemEventPropertyObject::Set(std::string_view key, Uds::Value value) { diff --git a/core/libs/prx/libSceNpUniversalDataSystem/tests/Events.cpp b/core/libs/prx/libSceNpUniversalDataSystem/tests/Events.cpp index 92274b6f56..d9cca747e4 100644 --- a/core/libs/prx/libSceNpUniversalDataSystem/tests/Events.cpp +++ b/core/libs/prx/libSceNpUniversalDataSystem/tests/Events.cpp @@ -5,9 +5,12 @@ #include <cstddef> #include <cstdint> #include <cstdio> +#include <memory> #include <optional> #include <stdexcept> #include <string> +#include <tuple> +#include <vector> extern "C" { int APS5_VABI sceNpUniversalDataSystemCreateContext(int*, int, uint32_t, uint64_t); @@ -198,6 +201,36 @@ void testRecognition() { } } +void testStatEvent() { + NpUniversalDataSystemEvent event; + event.name = "STEPS"; + event.properties.Set("count", std::uint32_t{4}); + event.properties.Set("signed", std::int32_t{-2}); + event.properties.Set("wide", std::int64_t{7}); + event.properties.Set("huge", std::uint64_t{18000000000000000000ull}); + event.properties.Set("ratio", 0.5); + event.properties.Set("name", std::string("x")); + auto player = std::make_shared<NpUniversalDataSystemEventPropertyObject>(); + player->Set("steps", std::uint32_t{5}); + auto inner = std::make_shared<NpUniversalDataSystemEventPropertyObject>(); + inner->Set("depth", std::int32_t{1}); + player->Set("inner", inner); + event.properties.Set("player", player); + auto list = std::make_shared<NpUniversalDataSystemEventPropertyArray>(); + list->Add(std::int32_t{1}); + event.properties.Set("list", list); + const auto converted = Uds::StatEvent(event); + const std::vector<std::tuple<std::string, std::optional<std::uint64_t>, std::string>> expected{ + {"count", 4, "4"}, {"signed", std::nullopt, "-2"}, {"wide", 7, "7"}, {"huge", 18000000000000000000ull, "18000000000000000000"}, {"ratio", std::nullopt, "0.5"}, + {"name", std::nullopt, "\"x\""}, {"player.steps", 5, "5"}, {"player.inner.depth", 1, "1"}, {"list", std::nullopt, "[1]"}}; + check(converted.name == "STEPS" && converted.properties.size() == expected.size(), "the stat event does not hold every leaf property"); + for (std::size_t i = 0; i < expected.size(); ++i) { + const auto& [path, value, text] = expected[i]; + const auto& property = converted.properties[i]; + check(property.path == path && property.value == value && property.text == text, "a stat event property has the wrong path, value or text"); + } +} + struct Unlocks { int count = 0; }; @@ -252,6 +285,7 @@ int main() { testNestedValues(); testEvents(); testRecognition(); + testStatEvent(); testPostingIgnoredEvents(); std::puts("UDS event tests passed"); return 0; diff --git a/core/libs/tests/GuestNpTrophyStats.cpp b/core/libs/tests/GuestNpTrophyStats.cpp new file mode 100644 index 0000000000..2464aa36fc --- /dev/null +++ b/core/libs/tests/GuestNpTrophyStats.cpp @@ -0,0 +1,306 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libSceNpTrophy2/include/NpTrophy2Types.hpp" +#include "prx/libSceNpTrophy2/include/TrophyStore.hpp" + +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <filesystem> +#include <random> +#include <string> +#include <utility> +#include <vector> + +struct NpUniversalDataSystemEvent; +struct NpUniversalDataSystemEventPropertyObject; + +extern "C" { +int APS5_VABI sceNpUniversalDataSystemCreateContext(int*, int, uint32_t, uint64_t); +int APS5_VABI sceNpUniversalDataSystemRegisterContext(int, int, uint64_t); +int APS5_VABI sceNpUniversalDataSystemCreateHandle(int*); +int APS5_VABI sceNpUniversalDataSystemCreateEvent(const char*, const NpUniversalDataSystemEventPropertyObject*, NpUniversalDataSystemEvent**, NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemDestroyEvent(NpUniversalDataSystemEvent*); +int APS5_VABI sceNpUniversalDataSystemPostEvent(int, int, const void*, uint64_t); +int APS5_VABI sceNpUniversalDataSystemCreateEventPropertyObject(NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpUniversalDataSystemDestroyEventPropertyObject(NpUniversalDataSystemEventPropertyObject*); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetInt32(NpUniversalDataSystemEventPropertyObject*, const char*, int32_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetUInt32(NpUniversalDataSystemEventPropertyObject*, const char*, uint32_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetInt64(NpUniversalDataSystemEventPropertyObject*, const char*, int64_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetUInt64(NpUniversalDataSystemEventPropertyObject*, const char*, uint64_t); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetFloat32(NpUniversalDataSystemEventPropertyObject*, const char*, float); +int APS5_VABI sceNpUniversalDataSystemEventPropertyObjectSetObject(NpUniversalDataSystemEventPropertyObject*, const char*, const NpUniversalDataSystemEventPropertyObject*, NpUniversalDataSystemEventPropertyObject**); +int APS5_VABI sceNpTrophy2CreateContext(int*, int, uint32_t, uint64_t); +int APS5_VABI sceNpTrophy2GetGameInfo(int, int, NpTrophy2GameDetails*, NpTrophy2GameData*); +int APS5_VABI sceNpTrophy2GetTrophyInfo(int, int, int, NpTrophy2Details*, NpTrophy2Data*); +} + +static constexpr int InitialUser = 0x10000000; + +static const char TropConf[] = R"({"defaultLanguage":"en-US","trophies":[ + {"id":"0","grade":"P"}, + {"id":"1","grade":"G","platinumTrophyId":"0","unlockCondition":{"udsStatId":"10","comparator":"ge","targetValue":"10","progressive":false}}, + {"id":"2","grade":"S","platinumTrophyId":"0","unlockCondition":{"udsStatId":"20","comparator":"gt","targetValue":"100"}}, + {"id":"3","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"30","comparator":"le","targetValue":"5"}}, + {"id":"4","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"30","comparator":"lt","targetValue":"3"}}, + {"id":"5","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"40","comparator":"ge","targetValue":"5","progressive":true}}, + {"id":"6","grade":"B","groupId":"1","hidden":true,"hasReward":true}, + {"id":"7","grade":"B","groupId":"1"} +]})"; + +static const char TropMeta[] = R"({"metadata":{"titleMetadata":{"name":"Synthetic Game"}, + "trophyMetadata":[{"id":"0","name":"All of them"},{"id":"1","name":"Ten","detail":"Reach ten"},{"id":"5","name":"Five steps"}, + {"id":"6","name":"Secret","detail":"A hidden one","reward":"A hat"}], + "groupMetadata":[{"id":"1","name":"Expansion"}]}})"; + +static const char StatsDefinition[] = R"({"statDefinitionArray":[ + {"statId":10,"dataType":"uint64","aggregation":"latest"},{"statId":20,"dataType":"uint64","aggregation":"latest"}, + {"statId":30,"dataType":"uint32","aggregation":"latest"},{"statId":40,"dataType":"uint32","aggregation":"latest"}]})"; + +static const char StatsExtraction[] = R"({"statsExtractionRuleArray":[ + {"condition":{"eventName":"COLLECT"},"action":{"input":"$.counter","output":{"statId":10}}}, + {"condition":{"eventName":"COLLECT"},"action":{"input":"$.total","output":{"statId":20}}}, + {"condition":{"eventName":"TIME"},"action":{"input":"$.seconds","output":{"statId":30}}}, + {"condition":{"eventName":"STEPS"},"action":{"input":"$.player.steps","output":{"statId":40}}}]})"; + +static void Require(bool value, const char* message) { + if (value) return; + std::fprintf(stderr, "%s\n", message); + std::abort(); +} + +static void PutBigEndian(std::string& data, std::size_t offset, std::uint64_t value, std::size_t size) { + for (std::size_t i = 0; i < size; ++i) data[offset + i] = static_cast<char>((value >> (8 * (size - 1 - i))) & 0xFF); +} + +static std::string Ucp(const std::vector<std::pair<std::string, std::string>>& files) { + std::size_t offset = 0x60 + files.size() * 0x40; + std::size_t size = offset; + for (const auto& file : files) size += file.second.size(); + std::string data(size, '\0'); + PutBigEndian(data, 0, 0xB228C60A, 4); + PutBigEndian(data, 4, 1, 4); + PutBigEndian(data, 8, size, 8); + PutBigEndian(data, 0x10, files.size(), 4); + PutBigEndian(data, 0x14, 0x40, 4); + std::size_t entry = 0x60; + for (const auto& [name, contents] : files) { + data.replace(entry, name.size(), name); + PutBigEndian(data, entry + 0x20, offset, 8); + PutBigEndian(data, entry + 0x28, contents.size(), 8); + data.replace(offset, contents.size(), contents); + offset += contents.size(); + entry += 0x40; + } + return data; +} + +static void Write(const std::filesystem::path& path, const std::string& text) { + std::filesystem::create_directories(path.parent_path()); + std::FILE* file = std::fopen(path.string().c_str(), "wb"); + Require(file != nullptr, "a test file was not created"); + const bool written = std::fwrite(text.data(), 1, text.size(), file) == text.size(); + Require(std::fclose(file) == 0 && written, "a test file was not written"); +} + +static std::string Read(const char* path) { + std::string text; + std::FILE* file = std::fopen(path, "rb"); + if (file == nullptr) return text; + char buffer[256]; + for (std::size_t count; (count = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, count); + std::fclose(file); + return text; +} + +static bool Exists(const char* path) { + std::FILE* file = std::fopen(path, "rb"); + if (file == nullptr) return false; + std::fclose(file); + return true; +} + +struct Uds { + int context = 0; + int handle = 0; +}; + +static Uds OpenUds(std::uint32_t serviceLabel) { + Uds uds; + Require(sceNpUniversalDataSystemCreateContext(&uds.context, InitialUser, serviceLabel, 0) == 0, "UDS context was not created"); + Require(sceNpUniversalDataSystemCreateHandle(&uds.handle) == 0, "UDS handle was not created"); + Require(sceNpUniversalDataSystemRegisterContext(uds.context, uds.handle, 0) == 0, "UDS context was not registered"); + return uds; +} + +template <typename TSet> +static void Post(const Uds& uds, const char* name, TSet set) { + NpUniversalDataSystemEventPropertyObject* properties = nullptr; + Require(sceNpUniversalDataSystemCreateEventPropertyObject(&properties) == 0, "property object was not created"); + set(properties); + NpUniversalDataSystemEvent* event = nullptr; + Require(sceNpUniversalDataSystemCreateEvent(name, properties, &event, nullptr) == 0, "event was not created"); + Require(sceNpUniversalDataSystemPostEvent(uds.context, uds.handle, event, 0) == 0, "event was not posted"); + Require(sceNpUniversalDataSystemDestroyEvent(event) == 0, "event was not destroyed"); + Require(sceNpUniversalDataSystemDestroyEventPropertyObject(properties) == 0, "property object was not destroyed"); +} + +static int TrophyContext(std::uint32_t serviceLabel) { + int context = 0; + Require(sceNpTrophy2CreateContext(&context, InitialUser, serviceLabel, 0) == 0, "trophy context was not created"); + return context; +} + +static NpTrophy2Data Trophy(int context, int trophyId, NpTrophy2Details* details = nullptr) { + NpTrophy2Data data{}; + Require(sceNpTrophy2GetTrophyInfo(context, 1, trophyId, details, &data) == 0, "trophy info was not returned"); + Require(data.trophy_id == trophyId, "trophy info is for another trophy"); + return data; +} + +static NpTrophy2GameData Game(int context, NpTrophy2GameDetails* details = nullptr) { + NpTrophy2GameData data{}; + Require(sceNpTrophy2GetGameInfo(context, 1, details, &data) == 0, "game info was not returned"); + return data; +} + +struct Unlocks { + std::vector<std::pair<std::uint32_t, std::int32_t>> trophies; +}; + +static void Remember(void* context, const NpTrophy2Unlock& unlock) { + Require(unlock.titleId == "PPSA00000" && unlock.userId == InitialUser && unlock.tick != 0, "an unlock does not carry its title, user and time"); + static_cast<Unlocks*>(context)->trophies.emplace_back(unlock.serviceLabel, unlock.trophyId); +} + +static void KeepStatsAcrossRuns(int context) { + NpTrophy2Details details{}; + const auto steps = Trophy(context, 5, &details); + Require(details.target.type == 1 && details.target.value == 5 && !steps.unlocked && steps.progress.type == 1 && steps.progress.value == 2, + "the progressive trophy does not report the stat kept from the last run"); +} + +static void UnlockThroughStats(int context, Unlocks& unlocks) { + const auto uds = OpenUds(0); + Post(uds, "COLLECT", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt64(properties, "counter", 9) == 0, "counter was not set"); + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt64(properties, "total", 100) == 0, "total was not set"); + }); + Require(!Trophy(context, 1).unlocked && !Trophy(context, 2).unlocked && unlocks.trophies.empty(), "ge or gt unlocked below its target"); + Post(uds, "COLLECT", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "counter", 10) == 0, "counter was not set"); + }); + Require(Trophy(context, 1).unlocked && !Trophy(context, 2).unlocked, "ge did not unlock at its target"); + Post(uds, "COLLECT", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetInt32(properties, "counter", 3) == 0, "counter was not set"); + Require(sceNpUniversalDataSystemEventPropertyObjectSetInt64(properties, "total", 101) == 0, "total was not set"); + }); + Require(Trophy(context, 2).unlocked, "gt did not unlock past its target"); + + Post(uds, "TIME", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "seconds", 6) == 0, "seconds was not set"); + }); + Require(!Trophy(context, 3).unlocked && !Trophy(context, 4).unlocked, "le or lt unlocked above its target"); + Post(uds, "TIME", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "seconds", 5) == 0, "seconds was not set"); + }); + Require(Trophy(context, 3).unlocked && !Trophy(context, 4).unlocked, "le did not unlock at its target"); + Post(uds, "TIME", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetInt32(properties, "seconds", -1) == 0, "seconds was not set"); + }); + Post(uds, "TIME", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetFloat32(properties, "seconds", 1.0f) == 0, "seconds was not set"); + }); + Require(!Trophy(context, 4).unlocked, "a negative or fractional value reached lt"); + Post(uds, "TIME", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "seconds", 2) == 0, "seconds was not set"); + }); + Require(Trophy(context, 4).unlocked, "lt did not unlock below its target"); + + Post(uds, "NOTHING", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "steps", 50) == 0, "steps was not set"); + }); + const auto stepped = [&uds](std::uint32_t steps) { + Post(uds, "STEPS", [steps](NpUniversalDataSystemEventPropertyObject* properties) { + NpUniversalDataSystemEventPropertyObject* player = nullptr; + Require(sceNpUniversalDataSystemEventPropertyObjectSetObject(properties, "player", nullptr, &player) == 0, "player was not set"); + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(player, "steps", steps) == 0, "steps was not set"); + }); + }; + stepped(4); + const auto partway = Trophy(context, 5); + Require(!partway.unlocked && partway.progress.type == 1 && partway.progress.value == 4 && !Trophy(context, 0).unlocked, "the progressive trophy does not report its progress"); + stepped(5); + const auto done = Trophy(context, 5); + Require(done.unlocked && done.progress.value == 5, "the progressive trophy did not unlock at its target"); + Require(Trophy(context, 0).unlocked, "the platinum did not unlock with the last trophy that names it"); + + const std::vector<std::pair<std::uint32_t, std::int32_t>> expected{{0, 1}, {0, 2}, {0, 3}, {0, 4}, {0, 5}, {0, 0}}; + Require(unlocks.trophies == expected, "the listener did not see each stat unlock and the platinum once, in order"); + const auto game = Game(context); + Require(game.unlocked_trophies == 6 && game.unlocked_platinum == 1 && game.unlocked_gold == 1 && game.unlocked_silver == 1 && game.unlocked_bronze == 3 && + game.progress_percentage == (6 + 2 + 3) * 100 / (6 + 2 + 5), + "the game does not count the unlocks by grade"); + Require(Read("_trophy/268435456_0.stats.txt") == "10 3\n20 101\n30 2\n40 5\n", "the stats were not stored next to the unlocks"); + const auto record = Read("_trophy/268435456_0.txt"); + for (const char* line : {"0 ", "1 ", "2 ", "3 ", "4 ", "5 "}) Require(record.find(std::string("\n") + line) != std::string::npos || record.starts_with(line), "an unlock was not stored"); +} + +static void IgnoreMissingAndMalformedFiles(Unlocks& unlocks) { + const auto missing = TrophyContext(1); + NpTrophy2GameDetails game{}; + Game(missing, &game); + Require(std::strcmp(game.title, "Game") == 0, "a title without trophy files does not report placeholders"); + Post(OpenUds(1), "COLLECT", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "counter", 10) == 0, "counter was not set"); + }); + Require(!Exists("_trophy/268435456_1.stats.txt"), "a title without stat rules stored a stat"); + + const auto brokenTrophies = TrophyContext(2); + Game(brokenTrophies, &game); + Require(std::strcmp(game.title, "Game") == 0, "a malformed trophy package was used"); + Post(OpenUds(2), "COLLECT", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "counter", 10) == 0, "counter was not set"); + }); + Require(Read("_trophy/268435456_2.stats.txt") == "10 10\n" && !Exists("_trophy/268435456_2.txt"), "a malformed trophy package disabled the stat rules, or stats unlocked a trophy without one"); + + const auto brokenRules = TrophyContext(3); + Game(brokenRules, &game); + Require(std::strcmp(game.title, "Synthetic Game") == 0, "malformed stat rules disabled the trophy package"); + Post(OpenUds(3), "COLLECT", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "counter", 10) == 0, "counter was not set"); + }); + Require(!Exists("_trophy/268435456_3.stats.txt") && !Trophy(brokenRules, 1).unlocked, "malformed stat rules were used"); + Require(unlocks.trophies.size() == 6, "a missing or malformed file unlocked a trophy"); +} + +int main() { + const auto previous = std::filesystem::current_path(); + const auto root = std::filesystem::temp_directory_path() / ("anyps5-trophy-stats-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(root); + std::filesystem::current_path(root); + Write("app0/sce_sys/param.json", R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}}})"); + const auto trophies = Ucp({{"tropconf.json", TropConf}, {"tropmeta_en-US.json", TropMeta}}); + const auto rules = Ucp({{"stats_definition.json", StatsDefinition}, {"stats_extraction.json", StatsExtraction}}); + Write("app0/sce_sys/trophy2/trophy00.ucp", trophies); + Write("app0/sce_sys/uds/uds00.ucp", rules); + Write("app0/sce_sys/trophy2/trophy02.ucp", "not a package"); + Write("app0/sce_sys/uds/uds02.ucp", rules); + Write("app0/sce_sys/trophy2/trophy03.ucp", trophies); + Write("app0/sce_sys/uds/uds03.ucp", rules.substr(0, rules.size() - 1)); + Write("_trophy/268435456_0.stats.txt", "40 2\n"); + + Unlocks unlocks; + const auto listener = NpTrophy2AddUnlockListener_nid_no_patch(Remember, &unlocks); + const auto context = TrophyContext(0); + KeepStatsAcrossRuns(context); + UnlockThroughStats(context, unlocks); + IgnoreMissingAndMalformedFiles(unlocks); + NpTrophy2RemoveUnlockListener_nid_no_patch(listener); + + std::filesystem::current_path(previous); + std::filesystem::remove_all(root); + std::puts("Trophy stat tests passed"); + return 0; +} diff --git a/docs/dev/debt/libSceNpTrophy2.md b/docs/dev/debt/libSceNpTrophy2.md index b7701b97c1..dbf8da133f 100644 --- a/docs/dev/debt/libSceNpTrophy2.md +++ b/docs/dev/debt/libSceNpTrophy2.md @@ -13,6 +13,6 @@ ### Functional - [Trophy queries](../../../core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp) report the title's [trophy data](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp) (`sce_sys/trophy2/trophyNN.ucp`) and the unlocks in the [trophy store](../../../core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp). Without the file, or when it is malformed, they report the old placeholders and `sceNpTrophy2GetTrophyInfoArray` lists only trophy 0. Icons are not read. Hidden trophies report their names and descriptions; whether the console blanks them before the unlock is not known. `progress_percentage` weighs gold 6, silver 2 and bronze 1 and leaves the platinum out, as KytyPS5 does (the ratio of the published PSN trophy points); the console's formula is not known. A trophy or group that is not in the file throws. A context that was not created or was destroyed reports placeholders and no unlocks and returns success; the console's errors for it are unknown. -- [Trophy data limits](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): a `trophyNN.ucp` larger than 256 MiB or listing more than 4096 files, and a JSON file in it that nests deeper than 64 levels or holds more than 1048576 values, are not used, and the title runs as without that file. The limits keep the memory the reader takes to the file and about 140 MiB of parsed JSON; the console's limits are not known. +- [Trophy data limits](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): a `trophyNN.ucp` or `udsNN.ucp` larger than 256 MiB or listing more than 4096 files, and a JSON file in it that nests deeper than 64 levels or holds more than 1048576 values, are not used, and the title runs as without that file. The limits keep the memory the reader takes to the file and about 140 MiB of parsed JSON; the console's limits are not known. - [Text language](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): texts come from `tropmeta_<tag>.json` for the console language, then `en-US`, then the file's `defaultLanguage`. The tags for languages 0 to 29 are KytyPS5's; `uk-UA` for 30 (Ukrainian) is an assumption. - [Platinum](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp): an unlock through any path unlocks the platinum that the trophy names in `platinumTrophyId` once every trophy naming that platinum is unlocked, as KytyPS5 does; trophies without `platinumTrophyId` are not required. A platinum or a progressive trophy can still be unlocked directly with `_UnlockTrophy`, which KytyPS5 refuses; the console's behaviour is not known. Without trophy data any id of 0 or more is stored, and the platinum is not unlocked when the trophies it needs are. diff --git a/docs/dev/debt/libSceNpUniversalDataSystem.md b/docs/dev/debt/libSceNpUniversalDataSystem.md index 3fcbc2508f..5dc32b7964 100644 --- a/docs/dev/debt/libSceNpUniversalDataSystem.md +++ b/docs/dev/debt/libSceNpUniversalDataSystem.md @@ -6,4 +6,5 @@ ### Functional -- [Trophy events](../../../core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp): `sceNpUniversalDataSystemPostEvent` stores an `_UnlockTrophy` event with an int32 `_trophy_id` of 0 or more as an unlock for the user and service label of its context, as KytyPS5 does. Unlocks that a title drives through statistics (`statsExtractionRuleArray` in `sce_sys/uds/udsNN.ucp` and `unlockCondition` in `sce_sys/trophy2/trophyNN.ucp`) are not evaluated, and `_UpdateTrophyProgress`, named only by a third-party unlocker, is logged once per trophy and not interpreted. A context need not be registered, and every post of an event that `sceNpUniversalDataSystemCreateEvent` returned and that is not yet destroyed returns success (a null or any other event pointer stops the game): KytyPS5's errors for an unknown or unregistered context and for a malformed unlock are not verified. The storage statistics stay zero. +- [Trophy events](../../../core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp): `sceNpUniversalDataSystemPostEvent` stores an `_UnlockTrophy` event with an int32 `_trophy_id` of 0 or more as an unlock for the user and service label of its context, as KytyPS5 does. `_UpdateTrophyProgress`, named only by a third-party unlocker, is logged once per trophy and not interpreted. A context need not be registered, and every post of an event that `sceNpUniversalDataSystemCreateEvent` returned and that is not yet destroyed returns success (a null or any other event pointer stops the game): KytyPS5's errors for an unknown or unregistered context and for a malformed unlock are not verified. The storage statistics stay zero. +- [Statistics](../../../core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp): every other event goes through the title's rules in `sce_sys/uds/udsNN.ucp`, which [libSceNpTrophy2](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp) reads. Only statistics with `"aggregation":"latest"` and `dataType` `uint32` or `uint64` are kept, as in KytyPS5; no public source documents the other aggregations or types, so their definitions and rules are skipped. Only rules whose condition is an `eventName` alone and whose input is a `$.name` path (dotted for nested objects) are applied. A value is used when it is an integer of any width that is not negative and fits the statistic's type and `minValue`/`maxValue`; KytyPS5 requires the statistic's exact width, and whether the console converts, clamps or rejects other values is not known. Statistics are stored per user and service label, and every trophy's `unlockCondition` is compared after each event that sets one. From 75ca9e469a1476096df26ed50add6dc7f3fed08e Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:29:17 +0000 Subject: [PATCH 074/103] docs: describe trophy statistics in USAGE --- docs/user/USAGE.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index c2f142dfe1..2686db9e09 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -159,13 +159,15 @@ Games ask for the console's language with `sceSystemServiceParamGetInt` and show ### Trophies -A game unlocks a trophy by posting a `_UnlockTrophy` event with an int32 `_trophy_id` property to `libSceNpUniversalDataSystem`. The unlock is stored in `_trophy/<user id>_<service label>.txt` in the directory the game runs in (the one that holds `app0/`), one `<trophy id> <unlock time>` line per trophy; the time is an RTC tick, microseconds since 0001-01-01 UTC. Delete the file before starting the game to lock its trophies again. Each new unlock prints one line, such as `[trophy] PPSA01556 unlocked trophy 12 for user 268435456`; unlocking a trophy again prints nothing. `sceNpTrophy2GetTrophyInfo`, `sceNpTrophy2GetTrophyInfoArray` and `sceNpTrophy2GetGameInfo` report the stored unlocks and their times to the game. Every posted `_trophy_id` of 0 or more is stored, even one the trophy data has no trophy for or makes the platinum or a progressive trophy. +A game unlocks a trophy by posting a `_UnlockTrophy` event with an int32 `_trophy_id` property to `libSceNpUniversalDataSystem`, or through the statistics rules described below. The unlock is stored in `_trophy/<user id>_<service label>.txt` in the directory the game runs in (the one that holds `app0/`), one `<trophy id> <unlock time>` line per trophy; the time is an RTC tick, microseconds since 0001-01-01 UTC. Delete the file before starting the game to lock its trophies again. Each new unlock prints one line, such as `[trophy] PPSA01556 unlocked trophy 12 for user 268435456`; unlocking a trophy again prints nothing. `sceNpTrophy2GetTrophyInfo`, `sceNpTrophy2GetTrophyInfoArray` and `sceNpTrophy2GetGameInfo` report the stored unlocks and their times to the game. Every posted `_trophy_id` of 0 or more is stored, even one the trophy data has no trophy for or makes the platinum or a progressive trophy. The trophy data comes from the game's own `app0/sce_sys/trophy2/trophyNN.ucp`, where `NN` is the service label (`00` for most games). The trophy queries report its names, descriptions and rewards in the console language (`ANYPS5_LANGUAGE`, see above), or in English (United States) or the data's default language when it has no text in that language, and its grades, groups, hidden flags, number of trophies and grade counts. A platinum trophy unlocks as soon as every trophy that names it in `platinumTrophyId` is unlocked. The first trophy query or event prints what was read, such as `[trophy] /app0/sce_sys/trophy2/trophy00.ucp: 52 trophies, 1 group, texts in en-US`. -`APS5_TRACE_UDS=1` prints every posted event, such as `[uds] context 1 posted _UnlockTrophy {"_trophy_id":12}`, and a `[uds]` line for a missing `trophyNN.ucp`. An event that is not used prints the reason in a `[trophy] ignored` line: an `_UnlockTrophy` event without an int32 `_trophy_id` of 0 or more, or posted to a context that does not exist, and the first `_UpdateTrophyProgress` event of each trophy, because trophy progress is not recorded. +Many games unlock trophies through statistics instead: `app0/sce_sys/uds/udsNN.ucp` names, for each event, the property whose value becomes a statistic, and a trophy's `unlockCondition` compares a statistic with a target. The latest value of each statistic is stored in `_trophy/<user id>_<service label>.stats.txt`, one `<stat id> <value>` line per statistic, and is kept across runs; delete it together with the unlock file to start over. A progressive trophy reports its statistic as progress. Only statistics that keep the latest value (`"aggregation":"latest"`) as `uint32` or `uint64`, and rules that copy one property of an event chosen by its name (`$.name` or `$.object.name`), are applied; the others are listed in `[trophy]` lines when the file is read. -A missing or malformed `trophyNN.ucp` makes the queries report placeholders, and a malformed one prints the reason once. Icons are not read: `sceNpTrophy2GetGameIcon`, `sceNpTrophy2GetGroupIcon`, `sceNpTrophy2GetTrophyIcon` and `sceNpTrophy2GetRewardIcon` stop the game with an error. So does an unlock file in `_trophy/` that is damaged (a line that is not a new `<id> <value>` pair, such as a line an edit left blank or ended with a carriage return) or cannot be read or written. No notification is shown, and callbacks registered with `sceNpTrophy2RegisterUnlockCallback` are not called. +`APS5_TRACE_UDS=1` prints every posted event, such as `[uds] context 1 posted _UnlockTrophy {"_trophy_id":12}`, every statistic an event sets, such as `[uds] stat 10 of user 268435456 is 10 (COLLECT $.counter)`, and a `[uds]` line for a missing `trophyNN.ucp` or `udsNN.ucp`. An event that is not used prints the reason in a `[trophy] ignored` line: an `_UnlockTrophy` event without an int32 `_trophy_id` of 0 or more, or posted to a context that does not exist; the first `_UpdateTrophyProgress` event of each trophy, because that event is not interpreted; and, once per rule, an event whose property is missing, negative, not an integer or outside the statistic's range. + +A missing or malformed `trophyNN.ucp` or `udsNN.ucp` turns off only what it provides, and a malformed one prints the reason once: without trophy data the queries report placeholders and statistics unlock nothing, and without statistics rules events change no statistic. Icons are not read: `sceNpTrophy2GetGameIcon`, `sceNpTrophy2GetGroupIcon`, `sceNpTrophy2GetTrophyIcon` and `sceNpTrophy2GetRewardIcon` stop the game with an error. So does an unlock or statistics file in `_trophy/` that is damaged (a line that is not a new `<id> <value>` pair, such as a line an edit left blank or ended with a carriage return) or cannot be read or written. No notification is shown, and callbacks registered with `sceNpTrophy2RegisterUnlockCallback` are not called. ### PlayGo dumps without metadata From e9611f13ae69cf7638df1fe1be09fb18b5e8fe6c Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:37:49 +0000 Subject: [PATCH 075/103] test(libSceNpUniversalDataSystem): check that a stat with a sum aggregation unlocks nothing A title whose uds file defines a stat with an aggregation other than latest gets one log line naming the stat. Posting an event for it through the exports stores no stat and unlocks no trophy. --- core/libs/tests/GuestNpTrophyStats.cpp | 59 ++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/core/libs/tests/GuestNpTrophyStats.cpp b/core/libs/tests/GuestNpTrophyStats.cpp index 2464aa36fc..a8e94f7516 100644 --- a/core/libs/tests/GuestNpTrophyStats.cpp +++ b/core/libs/tests/GuestNpTrophyStats.cpp @@ -12,6 +12,11 @@ #include <string> #include <utility> #include <vector> +#ifdef _WIN32 +#include <io.h> +#else +#include <unistd.h> +#endif struct NpUniversalDataSystemEvent; struct NpUniversalDataSystemEventPropertyObject; @@ -58,6 +63,8 @@ static const char StatsDefinition[] = R"({"statDefinitionArray":[ {"statId":10,"dataType":"uint64","aggregation":"latest"},{"statId":20,"dataType":"uint64","aggregation":"latest"}, {"statId":30,"dataType":"uint32","aggregation":"latest"},{"statId":40,"dataType":"uint32","aggregation":"latest"}]})"; +static const char SumDefinition[] = R"({"statDefinitionArray":[{"statId":10,"dataType":"uint64","aggregation":"sum"}]})"; + static const char StatsExtraction[] = R"({"statsExtractionRuleArray":[ {"condition":{"eventName":"COLLECT"},"action":{"input":"$.counter","output":{"statId":10}}}, {"condition":{"eventName":"COLLECT"},"action":{"input":"$.total","output":{"statId":20}}}, @@ -121,6 +128,43 @@ static bool Exists(const char* path) { return true; } +class StderrCapture { +public: + StderrCapture() { + std::fflush(stderr); + file = std::tmpfile(); + Require(file != nullptr, "cannot open a temporary file for stderr"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + Require(saved >= 0 && _dup2(_fileno(file), _fileno(stderr)) == 0, "cannot capture stderr"); +#else + saved = dup(fileno(stderr)); + Require(saved >= 0 && dup2(fileno(file), fileno(stderr)) == fileno(stderr), "cannot capture stderr"); +#endif + } + std::string Finish() { + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t count; (count = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, count); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } + +private: + std::FILE* file = nullptr; + int saved = -1; +}; + struct Uds { int context = 0; int handle = 0; @@ -275,6 +319,18 @@ static void IgnoreMissingAndMalformedFiles(Unlocks& unlocks) { Require(unlocks.trophies.size() == 6, "a missing or malformed file unlocked a trophy"); } +static void IgnoreOtherAggregations(Unlocks& unlocks) { + StderrCapture capture; + const auto context = TrophyContext(5); + Post(OpenUds(5), "COLLECT", [](NpUniversalDataSystemEventPropertyObject* properties) { + Require(sceNpUniversalDataSystemEventPropertyObjectSetUInt32(properties, "counter", 10) == 0, "counter was not set"); + }); + const auto log = capture.Finish(); + Require(log.find("[trophy] /app0/sce_sys/uds/uds05.ucp: stat 10 has aggregation \"sum\"; only \"latest\" is supported") != std::string::npos, + "a stat with a sum aggregation was not reported"); + Require(!Trophy(context, 1).unlocked && !Exists("_trophy/268435456_5.stats.txt") && unlocks.trophies.size() == 6, "a stat with a sum aggregation was stored or unlocked a trophy"); +} + int main() { const auto previous = std::filesystem::current_path(); const auto root = std::filesystem::temp_directory_path() / ("anyps5-trophy-stats-" + std::to_string(std::random_device{}())); @@ -289,6 +345,8 @@ int main() { Write("app0/sce_sys/uds/uds02.ucp", rules); Write("app0/sce_sys/trophy2/trophy03.ucp", trophies); Write("app0/sce_sys/uds/uds03.ucp", rules.substr(0, rules.size() - 1)); + Write("app0/sce_sys/trophy2/trophy05.ucp", trophies); + Write("app0/sce_sys/uds/uds05.ucp", Ucp({{"stats_definition.json", SumDefinition}, {"stats_extraction.json", StatsExtraction}})); Write("_trophy/268435456_0.stats.txt", "40 2\n"); Unlocks unlocks; @@ -297,6 +355,7 @@ int main() { KeepStatsAcrossRuns(context); UnlockThroughStats(context, unlocks); IgnoreMissingAndMalformedFiles(unlocks); + IgnoreOtherAggregations(unlocks); NpTrophy2RemoveUnlockListener_nid_no_patch(listener); std::filesystem::current_path(previous); From a8dd45faa3961654a2f6b796092e14c6435ee169 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:38:06 +0000 Subject: [PATCH 076/103] fix(libSceNpTrophy2): refuse a posted unlock of an unknown, platinum or progressive trophy With the title's trophy data loaded, an _UnlockTrophy posted through UDS no longer records an id the data does not have, the platinum, or a progressive trophy. Each refusal logs the trophy and the reason, returns false and calls no listener. The statistics path and the platinum award now use an internal record that keeps part 1's behaviour, so a progressive trophy still unlocks through its statistic and the platinum with the last trophy that names it. Without trophy data every id of 0 or more is recorded as before. --- .../libSceNpTrophy2/include/TitleTrophies.hpp | 1 + .../libSceNpTrophy2/include/TrophyStore.hpp | 1 + .../prx/libSceNpTrophy2/src/TitleTrophies.cpp | 10 +- .../prx/libSceNpTrophy2/src/TrophyStore.cpp | 30 +++-- .../prx/libSceNpTrophy2/tests/TrophyStore.cpp | 109 ++++++++++++++++++ core/libs/tests/GuestNpTrophyData.cpp | 15 +-- 6 files changed, 148 insertions(+), 18 deletions(-) diff --git a/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp b/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp index cfea54f627..9d4653450c 100644 --- a/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp +++ b/core/libs/prx/libSceNpTrophy2/include/TitleTrophies.hpp @@ -53,6 +53,7 @@ struct StatChanges { StatChanges ApplyStatRules(const StatRules& rules, const NpTrophy2Event& event, StatStore& stats, std::int32_t userId, std::uint32_t serviceLabel); std::vector<std::int32_t> ReachedTrophies(const TrophyPackage& package, const std::map<std::uint64_t, std::uint64_t>& stats, const UnlockedQuery& unlocked); std::optional<std::int32_t> PlatinumToUnlock(const TrophyPackage& package, std::int32_t trophyId, const UnlockedQuery& unlocked); +std::optional<std::string> PostedUnlockRefusal(const TrophyPackage& package, std::int32_t trophyId); std::optional<std::uint64_t> ProgressTarget(const TrophyDefinition& trophy); std::uint64_t TrophyProgress(const TrophyDefinition& trophy, std::optional<std::uint64_t> stat, bool unlocked); diff --git a/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp b/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp index 1418d70b58..d9ab341ae0 100644 --- a/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp +++ b/core/libs/prx/libSceNpTrophy2/include/TrophyStore.hpp @@ -94,6 +94,7 @@ void DestroyContext(int context); std::optional<ContextOwner> FindContext(int context); std::optional<std::uint64_t> ContextUnlockTick(int context, std::int32_t trophyId); std::size_t ContextUnlockCount(int context); +bool RecordUnlock(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId); } diff --git a/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp b/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp index 342f6ea8bb..813fa576f7 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp @@ -179,6 +179,14 @@ std::optional<std::int32_t> PlatinumToUnlock(const TrophyPackage& package, std:: return platinumId; } +std::optional<std::string> PostedUnlockRefusal(const TrophyPackage& package, std::int32_t trophyId) { + const auto trophy = package.trophies.find(trophyId); + if (trophy == package.trophies.end()) return "the title's trophy data has no such trophy"; + if (trophy->second.grade == GradePlatinum) return "it is the platinum, which unlocks with the last trophy that names it"; + if (trophy->second.progressive) return "it is progressive and unlocks through its statistic"; + return std::nullopt; +} + std::optional<std::uint64_t> ProgressTarget(const TrophyDefinition& trophy) { if (!trophy.progressive) return std::nullopt; return trophy.target; @@ -228,5 +236,5 @@ extern "C" void NpTrophy2ApplyEvent_nid_no_patch(std::int32_t userId, std::uint3 if (changes.updates.empty() || !title.package) return; auto stats = NpTrophy2::TitleStats().Values(userId, serviceLabel); for (const auto& update : changes.updates) stats[update.statId] = update.value; - for (const auto trophyId : NpTrophy2::ReachedTrophies(*title.package, stats, NpTrophy2::UnlockedBy(userId, serviceLabel))) NpTrophy2RecordUnlock_nid_no_patch(userId, serviceLabel, trophyId); + for (const auto trophyId : NpTrophy2::ReachedTrophies(*title.package, stats, NpTrophy2::UnlockedBy(userId, serviceLabel))) NpTrophy2::RecordUnlock(userId, serviceLabel, trophyId); } diff --git a/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp b/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp index 27f753283c..b19804fe10 100644 --- a/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp +++ b/core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp @@ -206,30 +206,40 @@ std::size_t ContextUnlockCount(int context) { return owner ? TitleStore().UnlockCount(owner->userId, owner->serviceLabel) : 0; } -} - -extern "C" bool NpTrophy2RecordUnlock_nid_no_patch(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId) { +bool RecordUnlock(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId) { const auto title = GetAppTitleId_nid_postfix(); const std::string_view titleId(title.value, static_cast<std::size_t>(std::find(std::begin(title.value), std::end(title.value), '\0') - std::begin(title.value))); - const NpTrophy2Unlock unlock{std::string(titleId), userId, serviceLabel, trophyId, NpTrophy2::CurrentTick()}; - if (!NpTrophy2::TitleStore().Unlock(userId, serviceLabel, trophyId, unlock.tick)) return false; + const NpTrophy2Unlock unlock{std::string(titleId), userId, serviceLabel, trophyId, CurrentTick()}; + if (!TitleStore().Unlock(userId, serviceLabel, trophyId, unlock.tick)) return false; std::fprintf(stderr, "[trophy] %s unlocked trophy %d for user %d\n", unlock.titleId.c_str(), trophyId, userId); - auto& registry = NpTrophy2::listeners(); + auto& registry = listeners(); std::lock_guard dispatching(registry.dispatch); - std::vector<NpTrophy2::Listener> current; + std::vector<Listener> current; { std::lock_guard lock(registry.mutex); current = registry.entries; } for (const auto& entry : current) { - if (NpTrophy2::registered(registry, entry.id)) entry.callback(entry.context, unlock); + if (registered(registry, entry.id)) entry.callback(entry.context, unlock); } - const auto* package = NpTrophy2::Package(serviceLabel); + const auto* package = Package(serviceLabel); if (package == nullptr) return true; - if (const auto platinum = NpTrophy2::PlatinumToUnlock(*package, trophyId, NpTrophy2::UnlockedBy(userId, serviceLabel))) NpTrophy2RecordUnlock_nid_no_patch(userId, serviceLabel, *platinum); + if (const auto platinum = PlatinumToUnlock(*package, trophyId, UnlockedBy(userId, serviceLabel))) RecordUnlock(userId, serviceLabel, *platinum); return true; } +} + +extern "C" bool NpTrophy2RecordUnlock_nid_no_patch(std::int32_t userId, std::uint32_t serviceLabel, std::int32_t trophyId) { + if (const auto* package = NpTrophy2::Package(serviceLabel)) { + if (const auto refusal = NpTrophy2::PostedUnlockRefusal(*package, trophyId)) { + std::fprintf(stderr, "[trophy] ignored the posted unlock of trophy %d for user %d: %s\n", trophyId, userId, refusal->c_str()); + return false; + } + } + return NpTrophy2::RecordUnlock(userId, serviceLabel, trophyId); +} + extern "C" std::uint64_t NpTrophy2AddUnlockListener_nid_no_patch(NpTrophy2UnlockListener listener, void* context) { if (listener == nullptr) { APS5_INVALID_ARG_EX; diff --git a/core/libs/prx/libSceNpTrophy2/tests/TrophyStore.cpp b/core/libs/prx/libSceNpTrophy2/tests/TrophyStore.cpp index 6a29e7e9c8..10352c9edf 100644 --- a/core/libs/prx/libSceNpTrophy2/tests/TrophyStore.cpp +++ b/core/libs/prx/libSceNpTrophy2/tests/TrophyStore.cpp @@ -1,4 +1,5 @@ #include "prx/libSceNpTrophy2/include/NpTrophy2Types.hpp" +#include "prx/libSceNpTrophy2/include/TitleTrophies.hpp" #include "prx/libSceNpTrophy2/include/TrophyStore.hpp" #include "prx/libc/include/general/VabiMacros.hpp" @@ -14,7 +15,13 @@ #include <stdexcept> #include <string> #include <thread> +#include <utility> #include <vector> +#ifdef _WIN32 +#include <io.h> +#else +#include <unistd.h> +#endif extern "C" { int APS5_VABI sceNpTrophy2CreateContext(int*, int, uint32_t, uint64_t); @@ -61,6 +68,69 @@ void write(const char* path, const std::string& text) { check(std::fclose(file) == 0 && written, "test file was not written"); } +void putBigEndian(std::string& data, std::size_t offset, std::uint64_t value, std::size_t size) { + for (std::size_t i = 0; i < size; ++i) data[offset + i] = static_cast<char>((value >> (8 * (size - 1 - i))) & 0xFF); +} + +std::string ucp(const std::vector<std::pair<std::string, std::string>>& files) { + std::size_t offset = 0x60 + files.size() * 0x40; + std::size_t size = offset; + for (const auto& file : files) size += file.second.size(); + std::string data(size, '\0'); + putBigEndian(data, 0, 0xB228C60A, 4); + putBigEndian(data, 4, 1, 4); + putBigEndian(data, 8, size, 8); + putBigEndian(data, 0x10, files.size(), 4); + putBigEndian(data, 0x14, 0x40, 4); + std::size_t entry = 0x60; + for (const auto& [name, contents] : files) { + data.replace(entry, name.size(), name); + putBigEndian(data, entry + 0x20, offset, 8); + putBigEndian(data, entry + 0x28, contents.size(), 8); + data.replace(offset, contents.size(), contents); + offset += contents.size(); + entry += 0x40; + } + return data; +} + +class StderrCapture { +public: + StderrCapture() { + std::fflush(stderr); + file = std::tmpfile(); + check(file != nullptr, "cannot open a temporary file for stderr"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + check(saved >= 0 && _dup2(_fileno(file), _fileno(stderr)) == 0, "cannot capture stderr"); +#else + saved = dup(fileno(stderr)); + check(saved >= 0 && dup2(fileno(file), fileno(stderr)) == fileno(stderr), "cannot capture stderr"); +#endif + } + std::string Finish() { + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t count; (count = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, count); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } + +private: + std::FILE* file = nullptr; + int saved = -1; +}; + void testPersistence() { const std::filesystem::path directory("store"); { @@ -176,6 +246,44 @@ void testQueries() { check(sceNpTrophy2DestroyContext(context) == 0 && sceNpTrophy2DestroyContext(other) == 0, "contexts were not destroyed"); } + +void testPostedUnlockRules() { + std::filesystem::create_directories("app0/sce_sys/trophy2"); + std::filesystem::create_directories("app0/sce_sys/uds"); + write("app0/sce_sys/trophy2/trophy04.ucp", ucp({{"tropconf.json", R"({"defaultLanguage":"en-US","trophies":[ + {"id":"0","grade":"P"}, + {"id":"1","grade":"G","platinumTrophyId":"0"}, + {"id":"2","grade":"B","platinumTrophyId":"0","unlockCondition":{"udsStatId":"40","comparator":"ge","targetValue":"5","progressive":true}}, + {"id":"3","grade":"B"}]})"}, + {"tropmeta_en-US.json", R"({"metadata":{"titleMetadata":{"name":"Rules"},"trophyMetadata":[]}})"}})); + write("app0/sce_sys/uds/uds04.ucp", ucp({{"stats_definition.json", R"({"statDefinitionArray":[{"statId":40,"dataType":"uint32","aggregation":"latest"}]})"}, + {"stats_extraction.json", R"({"statsExtractionRuleArray":[{"condition":{"eventName":"STEPS"},"action":{"input":"$.steps","output":{"statId":40}}}]})"}})); + check(NpTrophy2::Package(4) != nullptr, "the trophy data of service label 4 was not loaded"); + Seen seen; + const auto listener = NpTrophy2AddUnlockListener_nid_no_patch(remember, &seen); + StderrCapture capture; + const bool unknown = NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 4, 9); + const bool platinum = NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 4, 0); + const bool progressive = NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 4, 2); + const auto log = capture.Finish(); + check(!unknown && !platinum && !progressive, "an unknown, platinum or progressive trophy posted through UDS was recorded"); + check(NpTrophy2::TitleStore().UnlockCount(InitialUser, 4) == 0 && seen.unlocks.empty(), "a refused unlock reached the record or a listener"); + for (const char* line : {"[trophy] ignored the posted unlock of trophy 9 for user 268435456: the title's trophy data has no such trophy\n", + "[trophy] ignored the posted unlock of trophy 0 for user 268435456: it is the platinum, which unlocks with the last trophy that names it\n", + "[trophy] ignored the posted unlock of trophy 2 for user 268435456: it is progressive and unlocks through its statistic\n"}) { + check(log.find(line) != std::string::npos, "a refused unlock did not log its trophy and reason"); + } + check(NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 4, 1) && NpTrophy2RecordUnlock_nid_no_patch(InitialUser, 4, 3), "an ordinary trophy posted through UDS was not recorded"); + check(!NpTrophy2::TitleStore().UnlockTick(InitialUser, 4, 0), "the platinum unlocked before every trophy that names it"); + NpTrophy2ApplyEvent_nid_no_patch(InitialUser, 4, NpTrophy2Event{"STEPS", {{"steps", 5, "5"}}}); + check(NpTrophy2::TitleStore().UnlockTick(InitialUser, 4, 2).has_value(), "the progressive trophy did not unlock through its statistic"); + check(NpTrophy2::TitleStore().UnlockTick(InitialUser, 4, 0).has_value(), "the platinum did not unlock with the last trophy that names it"); + NpTrophy2RemoveUnlockListener_nid_no_patch(listener); + std::vector<std::int32_t> seenIds; + for (const auto& unlock : seen.unlocks) seenIds.push_back(unlock.trophyId); + check(seenIds == std::vector<std::int32_t>{1, 3, 2, 0}, "the listener did not see the posted, statistic and platinum unlocks once, in order"); +} + } int main() { @@ -187,6 +295,7 @@ int main() { testPersistence(); testDamagedRecords(); testQueries(); + testPostedUnlockRules(); testListenerRemovalWaitsForItsCallback(); std::filesystem::current_path(previous); std::filesystem::remove_all(root); diff --git a/core/libs/tests/GuestNpTrophyData.cpp b/core/libs/tests/GuestNpTrophyData.cpp index eabbe68eae..5c4eaf025a 100644 --- a/core/libs/tests/GuestNpTrophyData.cpp +++ b/core/libs/tests/GuestNpTrophyData.cpp @@ -193,7 +193,7 @@ static void ReportTrophyData(int context) { "the group list does not hold the base game and group 1"); } -static void UnlockPlatinum(int context) { +static void PostUnlocks(int context) { Unlocks unlocks; const auto listener = NpTrophy2AddUnlockListener_nid_no_patch(Remember, &unlocks); for (const int trophyId : {1, 2, 3, 6}) PostUnlock(trophyId); @@ -203,12 +203,13 @@ static void UnlockPlatinum(int context) { "the game does not count the unlocks by grade"); PostUnlock(4); Require(!Trophy(context, 0).unlocked, "the platinum unlocked before every trophy that names it"); - PostUnlock(5); - const auto progressive = Trophy(context, 5); - Require(progressive.unlocked && progressive.progress.value == 5, "an unlocked progressive trophy does not report its target as progress"); - Require(Trophy(context, 0).unlocked && Game(context).unlocked_platinum == 1, "the platinum did not unlock with the last trophy that names it"); + for (const int trophyId : {5, 0, 42}) PostUnlock(trophyId); + const auto refused = Game(context); + Require(!Trophy(context, 5).unlocked && !Trophy(context, 0).unlocked && refused.unlocked_trophies == 5 && refused.unlocked_platinum == 0 && refused.unlocked_bronze == 3 && + refused.progress_percentage == (6 + 2 + 3) * 100 / (6 + 2 + 5), + "a progressive, platinum or unknown trophy posted through UDS unlocked, or changed the counts"); NpTrophy2RemoveUnlockListener_nid_no_patch(listener); - Require(unlocks.trophies == (std::vector<std::int32_t>{1, 2, 3, 6, 4, 5, 0}), "the listener did not see each unlock and the platinum once, in order"); + Require(unlocks.trophies == (std::vector<std::int32_t>{1, 2, 3, 6, 4}), "the listener did not see each posted unlock once, in order, or saw a refused one"); } static void ReportPlaceholders() { @@ -237,7 +238,7 @@ int main() { const auto context = TrophyContext(0); ReportTrophyData(context); - UnlockPlatinum(context); + PostUnlocks(context); ReportPlaceholders(); std::filesystem::current_path(previous); From be9ade0f01ac3970e61bb18164acf11922c43910 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:38:18 +0000 Subject: [PATCH 077/103] docs: say which posted trophy unlocks are refused and what stops a title USAGE.md describes the refused posted unlocks and their log line, and says that the icon queries and a damaged or unwritable unlock or statistics file stop the game with an error. The debt entries of libSceNpTrophy2 and libSceNpUniversalDataSystem no longer say that any posted id is stored. Both USAGE.md and the libSceNpTrophy2 entry say that without trophy data the platinum is not unlocked when the other trophies are. --- docs/dev/debt/libSceNpTrophy2.md | 2 +- docs/dev/debt/libSceNpUniversalDataSystem.md | 2 +- docs/user/USAGE.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/dev/debt/libSceNpTrophy2.md b/docs/dev/debt/libSceNpTrophy2.md index dbf8da133f..7cb85f5310 100644 --- a/docs/dev/debt/libSceNpTrophy2.md +++ b/docs/dev/debt/libSceNpTrophy2.md @@ -15,4 +15,4 @@ - [Trophy queries](../../../core/libs/prx/libSceNpTrophy2/src/TrophyInfo.cpp) report the title's [trophy data](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp) (`sce_sys/trophy2/trophyNN.ucp`) and the unlocks in the [trophy store](../../../core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp). Without the file, or when it is malformed, they report the old placeholders and `sceNpTrophy2GetTrophyInfoArray` lists only trophy 0. Icons are not read. Hidden trophies report their names and descriptions; whether the console blanks them before the unlock is not known. `progress_percentage` weighs gold 6, silver 2 and bronze 1 and leaves the platinum out, as KytyPS5 does (the ratio of the published PSN trophy points); the console's formula is not known. A trophy or group that is not in the file throws. A context that was not created or was destroyed reports placeholders and no unlocks and returns success; the console's errors for it are unknown. - [Trophy data limits](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): a `trophyNN.ucp` or `udsNN.ucp` larger than 256 MiB or listing more than 4096 files, and a JSON file in it that nests deeper than 64 levels or holds more than 1048576 values, are not used, and the title runs as without that file. The limits keep the memory the reader takes to the file and about 140 MiB of parsed JSON; the console's limits are not known. - [Text language](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): texts come from `tropmeta_<tag>.json` for the console language, then `en-US`, then the file's `defaultLanguage`. The tags for languages 0 to 29 are KytyPS5's; `uk-UA` for 30 (Ukrainian) is an assumption. -- [Platinum](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp): an unlock through any path unlocks the platinum that the trophy names in `platinumTrophyId` once every trophy naming that platinum is unlocked, as KytyPS5 does; trophies without `platinumTrophyId` are not required. A platinum or a progressive trophy can still be unlocked directly with `_UnlockTrophy`, which KytyPS5 refuses; the console's behaviour is not known. Without trophy data any id of 0 or more is stored, and the platinum is not unlocked when the trophies it needs are. +- [Platinum](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp): an unlock through any path unlocks the platinum that the trophy names in `platinumTrophyId` once every trophy naming that platinum is unlocked, as KytyPS5 does; trophies without `platinumTrophyId` are not required. With the title's trophy data loaded, a posted `_UnlockTrophy` of a trophy that is not in the data, of a platinum or of a progressive trophy is not stored and prints a `[trophy] ignored` line, as KytyPS5 refuses them, so a progressive trophy unlocks only through its statistic; without trophy data any id of 0 or more is stored, including ids the data would make the platinum, a progressive trophy or no trophy, and the platinum is not unlocked when the trophies it needs are. The console's behaviour is not known. diff --git a/docs/dev/debt/libSceNpUniversalDataSystem.md b/docs/dev/debt/libSceNpUniversalDataSystem.md index 5dc32b7964..6c8a996278 100644 --- a/docs/dev/debt/libSceNpUniversalDataSystem.md +++ b/docs/dev/debt/libSceNpUniversalDataSystem.md @@ -6,5 +6,5 @@ ### Functional -- [Trophy events](../../../core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp): `sceNpUniversalDataSystemPostEvent` stores an `_UnlockTrophy` event with an int32 `_trophy_id` of 0 or more as an unlock for the user and service label of its context, as KytyPS5 does. `_UpdateTrophyProgress`, named only by a third-party unlocker, is logged once per trophy and not interpreted. A context need not be registered, and every post of an event that `sceNpUniversalDataSystemCreateEvent` returned and that is not yet destroyed returns success (a null or any other event pointer stops the game): KytyPS5's errors for an unknown or unregistered context and for a malformed unlock are not verified. The storage statistics stay zero. +- [Trophy events](../../../core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp): `sceNpUniversalDataSystemPostEvent` stores an `_UnlockTrophy` event with an int32 `_trophy_id` of 0 or more as an unlock for the user and service label of its context, as KytyPS5 does, except an id that [libSceNpTrophy2](../../../core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp) refuses when the title's trophy data is loaded (one not in the data, a platinum or a progressive trophy). `_UpdateTrophyProgress`, named only by a third-party unlocker, is logged once per trophy and not interpreted. A context need not be registered, and every post of an event that `sceNpUniversalDataSystemCreateEvent` returned and that is not yet destroyed returns success (a null or any other event pointer stops the game): KytyPS5's errors for an unknown or unregistered context and for a malformed unlock are not verified. The storage statistics stay zero. - [Statistics](../../../core/libs/prx/libSceNpUniversalDataSystem/src/Event.cpp): every other event goes through the title's rules in `sce_sys/uds/udsNN.ucp`, which [libSceNpTrophy2](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp) reads. Only statistics with `"aggregation":"latest"` and `dataType` `uint32` or `uint64` are kept, as in KytyPS5; no public source documents the other aggregations or types, so their definitions and rules are skipped. Only rules whose condition is an `eventName` alone and whose input is a `$.name` path (dotted for nested objects) are applied. A value is used when it is an integer of any width that is not negative and fits the statistic's type and `minValue`/`maxValue`; KytyPS5 requires the statistic's exact width, and whether the console converts, clamps or rejects other values is not known. Statistics are stored per user and service label, and every trophy's `unlockCondition` is compared after each event that sets one. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 2686db9e09..83c37d8b6a 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -159,7 +159,7 @@ Games ask for the console's language with `sceSystemServiceParamGetInt` and show ### Trophies -A game unlocks a trophy by posting a `_UnlockTrophy` event with an int32 `_trophy_id` property to `libSceNpUniversalDataSystem`, or through the statistics rules described below. The unlock is stored in `_trophy/<user id>_<service label>.txt` in the directory the game runs in (the one that holds `app0/`), one `<trophy id> <unlock time>` line per trophy; the time is an RTC tick, microseconds since 0001-01-01 UTC. Delete the file before starting the game to lock its trophies again. Each new unlock prints one line, such as `[trophy] PPSA01556 unlocked trophy 12 for user 268435456`; unlocking a trophy again prints nothing. `sceNpTrophy2GetTrophyInfo`, `sceNpTrophy2GetTrophyInfoArray` and `sceNpTrophy2GetGameInfo` report the stored unlocks and their times to the game. Every posted `_trophy_id` of 0 or more is stored, even one the trophy data has no trophy for or makes the platinum or a progressive trophy. +A game unlocks a trophy by posting a `_UnlockTrophy` event with an int32 `_trophy_id` property to `libSceNpUniversalDataSystem`, or through the statistics rules described below. The unlock is stored in `_trophy/<user id>_<service label>.txt` in the directory the game runs in (the one that holds `app0/`), one `<trophy id> <unlock time>` line per trophy; the time is an RTC tick, microseconds since 0001-01-01 UTC. Delete the file before starting the game to lock its trophies again. Each new unlock prints one line, such as `[trophy] PPSA01556 unlocked trophy 12 for user 268435456`; unlocking a trophy again prints nothing. `sceNpTrophy2GetTrophyInfo`, `sceNpTrophy2GetTrophyInfoArray` and `sceNpTrophy2GetGameInfo` report the stored unlocks and their times to the game. When the game's trophy data is loaded (see below), a posted unlock of a trophy that the data does not have, of the platinum, or of a progressive trophy is not stored and prints the reason, such as `[trophy] ignored the posted unlock of trophy 0 for user 268435456: it is the platinum, which unlocks with the last trophy that names it`; the platinum unlocks with the last trophy that names it, and a progressive trophy through its statistic. Without trophy data every posted id of 0 or more is stored, and the platinum is not unlocked when the other trophies are. The trophy data comes from the game's own `app0/sce_sys/trophy2/trophyNN.ucp`, where `NN` is the service label (`00` for most games). The trophy queries report its names, descriptions and rewards in the console language (`ANYPS5_LANGUAGE`, see above), or in English (United States) or the data's default language when it has no text in that language, and its grades, groups, hidden flags, number of trophies and grade counts. A platinum trophy unlocks as soon as every trophy that names it in `platinumTrophyId` is unlocked. The first trophy query or event prints what was read, such as `[trophy] /app0/sce_sys/trophy2/trophy00.ucp: 52 trophies, 1 group, texts in en-US`. From 8e927be27f54ce786217d1c0a46c4b85669c5cc0 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:30:58 +0000 Subject: [PATCH 078/103] docs: say that a malformed trophy or stat record line fails the query --- docs/dev/debt/libSceNpTrophy2.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/debt/libSceNpTrophy2.md b/docs/dev/debt/libSceNpTrophy2.md index 7cb85f5310..33aa1abb92 100644 --- a/docs/dev/debt/libSceNpTrophy2.md +++ b/docs/dev/debt/libSceNpTrophy2.md @@ -16,3 +16,4 @@ - [Trophy data limits](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): a `trophyNN.ucp` or `udsNN.ucp` larger than 256 MiB or listing more than 4096 files, and a JSON file in it that nests deeper than 64 levels or holds more than 1048576 values, are not used, and the title runs as without that file. The limits keep the memory the reader takes to the file and about 140 MiB of parsed JSON; the console's limits are not known. - [Text language](../../../core/libs/prx/libSceNpTrophy2/src/TrophyPackage.cpp): texts come from `tropmeta_<tag>.json` for the console language, then `en-US`, then the file's `defaultLanguage`. The tags for languages 0 to 29 are KytyPS5's; `uk-UA` for 30 (Ukrainian) is an assumption. - [Platinum](../../../core/libs/prx/libSceNpTrophy2/src/TitleTrophies.cpp): an unlock through any path unlocks the platinum that the trophy names in `platinumTrophyId` once every trophy naming that platinum is unlocked, as KytyPS5 does; trophies without `platinumTrophyId` are not required. With the title's trophy data loaded, a posted `_UnlockTrophy` of a trophy that is not in the data, of a platinum or of a progressive trophy is not stored and prints a `[trophy] ignored` line, as KytyPS5 refuses them, so a progressive trophy unlocks only through its statistic; without trophy data any id of 0 or more is stored, including ids the data would make the platinum, a progressive trophy or no trophy, and the platinum is not unlocked when the trophies it needs are. The console's behaviour is not known. +- [Damaged records](../../../core/libs/prx/libSceNpTrophy2/src/TrophyStore.cpp): a line in `_trophy/<user id>_<service label>.txt` or `.stats.txt` that is not a new `<id> <value>` pair (a blank line, a carriage return, a repeated or negative id, a value that is not a number) makes `readRecord` throw a `std::runtime_error` that names the file and the line. Every call that reads that record then throws it instead of returning, which stops the title as any throwing export does, so the title gets no return value; main returned 0 from these calls. The failed read is not kept, so each later call reads the file again and throws again until the file is fixed or deleted. The unlock record is read by `sceNpTrophy2GetTrophyInfo`, `sceNpTrophy2GetTrophyInfoArray` and `sceNpTrophy2GetGameInfo` for a created context, by `sceNpTrophy2GetGroupInfo` and `sceNpTrophy2GetGroupInfoArray` when the title's trophy data is loaded, by a `sceNpUniversalDataSystemPostEvent` of an `_UnlockTrophy` that the trophy data does not refuse, and by a statistic change when the trophy data is loaded; the stat record by a `sceNpUniversalDataSystemPostEvent` of an event that a statistics rule copies and by the trophy queries of a progressive trophy with a statistic. What the console does with a damaged record is not known. From a37797625cb7b2b2c96e44eb1fec0bc256c8858d Mon Sep 17 00:00:00 2001 From: Destroyers212x <65466724+Destroyers212x@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:36:51 +0000 Subject: [PATCH 079/103] ci(relinker): build and test the relinker for x86-64 on macOS under Rosetta --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 200e825e3a..cab59565e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,6 +140,7 @@ jobs: ...(full && linux ? [] : [{ os: 'ubuntu-latest', compiler: 'g++' }]), ...(full ? [] : [{ os: 'windows-latest', compiler: 'g++' }]), { os: 'macos-latest', compiler: 'clang++' }, + { os: 'macos-latest', compiler: 'clang++', architectures: 'x86_64' }, ]); if (context.eventName === 'push') { if (context.ref !== 'refs/heads/main') { @@ -314,7 +315,9 @@ jobs: 7z x "$RUNNER_TEMP/winlibs.zip" -oC:/winlibs > /dev/null - if: runner.os == 'Windows' run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH" - - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON + - if: ${{ matrix.architectures == 'x86_64' }} + run: sudo softwareupdate --install-rosetta --agree-to-license + - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON -DCMAKE_OSX_ARCHITECTURES=${{ matrix.architectures }} - run: cmake --build build-relinker --parallel 3 - run: ctest --test-dir build-relinker --output-on-failure --timeout 120 --parallel 4 From 4d741c26de4329c8391811dc76c3f7dadd9395ed Mon Sep 17 00:00:00 2001 From: MontyMcK <MontyMcK@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:30:43 +1300 Subject: [PATCH 080/103] feat(libSceHmd2): report a console without a PS VR2 headset --- core/libs/prx/libSceHmd2/CMakeLists.txt | 1 + core/libs/prx/libSceHmd2/Export.cpp | 154 +++++++++++++------ core/libs/prx/libSceHmd2/Hmd2.hpp | 66 ++++++++ core/libs/tests/GuestHmd2.cpp | 124 +++++++++++++-- core/libs/tests/ProcessParametersFixture.hpp | 91 +++++++++++ docs/dev/debt/libSceHmd2.md | 12 +- docs/dev/debt/libScePad.md | 1 + 7 files changed, 382 insertions(+), 67 deletions(-) create mode 100644 core/libs/prx/libSceHmd2/Hmd2.hpp create mode 100644 core/libs/tests/ProcessParametersFixture.hpp diff --git a/core/libs/prx/libSceHmd2/CMakeLists.txt b/core/libs/prx/libSceHmd2/CMakeLists.txt index 84efff8feb..fc0c32cbf4 100644 --- a/core/libs/prx/libSceHmd2/CMakeLists.txt +++ b/core/libs/prx/libSceHmd2/CMakeLists.txt @@ -1,5 +1,6 @@ add_library(libSceHmd2 SHARED EXCLUDE_FROM_ALL Export.cpp) target_include_directories(libSceHmd2 PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(libSceHmd2 PRIVATE libc) configure_windows_unwind(libSceHmd2) if(BUILD_TESTING) diff --git a/core/libs/prx/libSceHmd2/Export.cpp b/core/libs/prx/libSceHmd2/Export.cpp index e7361d9b35..b23019b0df 100644 --- a/core/libs/prx/libSceHmd2/Export.cpp +++ b/core/libs/prx/libSceHmd2/Export.cpp @@ -1,16 +1,117 @@ -#include <cstdint> -#include <cstddef> -#include "SceTypes.hpp" +#include "Hmd2.hpp" +#include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libSceUserService/UserService.hpp" +#include "SceTypes.hpp" + +#include <cstring> +#include <mutex> +#include <stdexcept> namespace { -constexpr std::int32_t SCE_HMD2_ERROR_UNSUPPORTED_FEATURE = static_cast<std::int32_t>(0x81110016); +constexpr std::int32_t DeviceHandle = 0x0e010000; +constexpr Hmd2::FieldOfView DefaultFieldOfView{1.84177089f, 0.947190642f, 1.32898104f, 1.32898104f}; + +std::mutex stateMutex; +bool initialized = false; +std::int32_t openHandle = 0; + +std::uint32_t CompiledSdkVersion() { + static const auto* processParameters = static_cast<const std::byte*>(ApplicationProcessParameters_nid_no_patch()); + std::uint32_t version; + std::memcpy(&version, processParameters + 0x10, sizeof(version)); + return version; +} + +std::int32_t GetDeviceInformation(Hmd2::DeviceInformation* info) { + if (!info) return Hmd2::ParameterNull; + *info = {}; + if (!initialized) return Hmd2::NotInitialized; + info->status = Hmd2::DeviceStatus::NotDetected; + if (CompiledSdkVersion() < Hmd2::UserIdSdkVersion) info->userId = -1; + info->panelWidth = 4000; + info->panelHeight = 2040; + info->latency90Hz = 5555; + info->latency120Hz = 4160; + info->virtualImageDistance = 2.0f; + return 0; +} } extern "C" { +std::int32_t APS5_VABI sceHmd2Initialize(const void* param) { + std::lock_guard lock(stateMutex); + if (initialized) return Hmd2::AlreadyInitialized; + if (!param) return Hmd2::ParameterNull; + initialized = true; + return 0; +} + +std::int32_t APS5_VABI sceHmd2Terminate() { + std::lock_guard lock(stateMutex); + if (!initialized) return Hmd2::NotInitialized; + initialized = false; + return 0; +} + +std::int32_t APS5_VABI sceHmd2Open(std::int32_t userId, std::int32_t type, std::int32_t index, const void* param) { + std::lock_guard lock(stateMutex); + if (!initialized) return Hmd2::NotInitialized; + if (type != 0 || index != 0 || param) return Hmd2::ParameterInvalid; + if (userId != USER_SERVICE_INITIAL_USER_ID) { + if (CompiledSdkVersion() >= Hmd2::UserIdSdkVersion) return Hmd2::ParameterInvalid; + throw std::runtime_error("sceHmd2Open: user is not the initial user"); + } + if (openHandle != 0) return Hmd2::AlreadyOpened; + openHandle = DeviceHandle; + return DeviceHandle; +} + +std::int32_t APS5_VABI sceHmd2Close(std::int32_t handle) { + std::lock_guard lock(stateMutex); + if (!initialized) return Hmd2::NotInitialized; + if (handle != openHandle) return Hmd2::InvalidHandle; + openHandle = 0; + return 0; +} + +std::int32_t APS5_VABI sceHmd2GetDeviceInformation(Hmd2::DeviceInformation* info) { + std::lock_guard lock(stateMutex); + return GetDeviceInformation(info); +} + +std::int32_t APS5_VABI sceHmd2GetDeviceInformationByHandle(std::int32_t handle, Hmd2::DeviceInformation* info) { + std::lock_guard lock(stateMutex); + if (handle != openHandle) return Hmd2::InvalidHandle; + return GetDeviceInformation(info); +} + +std::int32_t APS5_VABI sceHmd2GetFieldOfView(std::int32_t handle, Hmd2::FieldOfView* fieldOfView) { + std::lock_guard lock(stateMutex); + if (!fieldOfView) return Hmd2::ParameterNull; + if (handle == 0 || handle != openHandle) return Hmd2::InvalidHandle; + if (!initialized) return Hmd2::NotInitialized; + *fieldOfView = DefaultFieldOfView; + return 0; +} + +std::int32_t APS5_VABI sceHmd2GetFieldOfViewWithoutHandle(Hmd2::FieldOfView* fieldOfView) { + std::lock_guard lock(stateMutex); + if (!fieldOfView) return Hmd2::ParameterNull; + if (!initialized) return Hmd2::NotInitialized; + *fieldOfView = DefaultFieldOfView; + return 0; +} + +std::int32_t APS5_VABI sceHmd2SetVibration(std::int32_t handle, std::uint32_t level) { + (void)level; + std::lock_guard lock(stateMutex); + return handle == openHandle ? 0 : Hmd2::InvalidHandle; +} + int APS5_VABI sceHmd2ReprojectionGetStatus() { NotImplemented_nid_no_patch(__func__); return 0; @@ -21,31 +122,6 @@ int APS5_VABI sceHmd2ReprojectionQuerySeeThroughBufferSizeAlign() { return 0; } -std::int32_t APS5_VABI sceHmd2Initialize(const void* param) { - (void)param; - return SCE_HMD2_ERROR_UNSUPPORTED_FEATURE; -} - -int APS5_VABI sceHmd2GetFieldOfView() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceHmd2GetDeviceInformationByHandle() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceHmd2Terminate() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceHmd2Close() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI sceHmd2GazeGetResult() { NotImplemented_nid_no_patch(__func__); return 0; @@ -56,21 +132,6 @@ int APS5_VABI sceHmd2GazeGetResultForFoveatedRendering() { return 0; } -int APS5_VABI sceHmd2GetDeviceInformation() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceHmd2GetFieldOfViewWithoutHandle() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceHmd2Open() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI sceHmd2ReprojectionBeginFrame() { NotImplemented_nid_no_patch(__func__); return 0; @@ -161,9 +222,4 @@ int APS5_VABI sceHmd2ReprojectionTerminate() { return 0; } -int APS5_VABI sceHmd2SetVibration() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - } diff --git a/core/libs/prx/libSceHmd2/Hmd2.hpp b/core/libs/prx/libSceHmd2/Hmd2.hpp new file mode 100644 index 0000000000..3594b32bd1 --- /dev/null +++ b/core/libs/prx/libSceHmd2/Hmd2.hpp @@ -0,0 +1,66 @@ +#ifndef CORE_LIBS_PRX_LIBSCEHMD2_HMD2_HPP +#define CORE_LIBS_PRX_LIBSCEHMD2_HMD2_HPP + +#include "prx/libc/include/general/VabiMacros.hpp" + +#include <cstddef> +#include <cstdint> + +namespace Hmd2 { + +inline constexpr std::int32_t AlreadyInitialized = static_cast<std::int32_t>(0x8a720001); +inline constexpr std::int32_t NotInitialized = static_cast<std::int32_t>(0x8a720002); +inline constexpr std::int32_t InvalidHandle = static_cast<std::int32_t>(0x8a720003); +inline constexpr std::int32_t AlreadyOpened = static_cast<std::int32_t>(0x8a720006); +inline constexpr std::int32_t ParameterNull = static_cast<std::int32_t>(0x8a720008); +inline constexpr std::int32_t ParameterInvalid = static_cast<std::int32_t>(0x8a720009); + +inline constexpr std::uint32_t UserIdSdkVersion = 0x07000000; + +enum class DeviceStatus : std::uint32_t { + NotDetected = 2, +}; + +struct DeviceInformation { + DeviceStatus status; + std::int32_t userId; + std::uint32_t panelWidth; + std::uint32_t panelHeight; + std::uint16_t latency90Hz; + std::uint16_t latency120Hz; + std::uint8_t hmuMount; + std::uint8_t lensSeparationDistance; + std::uint8_t reserved[2]; + float virtualImageDistance; +}; + +struct FieldOfView { + float tanOut; + float tanIn; + float tanTop; + float tanBottom; +}; + +static_assert(sizeof(DeviceInformation) == 28); +static_assert(offsetof(DeviceInformation, userId) == 4); +static_assert(offsetof(DeviceInformation, panelWidth) == 8); +static_assert(offsetof(DeviceInformation, latency90Hz) == 16); +static_assert(offsetof(DeviceInformation, hmuMount) == 20); +static_assert(offsetof(DeviceInformation, virtualImageDistance) == 24); +static_assert(sizeof(FieldOfView) == 16); + +} + +extern "C" { +std::int32_t APS5_VABI sceHmd2Initialize(const void* param); +std::int32_t APS5_VABI sceHmd2Terminate(); +std::int32_t APS5_VABI sceHmd2Open(std::int32_t userId, std::int32_t type, std::int32_t index, const void* param); +std::int32_t APS5_VABI sceHmd2Close(std::int32_t handle); +std::int32_t APS5_VABI sceHmd2GetDeviceInformation(Hmd2::DeviceInformation* info); +std::int32_t APS5_VABI sceHmd2GetDeviceInformationByHandle(std::int32_t handle, Hmd2::DeviceInformation* info); +std::int32_t APS5_VABI sceHmd2GetFieldOfView(std::int32_t handle, Hmd2::FieldOfView* fieldOfView); +std::int32_t APS5_VABI sceHmd2GetFieldOfViewWithoutHandle(Hmd2::FieldOfView* fieldOfView); +std::int32_t APS5_VABI sceHmd2SetVibration(std::int32_t handle, std::uint32_t level); +} + +#endif diff --git a/core/libs/tests/GuestHmd2.cpp b/core/libs/tests/GuestHmd2.cpp index 85af8b0350..8ec9cbeb30 100644 --- a/core/libs/tests/GuestHmd2.cpp +++ b/core/libs/tests/GuestHmd2.cpp @@ -1,17 +1,18 @@ -#include "prx/libc/include/general/VabiMacros.hpp" +#include "ProcessParametersFixture.hpp" +#include "prx/libSceHmd2/Hmd2.hpp" +#include "prx/libSceUserService/UserService.hpp" +#include <array> #include <cstdint> #include <cstdio> #include <cstdlib> +#include <cstring> #include <stdexcept> -extern "C" { -std::int32_t APS5_VABI sceHmd2Initialize(const void* param); -int APS5_VABI sceHmd2Open(); -} - namespace { +alignas(16) std::array<std::byte, 0x40> process{}; + void Require(bool condition, const char* message) { if (!condition) { std::fprintf(stderr, "HMD2: %s\n", message); @@ -19,19 +20,120 @@ void Require(bool condition, const char* message) { } } +void SetSdkVersion(std::uint32_t version) { + std::memcpy(process.data() + 0x10, &version, sizeof(version)); } -int main() { +void InstallProcessParameters() { + const std::uint64_t size = process.size(); + const std::uint32_t magic = 0x4942524f; + std::memcpy(process.data(), &size, sizeof(size)); + std::memcpy(process.data() + 8, &magic, sizeof(magic)); + Require(ProcessParametersFixture::Install(process.data(), process.size()), "process parameters were not installed"); +} + +bool IsDefaultFieldOfView(const Hmd2::FieldOfView& fieldOfView) { + return fieldOfView.tanOut == 1.84177089f && fieldOfView.tanIn == 0.947190642f && + fieldOfView.tanTop == 1.32898104f && fieldOfView.tanBottom == 1.32898104f; +} + +void RequireNotDetected(const Hmd2::DeviceInformation& info, std::int32_t userId) { + Require(info.status == Hmd2::DeviceStatus::NotDetected, "device status is not NOT_DETECTED"); + Require(info.userId == userId, "device user does not match the SDK version"); + Require(info.panelWidth == 4000 && info.panelHeight == 2040, "panel resolution differs from the firmware"); + Require(info.latency90Hz == 5555 && info.latency120Hz == 4160, "flip latencies differ from the firmware"); + Require(info.hmuMount == 0 && info.lensSeparationDistance == 0, "a missing headset reported a mount or lens separation"); + Require(info.reserved[0] == 0 && info.reserved[1] == 0, "reserved bytes were not cleared"); + Require(info.virtualImageDistance == 2.0f, "virtual image distance differs from the firmware"); +} + +void TestBeforeInitialize() { + Hmd2::DeviceInformation info; + std::memset(&info, 0xcc, sizeof(info)); + Require(sceHmd2GetDeviceInformation(&info) == Hmd2::NotInitialized, "device information before initialization"); + const Hmd2::DeviceInformation cleared{}; + Require(std::memcmp(&info, &cleared, sizeof(info)) == 0, "device information was not cleared before the initialization check"); + Require(sceHmd2GetDeviceInformation(nullptr) == Hmd2::ParameterNull, "null device information before initialization"); + Hmd2::FieldOfView fieldOfView{}; + Require(sceHmd2GetFieldOfViewWithoutHandle(&fieldOfView) == Hmd2::NotInitialized, "field of view before initialization"); + Require(sceHmd2GetFieldOfView(0, &fieldOfView) == Hmd2::InvalidHandle, "field of view of handle 0 before initialization"); + Require(sceHmd2GetDeviceInformationByHandle(0, &info) == Hmd2::NotInitialized, "device information of handle 0 before initialization"); + Require(sceHmd2Open(USER_SERVICE_INITIAL_USER_ID, 0, 0, nullptr) == Hmd2::NotInitialized, "open before initialization"); + Require(sceHmd2Close(0) == Hmd2::NotInitialized, "close before initialization"); + Require(sceHmd2Terminate() == Hmd2::NotInitialized, "terminate before initialization"); + Require(sceHmd2SetVibration(0, 0) == 0, "vibration of the unopened handle 0 failed"); +} + +void TestInitialized() { const std::uint8_t param[16]{}; - Require(sceHmd2Initialize(param) == static_cast<std::int32_t>(0x81110016), "initialization did not report the unsupported feature"); - Require(sceHmd2Initialize(nullptr) == static_cast<std::int32_t>(0x81110016), "initialization without a param did not report the unsupported feature"); + Require(sceHmd2Initialize(nullptr) == Hmd2::ParameterNull, "initialization without a param"); + Require(sceHmd2Initialize(param) == 0, "initialization failed"); + Require(sceHmd2Initialize(param) == Hmd2::AlreadyInitialized, "second initialization"); + + Hmd2::DeviceInformation info; + std::memset(&info, 0xcc, sizeof(info)); + SetSdkVersion(0x06000000); + Require(sceHmd2GetDeviceInformation(&info) == 0, "device information failed"); + RequireNotDetected(info, -1); + SetSdkVersion(0x07590001); + std::memset(&info, 0xcc, sizeof(info)); + Require(sceHmd2GetDeviceInformation(&info) == 0, "device information for SDK 7.59 failed"); + RequireNotDetected(info, 0); + + Hmd2::FieldOfView fieldOfView{}; + Require(sceHmd2GetFieldOfViewWithoutHandle(&fieldOfView) == 0 && IsDefaultFieldOfView(fieldOfView), "default field of view"); + Require(sceHmd2GetFieldOfViewWithoutHandle(nullptr) == Hmd2::ParameterNull, "null field of view"); + Require(sceHmd2GetFieldOfView(12345, &fieldOfView) == Hmd2::InvalidHandle, "field of view of an unknown handle"); + Require(sceHmd2GetFieldOfView(12345, nullptr) == Hmd2::ParameterNull, "null field of view of an unknown handle"); + Require(sceHmd2GetDeviceInformationByHandle(12345, &info) == Hmd2::InvalidHandle, "device information of an unknown handle"); + Require(sceHmd2SetVibration(12345, 0) == Hmd2::InvalidHandle, "vibration of an unknown handle"); + Require(sceHmd2Close(12345) == Hmd2::InvalidHandle, "close of an unknown handle"); + + Require(sceHmd2Open(USER_SERVICE_INITIAL_USER_ID, 1, 0, nullptr) == Hmd2::ParameterInvalid, "open with a device type"); + Require(sceHmd2Open(USER_SERVICE_INITIAL_USER_ID, 0, 1, nullptr) == Hmd2::ParameterInvalid, "open with a device index"); + Require(sceHmd2Open(USER_SERVICE_INITIAL_USER_ID, 0, 0, param) == Hmd2::ParameterInvalid, "open with a param"); + Require(sceHmd2Open(USER_SERVICE_INITIAL_USER_ID + 1, 0, 0, nullptr) == Hmd2::ParameterInvalid, "open for another user on SDK 7.59"); + + const auto handle = sceHmd2Open(USER_SERVICE_INITIAL_USER_ID, 0, 0, nullptr); + Require(handle > 0, "open without a headset failed"); + Require(sceHmd2Open(USER_SERVICE_INITIAL_USER_ID, 0, 0, nullptr) == Hmd2::AlreadyOpened, "second open"); + std::memset(&info, 0xcc, sizeof(info)); + Require(sceHmd2GetDeviceInformationByHandle(handle, &info) == 0, "device information by handle failed"); + RequireNotDetected(info, 0); + fieldOfView = {}; + Require(sceHmd2GetFieldOfView(handle, &fieldOfView) == 0 && IsDefaultFieldOfView(fieldOfView), "field of view by handle"); + Require(sceHmd2SetVibration(handle, 26) == 0, "vibration of a disconnected headset failed"); + Require(sceHmd2Close(handle) == 0, "close failed"); + Require(sceHmd2Close(handle) == Hmd2::InvalidHandle, "second close"); + Require(sceHmd2Close(0) == 0, "close of the unopened handle 0 failed"); + + const auto kept = sceHmd2Open(USER_SERVICE_INITIAL_USER_ID, 0, 0, nullptr); + Require(kept > 0, "reopen failed"); + Require(sceHmd2Terminate() == 0, "terminate failed"); + Require(sceHmd2Terminate() == Hmd2::NotInitialized, "second terminate"); + Require(sceHmd2GetFieldOfView(kept, &fieldOfView) == Hmd2::NotInitialized, "field of view after terminate"); + Require(sceHmd2Close(kept) == Hmd2::NotInitialized, "close after terminate"); + Require(sceHmd2Initialize(param) == 0, "reinitialization failed"); + Require(sceHmd2Open(USER_SERVICE_INITIAL_USER_ID, 0, 0, nullptr) == Hmd2::AlreadyOpened, "terminate closed the open handle"); + Require(sceHmd2Close(kept) == 0, "close after reinitialization failed"); + + SetSdkVersion(0x06000000); bool threw = false; try { - sceHmd2Open(); + sceHmd2Open(USER_SERVICE_INITIAL_USER_ID + 1, 0, 0, nullptr); } catch (const std::runtime_error&) { threw = true; } - Require(threw, "open after failed initialization did not throw"); + Require(threw, "open for another user before SDK 7.00 did not throw"); + Require(sceHmd2Terminate() == 0, "final terminate failed"); +} + +} + +int main() { + InstallProcessParameters(); + TestBeforeInitialize(); + TestInitialized(); std::puts("HMD2 tests passed"); return 0; } diff --git a/core/libs/tests/ProcessParametersFixture.hpp b/core/libs/tests/ProcessParametersFixture.hpp new file mode 100644 index 0000000000..794ba40ac9 --- /dev/null +++ b/core/libs/tests/ProcessParametersFixture.hpp @@ -0,0 +1,91 @@ +#ifndef CORE_LIBS_TESTS_PROCESSPARAMETERSFIXTURE_HPP +#define CORE_LIBS_TESTS_PROCESSPARAMETERSFIXTURE_HPP + +#include <cstddef> +#include <cstdint> +#include <cstring> +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include <windows.h> +#elif defined(__linux__) +#include "prx/libc/include/specifics/linux/ElfTypes.hpp" +#include <sys/mman.h> +#include <unistd.h> +#endif + +namespace ProcessParametersFixture { +namespace { + +#ifdef _WIN32 +[[gnu::section(".proctst"), gnu::used]] alignas(4) std::uint32_t metadata[2]{}; +#endif + +bool Install(std::byte* process, std::size_t size) { +#ifdef _WIN32 + auto* image = reinterpret_cast<std::byte*>(GetModuleHandleW(nullptr)); + if (image == nullptr) return false; + auto* dos = reinterpret_cast<IMAGE_DOS_HEADER*>(image); + auto* nt = reinterpret_cast<IMAGE_NT_HEADERS64*>(image + dos->e_lfanew); + auto* sections = IMAGE_FIRST_SECTION(nt); + IMAGE_SECTION_HEADER* target = nullptr; + for (unsigned index = 0; index < nt->FileHeader.NumberOfSections; ++index) { + if (std::memcmp(sections[index].Name, ".proctst", 8) == 0) { + target = §ions[index]; + break; + } + } + if (target == nullptr) return false; + metadata[0] = static_cast<std::uint32_t>(process - image); + metadata[1] = static_cast<std::uint32_t>(size); + DWORD oldProtect = 0; + if (VirtualProtect(target, sizeof(*target), PAGE_READWRITE, &oldProtect) == 0) return false; + std::memcpy(target->Name, ".procpar", 8); + target->Misc.VirtualSize = sizeof(metadata); + DWORD restored = 0; + VirtualProtect(target, sizeof(*target), oldProtect, &restored); + return true; +#elif defined(__linux__) + struct Request { + std::byte* process; + std::size_t size; + bool installed; + } request{process, size, false}; + dl_iterate_phdr([](dl_phdr_info* image, std::size_t, void* data) { + auto& request = *static_cast<Request*>(data); + if (image->dlpi_name != nullptr && image->dlpi_name[0] != '\0') return 0; + for (std::uint16_t index = 0; index < image->dlpi_phnum; ++index) { + const auto& header = image->dlpi_phdr[index]; + if (header.p_type == PT_LOAD || header.p_type == PT_GNU_EH_FRAME || header.p_type < 0x60000000u) continue; + auto* mutableHeader = const_cast<Elf64_Phdr*>(&header); + const auto pageSize = static_cast<std::uintptr_t>(sysconf(_SC_PAGESIZE)); + const auto pageStart = reinterpret_cast<std::uintptr_t>(mutableHeader) & ~(pageSize - 1); + const auto pageEnd = (reinterpret_cast<std::uintptr_t>(mutableHeader + 1) + pageSize - 1) & ~(pageSize - 1); + int protection = PROT_READ; + for (std::uint16_t loadIndex = 0; loadIndex < image->dlpi_phnum; ++loadIndex) { + const auto& load = image->dlpi_phdr[loadIndex]; + const auto start = image->dlpi_addr + load.p_vaddr; + if (load.p_type == PT_LOAD && pageStart >= (start & ~(pageSize - 1)) && pageStart < start + load.p_memsz && (load.p_flags & PF_X)) protection |= PROT_EXEC; + } + if (mprotect(reinterpret_cast<void*>(pageStart), pageEnd - pageStart, protection | PROT_WRITE) != 0) return 1; + mutableHeader->p_type = 0x61000001; + mutableHeader->p_vaddr = reinterpret_cast<std::uintptr_t>(request.process) - image->dlpi_addr; + mutableHeader->p_paddr = mutableHeader->p_vaddr; + mutableHeader->p_filesz = request.size; + mutableHeader->p_memsz = request.size; + request.installed = mprotect(reinterpret_cast<void*>(pageStart), pageEnd - pageStart, protection) == 0; + return 1; + } + return 1; + }, &request); + return request.installed; +#else + return false; +#endif +} + +} +} + +#endif diff --git a/docs/dev/debt/libSceHmd2.md b/docs/dev/debt/libSceHmd2.md index 9b27109ad4..70033deefa 100644 --- a/docs/dev/debt/libSceHmd2.md +++ b/docs/dev/debt/libSceHmd2.md @@ -2,16 +2,14 @@ ### Silent stubs -- [libSceHmd2](../../../core/libs/prx/libSceHmd2/Export.cpp) has no PS VR2 runtime: `sceHmd2Initialize` fails, and every other export it has (and every [libSceVrTracker2](../../../core/libs/prx/libSceVrTracker2/Export.cpp) export) throws. PPSA01317 skips its whole VR set-up when initialization fails +- [libSceHmd2](../../../core/libs/prx/libSceHmd2/Export.cpp) reports a console without a PS VR2 headset, as firmware 13.20's libSceHmd2 does with the `/dev/hmd2_cmd` state measured on a retail PS5 with none connected (connected flag 0, status 2, user -1): initialization succeeds, `sceHmd2Open` returns a handle, device information reports `NotDetected` with the firmware's fixed panel, latency and image distance values, both field-of-view calls return the firmware's default tangents and `sceHmd2SetVibration` succeeds without vibrating. The reprojection and gaze exports and every [libSceVrTracker2](../../../core/libs/prx/libSceVrTracker2/Export.cpp) export throw. PPSA01317, PPSA07411, PPSA09030, PPSA15878 and PPSA29866, which skipped their VR set-up while initialization failed, have not been run with initialization succeeding ### Unknown function info - PPSA29866 imports declared without parameters, signatures unknown, added without an implementation because the title fails to load without them: [sceHmd2ReprojectionQuerySeeThroughBufferSizeAlign](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2), [sceVideoOutSetFlipMaster and sceVideoOutClearFlipMaster](../../../core/libs/prx/libSceVideoOut/Unimplemented.cpp) (libSceVideoOut) - PPSA09030 imports from its `Media/Plugins/psvr2.prx` declared without parameters, signatures unknown, added without an implementation because the title fails to load the plugin without them: [sceHmd2ReprojectionGetStatus](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2), [sceVrTracker2GetCoordinateSystem and sceVrTracker2GetPlayAreaBoundaryGeometry](../../../core/libs/prx/libSceVrTracker2/Export.cpp) (libSceVrTracker2) -- [sceHmd2Initialize](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the error the console returns when the PS VR2 runtime is unavailable is unknown; it returns libSceHmd's `UNSUPPORTED_FEATURE` (0x81110016). PPSA01317 passes a pointer to 16 zeroed bytes and only tests the result for non-zero -- PPSA01317 imports declared without parameters, signatures unknown: the remaining [libSceHmd2](../../../core/libs/prx/libSceHmd2/Export.cpp) and [libSceVrTracker2](../../../core/libs/prx/libSceVrTracker2/Export.cpp) exports. Names from the aerolib NID list +- [sceHmd2Initialize](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the firmware also fails with 0x8a72001f when `sceKernelGetAppInfo` fails and with 0x8a72000a for some applications, depending on app info fields at 0x2c, 0x4b and 0x4d whose meaning is unknown, and then calls `sceMbusInit`; none of this is modelled +- [sceHmd2Open](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the handle is the value the console's driver returned to the first open; titles built with an SDK before 7.00 may open for any user, and the firmware's `sceMbusAddHandleByUserId` result for a user other than the initial one is unknown, so that case throws. 0x8a720006 for a second open is named `AlreadyOpened` from its use +- [sceHmd2SetVibration](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the second parameter is checked against 25 only while a headset is connected; its name and unit are unknown. The field of view member names are assumed: of the firmware's default tangents the first is the widest and the last two are equal +- PPSA01317 imports declared without parameters, signatures unknown: the [libSceHmd2](../../../core/libs/prx/libSceHmd2/Export.cpp) exports that throw and the remaining [libSceVrTracker2](../../../core/libs/prx/libSceVrTracker2/Export.cpp) exports. Names from the aerolib NID list - PPSA15878 imports declared without parameters, signatures unknown: the [libSceHmd2](../../../core/libs/prx/libSceHmd2/Export.cpp) reprojection and gaze exports, `sceVrTracker2Finalize`, `sceVrTracker2GetPlayAreaOrientedBoundingBox` and `sceVrTracker2LocateCoordinateSystem` in [libSceVrTracker2](../../../core/libs/prx/libSceVrTracker2/Export.cpp), and the `scePadVrController` trigger effect exports in [libScePad](../../../core/libs/prx/libScePad/Export.cpp). Names from the aerolib NID list - -### Functional - -- PPSA07411 PS VR2 imports declared without parameters, signatures unknown, added without an implementation because the title fails to load without them: [sceHmd2GetDeviceInformationByHandle, sceHmd2GetFieldOfView and sceHmd2Terminate](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2), [scePadVrControllerSetVibration, scePadVrControllerSetVibrationMode and scePadVrControllerReadCombinedState](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) and [sceVrTracker2ResetLocalCoordinate](../../../core/libs/prx/libSceVrTracker2/Export.cpp) (libSceVrTracker2) diff --git a/docs/dev/debt/libScePad.md b/docs/dev/debt/libScePad.md index 573dc9d373..291c86d4e2 100644 --- a/docs/dev/debt/libScePad.md +++ b/docs/dev/debt/libScePad.md @@ -11,3 +11,4 @@ - [scePadSetAngularVelocityBiasCorrectionState](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) - inferred signature `(int handle, bool enabled)` follows `scePadSetTiltCorrectionState` and `scePadSetAngularVelocityDeadbandState`; an invalid handle returns `PAD_ERROR_INVALID_HANDLE`, disabling is a no-op, and enabling throws because bias correction is not modelled - [scePadUnknown_zBsgF0q8DIM](../../../core/libs/prx/libScePad/Export.cpp) (libScePad, NID `zBsgF0q8DIM`) - PPSA03671 imports it and fails to load without it. The aerolib list has no name for the NID, and none of the pad names in SharpProspero, shadPS4 or OpenOrbis, nor the trigger effect, motion sensor, VR controller, HD and haptic names tried, hashes to it. It is declared without parameters and throws. The title's executable has an import stub for it but no call, jump or address reference to that stub, so the argument count and whether the result is used are unknown - [scePadUnknown_n3kSX62fgNo](../../../core/libs/prx/libScePad/Export.cpp) (libScePad, NID `n3kSX62fgNo`) - PPSA01530 and PPSA13356 import it and fail to load without it. The real name is unknown; it is declared without parameters and throws +- PPSA07411 PS VR2 imports declared without parameters, signatures unknown, added without an implementation because the title fails to load without them: [scePadVrControllerSetVibration, scePadVrControllerSetVibrationMode and scePadVrControllerReadCombinedState](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) and [sceVrTracker2ResetLocalCoordinate](../../../core/libs/prx/libSceVrTracker2/Export.cpp) (libSceVrTracker2) From bea4be8f9510450a3a164d7f348b052952ed8d6f Mon Sep 17 00:00:00 2001 From: thibaut <thibaut-leaux-angeli@live.fr> Date: Sun, 18 Oct 2026 01:27:59 +0200 Subject: [PATCH 081/103] perf(agc): let the resource cache bound come back down Halve the grown bound (not below 1024 entries) once a window of the bound's uses times a shrink window was served by at most a quarter of it, and reset the bound on Clear(). The shrink window starts at 8 and doubles (up to 1024) each time growth undoes the last shrink, so a set of keys that comes back over a longer span stops the bound from oscillating. The entries over the bound leave at the next insert, with the other evictions. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XBXM4pgkMHYEtqN1EQni2P --- .../Graphics/include/ShaderResources.hpp | 8 +++++ .../Graphics/src/ShaderResources.cpp | 29 ++++++++++++++++ .../tests/execution/ResourceCacheGrowth.cpp | 34 +++++++++++++++++-- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 30e1012671..ef70d27744 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -18,6 +18,7 @@ #include <mutex> #include <string> #include <unordered_map> +#include <unordered_set> #include <vector> namespace AgcDriver { class VulkanDevice; } @@ -640,6 +641,7 @@ private: void noteMiss(const Key& key); void noteEvicted(const Key& key); void noteReturn(const Key& key); + void noteUse(const Key& key); struct KeyHash { std::size_t operator()(const Key& key) const noexcept { std::uint64_t hash = 14695981039346656037ull; @@ -656,6 +658,12 @@ private: std::unordered_map<std::size_t, decltype(evictedOrder)::iterator> evictedKeys; std::size_t evictionsSinceCheck = 0; std::size_t returnsSinceCheck = 0; + std::unordered_set<std::size_t> usedKeys; + std::size_t usesSinceCheck = 0; + static constexpr std::size_t InitialShrinkWindow = 8; + static constexpr std::size_t MaximumShrinkWindow = 1024; + std::size_t shrinkWindow = InitialShrinkWindow; + bool shrankLast = false; }; // The process-wide cache the device and the draw path share (keys name the device; a device clears diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index fc09e2f367..907062d951 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -3150,6 +3150,7 @@ std::shared_ptr<ShaderResources> ResourceCache::Find(const Key& key) { return nullptr; } entries.splice(entries.begin(), entries, found->second); + noteUse(key); return found->second->second; } @@ -3170,9 +3171,30 @@ void ResourceCache::noteEvicted(const Key& key) { if (!churning || capacity >= MaximumEntries) return; const auto previous = capacity; capacity = std::min(capacity * 2u, MaximumEntries); + if (shrankLast) shrinkWindow = std::min(shrinkWindow * 2u, MaximumShrinkWindow); + shrankLast = false; + usedKeys.clear(); + usesSinceCheck = 0; std::fprintf(stderr, "[rescache] capacity %zu -> %zu entries: keys it evicted keep coming back\n", previous, capacity); } +void ResourceCache::noteUse(const Key& key) { + if (fixedCapacity || capacity <= InitialEntries) return; + usedKeys.insert(KeyHash{}(key)); + if (++usesSinceCheck < capacity * shrinkWindow) return; + const auto used = usedKeys.size(); + const auto uses = usesSinceCheck; + usedKeys.clear(); + usesSinceCheck = 0; + if (used * 4u > capacity) return; + const auto previous = capacity; + capacity = std::max(capacity / 2u, InitialEntries); + shrankLast = true; + evictionsSinceCheck = 0; + returnsSinceCheck = 0; + std::fprintf(stderr, "[rescache] capacity %zu -> %zu entries: %zu keys served the last %zu uses\n", previous, capacity, used, uses); +} + void ResourceCache::noteReturn(const Key& key) { if (fixedCapacity) return; const auto found = evictedKeys.find(KeyHash{}(key)); @@ -3315,6 +3337,7 @@ void ResourceCache::Insert(const Key& key, std::shared_ptr<ShaderResources> reso fixedCapacity = parsed != 0; capacity = fixedCapacity ? static_cast<std::size_t>(parsed) : InitialEntries; } + noteUse(key); while (entries.size() > capacity) { noteEvicted(entries.back().first); if (evicted != nullptr) evicted->push_back(std::move(entries.back().second)); @@ -3365,6 +3388,7 @@ bool ResourceCache::Touch(const Key& key) { const auto found = index.find(key); if (found == index.end()) return false; entries.splice(entries.begin(), entries, found->second); + noteUse(key); return true; } @@ -3384,6 +3408,11 @@ void ResourceCache::Clear() { evictedOrder.clear(); evictionsSinceCheck = 0; returnsSinceCheck = 0; + usedKeys.clear(); + usesSinceCheck = 0; + shrinkWindow = InitialShrinkWindow; + shrankLast = false; + if (!fixedCapacity) capacity = 0; } std::size_t ResourceCache::Size() const { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp index fbb0ccec87..bae94f9150 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp @@ -31,6 +31,8 @@ using AgcDriver::Graphics::Require; constexpr std::uint32_t Dispatches = 1536; constexpr std::uint32_t Passes = 4; +constexpr std::uint32_t HotDispatches = 64; +constexpr std::uint32_t HotRepeats = 640; constexpr std::size_t SrtBytes = 64; constexpr std::size_t OutputBytes = 256; constexpr std::size_t SrtOffset = 4096; @@ -88,10 +90,11 @@ void Prepare(const GuestBlock& block) { std::memset(block.Bytes(OutputOffset), 0, static_cast<std::size_t>(Dispatches) * OutputBytes); } -void RunPass(const GuestBlock& block) { +void RunPass(const GuestBlock& block, std::uint32_t count = Dispatches, std::uint32_t repeats = 1) { const auto kernel = block.At(0); std::vector<std::uint32_t> words; - for (std::uint32_t dispatch = 0; dispatch < Dispatches; ++dispatch) { + for (std::uint32_t use = 0; use < count * repeats; ++use) { + const auto dispatch = use % count; const auto srt = block.At(SrtOffset + static_cast<std::size_t>(dispatch) * SrtBytes); for (const auto& packet : {Pm4(0x76, {0x20c, static_cast<std::uint32_t>(kernel >> 8u)}), Pm4(0x76, {0x20d, static_cast<std::uint32_t>(kernel >> 40u) & 0xffu}), Pm4(0x76, {0x207, 64}), Pm4(0x76, {0x208, 1}), Pm4(0x76, {0x209, 1}), Pm4(0x76, {0x213, 2u << 1u}), Pm4(0x76, {0x240, static_cast<std::uint32_t>(srt)}), Pm4(0x76, {0x241, static_cast<std::uint32_t>(srt >> 32u)}), Pm4(0x15, {1, 1, 1, 0x41})}) { words.insert(words.end(), packet.begin(), packet.end()); @@ -117,6 +120,32 @@ void WorkingSetLargerThanTheInitialBound(const GuestBlock& block) { Require(kept >= Dispatches, "resource cache growth: " + std::to_string(Dispatches) + " dispatches with distinct resources repeated " + std::to_string(Passes) + " times in turn, but the resource cache kept only " + std::to_string(kept) + " entries (initial bound " + std::to_string(InitialEntries) + "), so every pass rebuilt every set"); } +void BoundComesBackDown(const GuestBlock& block) { + auto& cache = AgcDriver::Graphics::SharedResourceCache(); + const auto grown = cache.Capacity(); + Require(grown > InitialEntries, "resource cache growth: the bound stayed at " + std::to_string(grown) + " entries after the large working set"); + RunPass(block, HotDispatches, HotRepeats); + const auto shrunk = cache.Capacity(); + Require(shrunk < grown, "resource cache growth: " + std::to_string(HotDispatches) + " sets served " + std::to_string(HotDispatches * HotRepeats) + " dispatches, but the bound stayed at " + std::to_string(shrunk) + " entries"); + + const std::uint32_t records = OutputBytes / 2; + std::memcpy(block.Bytes(SrtOffset + 8), &records, sizeof(records)); + RunPass(block, 1); + Require(cache.Size() <= shrunk, "resource cache growth: the bound came down to " + std::to_string(shrunk) + " entries, but the next insert left " + std::to_string(cache.Size())); + + for (std::uint32_t pass = 0; pass < Passes; ++pass) RunPass(block); + const auto regrown = cache.Capacity(); + Require(regrown > shrunk, "resource cache growth: the bound did not grow again when the large working set came back"); + RunPass(block, HotDispatches, HotRepeats); + Require(cache.Capacity() == regrown, "resource cache growth: the bound came down again after the same hot pass, although the last shrink had been undone"); + + cache.Clear(); + Require(cache.Capacity() == InitialEntries, "resource cache growth: Clear() left the bound at " + std::to_string(cache.Capacity()) + " entries instead of " + std::to_string(InitialEntries)); + Prepare(block); + for (std::uint32_t pass = 0; pass < Passes; ++pass) RunPass(block); + Require(cache.Capacity() > InitialEntries, "resource cache growth: after Clear() the large working set did not grow the bound again"); +} + } int main() { @@ -128,6 +157,7 @@ int main() { GuestBlock block; Prepare(block); WorkingSetLargerThanTheInitialBound(block); + BoundComesBackDown(block); AgcDriverShutdown_nid_postfix(); std::puts("resource cache growth tests passed"); return 0; From 316ad40b46167e726c6c3619e5bfae3c6d426ce0 Mon Sep 17 00:00:00 2001 From: Kaua <kaua02249@gmail.com> Date: Sat, 17 Oct 2026 21:06:51 -0300 Subject: [PATCH 082/103] fix(tools): write check_compile output as UTF-8 --- tools/check_compile.py | 3 +++ tools/tests/test_check_compile.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/check_compile.py b/tools/check_compile.py index cd8761de32..47076b046b 100644 --- a/tools/check_compile.py +++ b/tools/check_compile.py @@ -155,6 +155,9 @@ def check(root, build, files): def main(argv=None): + for stream in (sys.stdout, sys.stderr): + if hasattr(stream, "reconfigure"): + stream.reconfigure(encoding="utf-8", errors="replace") parser = argparse.ArgumentParser( description="compile the changed translation units only, to check a change builds without a full build") parser.add_argument("--build", type=Path, default=Path("build"), diff --git a/tools/tests/test_check_compile.py b/tools/tests/test_check_compile.py index 975094663a..3b83764f0a 100644 --- a/tools/tests/test_check_compile.py +++ b/tools/tests/test_check_compile.py @@ -61,7 +61,7 @@ class CheckCompileTests(unittest.TestCase): environment = dict(os.environ) environment.update(env or {}) return subprocess.run([sys.executable, "-I", str(TOOL), *arguments], cwd=self.root, - capture_output=True, text=True, env=environment) + capture_output=True, text=True, encoding="utf-8", env=environment) def source(self, relative, text="int x;\n"): path = self.root / relative From 051a61079830beb1784b131ce31a1da6435ef941 Mon Sep 17 00:00:00 2001 From: fma-rc <188743521+fma-rc@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:31:51 +0200 Subject: [PATCH 083/103] feat(libSceJobManager): implement compute sequences and compute dispatches --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 2 + .../tests/execution/JobManagerCompute.cpp | 242 ++++++++++++++++ core/libs/prx/libSceJobManager/CMakeLists.txt | 3 +- core/libs/prx/libSceJobManager/Export.cpp | 261 +++++++++++++++--- docs/dev/debt/libSceJobManager.md | 2 +- 5 files changed, 474 insertions(+), 36 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index fcd75d52e0..49d614a22a 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1564,6 +1564,8 @@ target_include_directories(agc_driver_swap_alt_blend_tests PRIVATE $<TARGET_PROP target_include_directories(agc_driver_packed_float_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>) +target_link_libraries(agc_driver_job_manager_compute_tests PRIVATE libSceJobManager libSceAgc libkernel) + target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>) target_include_directories(agc_driver_color1555_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp new file mode 100644 index 0000000000..79af7f5d54 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp @@ -0,0 +1,242 @@ +#include "VulkanTestDevice.hpp" +#include "SceTypes.hpp" +#include <array> +#include <atomic> +#include <chrono> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <iostream> +#include <new> +#include <stdexcept> +#include <string> +#include <thread> + +extern "C" void AgcDriverShutdown_nid_postfix(); + +struct MemorySizeQueryParams { + std::uint32_t version; + std::uint32_t workers; + std::uint32_t sequences; + std::uint32_t invocations; + std::uint32_t dispatches; + std::uint32_t syncPoints; + std::uint32_t waits; + std::uint32_t computeSequences; + std::uint64_t computeQueueMask; +}; + +struct ConfigParams { + std::uint32_t version; + std::uint32_t threadFlags; + std::uint32_t stackSize; + std::uint32_t priority; + const char* threadName; + std::uint32_t spinMicroseconds; + std::uint32_t computeAffinity; + std::uint32_t computePriority; + std::uint32_t reserved; +}; + +using JobEntry = void (APS5_VABI*)(void*, std::uint32_t); + +struct Sequence { + APS5_VABI virtual int Dispatch(JobEntry, void*, std::uint32_t, const char*) = 0; + APS5_VABI virtual int SyncPoint() = 0; + APS5_VABI virtual int Wait() = 0; + APS5_VABI virtual void CompleteDestructor() = 0; + APS5_VABI virtual void DeletingDestructor() = 0; + APS5_VABI virtual int DispatchCompute(const void*, void*, std::int32_t, std::int32_t, std::int32_t, const char*) = 0; + APS5_VABI virtual int DispatchComputeIndirect(const void*, void*, const void*, const char*) = 0; +}; + +struct SequenceFactory { + APS5_VABI virtual int Create(Sequence**, std::uint32_t, std::uint32_t, std::uint32_t, std::uint32_t, const char*) = 0; + APS5_VABI virtual int Destroy(Sequence*) = 0; + APS5_VABI virtual void CompleteDestructor() = 0; + APS5_VABI virtual void DeletingDestructor() = 0; + APS5_VABI virtual int CreateCompute(Sequence**, std::uint32_t, std::uint32_t, std::uint32_t, std::uint32_t, const char*) = 0; + APS5_VABI virtual int DestroyCompute(Sequence*) = 0; +}; + +extern "C" { +void APS5_VABI _ZN3sce3Job10JobManagerC1Ev(void*); +std::uint64_t APS5_VABI _ZN3sce3Job10JobManager27calculateRequiredMemorySizeEPKNS1_21MemorySizeQueryParamsE(const MemorySizeQueryParams*); +int APS5_VABI _ZN3sce3Job10JobManager10initializeEPvmjjjmPKNS1_12ConfigParamsE(void*, void*, std::uint64_t, std::uint32_t, std::uint32_t, std::uint32_t, std::uint64_t, const ConfigParams*); +int APS5_VABI _ZN3sce3Job10JobManager8shutdownEv(void*); +SequenceFactory* APS5_VABI _ZN3sce3Job10JobManager34getUnifiedSequenceFactoryInterfaceENS1_8PriorityE(void*, std::uint32_t); +} + +#define Construct _ZN3sce3Job10JobManagerC1Ev +#define RequiredSize _ZN3sce3Job10JobManager27calculateRequiredMemorySizeEPKNS1_21MemorySizeQueryParamsE +#define Initialize _ZN3sce3Job10JobManager10initializeEPvmjjjmPKNS1_12ConfigParamsE +#define Shutdown _ZN3sce3Job10JobManager8shutdownEv +#define Factories _ZN3sce3Job10JobManager34getUnifiedSequenceFactoryInterfaceENS1_8PriorityE + +namespace { + +constexpr int InvalidArgument = static_cast<int>(0x81581000u); +constexpr int OutOfRange = static_cast<int>(0x81580000u); +constexpr int NullArgument = static_cast<int>(0x81580001u); +constexpr int NoNodes = static_cast<int>(0x81580002u); +constexpr int LimitExceeded = static_cast<int>(0x81580003u); +constexpr int NoneFree = static_cast<int>(0x81580006u); +constexpr int Invalid = static_cast<int>(0x81580007u); + +void Require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +alignas(256) std::array<std::uint32_t, 12> ReaderCode{0xf4040100u, 0xfa000000u, 0xf4080201u, 0xfa000000u, 0xbf8cc07fu, 0xf4000182u, 0xfa000000u, 0xbf8cc07fu, 0x7e000206u, 0xe0700000u, 0x80020000u, 0xbf810000u}; +alignas(256) std::array<std::uint32_t, 4> Value{}; +alignas(256) std::array<std::uint32_t, 2> Table{}; +alignas(256) volatile std::uint32_t Output[4]{}; +alignas(256) std::array<std::uint32_t, 4> OutputDescriptor{}; +alignas(32) std::array<std::uint32_t, 8> IndirectArguments{1, 1, 1, 0, 0, 0, 0, 0}; +alignas(16) std::array<std::uint32_t, 8> Specials{}; +alignas(16) std::array<std::uint32_t, 8> SpecialsWithoutDispatch{}; +std::array<ShaderRegister, 9> Registers{}; +alignas(16) std::array<unsigned char, 0x80> Shader{}; +alignas(16) std::array<unsigned char, 0x80> ShaderWithoutDispatch{}; + +std::uint32_t Low(const volatile void* pointer) { + return static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(pointer)); +} + +std::uint32_t High(const volatile void* pointer) { + return static_cast<std::uint32_t>(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(pointer)) >> 32u); +} + +void BuildShader() { + Table = {Low(Value.data()), High(Value.data())}; + OutputDescriptor = {Low(Output), High(Output) & 0xffffu, sizeof(Output), 0x31016facu}; + const auto code = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(ReaderCode.data())); + Registers = {{{0x20c, static_cast<std::uint32_t>(code >> 8u)}, {0x20d, static_cast<std::uint32_t>(code >> 40u) & 0xffu}, {0x207, 1}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 4u << 1u}, + {0x242, Low(OutputDescriptor.data())}, {0x243, High(OutputDescriptor.data())}}}; + Specials[4] = 1; + const ShaderRegister* registers = Registers.data(); + for (auto* shader : {&Shader, &ShaderWithoutDispatch}) { + const std::uint32_t* specials = shader == &Shader ? Specials.data() : SpecialsWithoutDispatch.data(); + std::memcpy(shader->data() + 0x20, ®isters, sizeof(registers)); + std::memcpy(shader->data() + 0x28, &specials, sizeof(specials)); + (*shader)[0x5c] = static_cast<unsigned char>(Registers.size()); + } +} + +std::uint64_t SizeFor(std::uint32_t computeSequences, std::uint64_t mask, bool limits) { + const MemorySizeQueryParams query{1, 2, 1, limits ? 100u : 0u, limits ? 8u : 0u, limits ? 4u : 0u, limits ? 2u : 0u, computeSequences, mask}; + return RequiredSize(&query); +} + +std::atomic<bool> Released{false}; +std::atomic<std::uint32_t> SeenByJob{0}; + +void APS5_VABI BlockingJob(void*, std::uint32_t) { + while (!Released) std::this_thread::yield(); +} + +void APS5_VABI ReadOutputJob(void*, std::uint32_t) { + SeenByJob = Output[0]; +} + +void APS5_VABI Nothing(void*, std::uint32_t) { +} + +void SizeCase() { + Require(SizeFor(0, 0, false) == 0x10d0 && SizeFor(0, 0, true) == 0x14c0, "size without compute sequences"); + Require(SizeFor(1, 1, false) == 0x24fd8 && SizeFor(1, 1, true) == 0x253c8, "size with one compute sequence"); + Require(SizeFor(2, 3, false) == 0x48e20 && SizeFor(2, 3, true) == 0x49210, "size with two compute sequences"); + Require(SizeFor(3, 7, false) == 0x6cb68, "size with three compute sequences"); + Require(SizeFor(1, 3, true) == 0x491c8, "size with two queues for one compute sequence"); + Require(SizeFor(2, 1, true) == 0, "size with fewer queues than compute sequences"); +} + +void RunCase() { + const std::uint64_t size = SizeFor(1, 1, true); + void* memory = ::operator new((size + 255) & ~std::uint64_t{255}, std::align_val_t{256}); + Require(memory != nullptr, "allocation failed"); + alignas(64) static unsigned char manager[0x100]; + Construct(manager); + const ConfigParams badHandler{1, 0x00021fff, 0x00010000, 0x2bc, nullptr, 0, 0x1fff, 0x50, 0}; + Require(Initialize(manager, memory, size, 2, 1, 1, 1, &badHandler) == InvalidArgument, "a handler priority below 0x100 was accepted"); + Require(Initialize(manager, memory, size, 2, 1, 1, 1, nullptr) == 0, "initialize with one compute sequence failed"); + SequenceFactory* factory = Factories(manager, 0); + Sequence* sequence = nullptr; + Sequence* other = nullptr; + Require(factory->CreateCompute(&sequence, 100, 4, 4, 2, "compute") == 0, "the compute sequence was not created"); + Require(factory->CreateCompute(&other, 1, 1, 1, 1, "second") == NoneFree, "a second compute sequence was created"); + Require(factory->Create(&other, 1, 1, 1, 1, "cpu") == 0, "the CPU sequence was not created"); + Require(factory->DestroyCompute(other) == Invalid, "a CPU sequence was destroyed as a compute sequence"); + Require(factory->Destroy(sequence) == Invalid, "a compute sequence was destroyed as a CPU sequence"); + Require(factory->Destroy(other) == 0, "the CPU sequence was not destroyed"); + + Require(sequence->DispatchCompute(nullptr, Table.data(), 1, 1, 1, nullptr) == NullArgument, "a null shader was accepted"); + Require(sequence->DispatchCompute(Shader.data(), Table.data(), 0, 1, 1, nullptr) == OutOfRange, "a zero group count was accepted"); + Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, -1, nullptr) == OutOfRange, "a negative group count was accepted"); + Require(sequence->DispatchComputeIndirect(Shader.data(), Table.data(), nullptr, nullptr) == NullArgument, "null indirect arguments were accepted"); + Require(sequence->DispatchComputeIndirect(Shader.data(), Table.data(), reinterpret_cast<const char*>(IndirectArguments.data()) + 4, nullptr) == NullArgument, "misaligned indirect arguments were accepted"); + + Value[0] = 0x11110001u; + Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, "direct") == 0, "the compute dispatch failed"); + Require(sequence->Wait() == 0, "the wait after a compute dispatch failed"); + Require(Output[0] == 0x11110001u, "the compute dispatch had not run when the wait returned: " + std::to_string(Output[0])); + + Value[0] = 0x22220002u; + Require(sequence->DispatchComputeIndirect(Shader.data(), Table.data(), IndirectArguments.data(), nullptr) == 0, "the indirect compute dispatch failed"); + Require(sequence->Wait() == 0, "the wait after an indirect compute dispatch failed"); + Require(Output[0] == 0x22220002u, "the indirect compute dispatch had not run when the wait returned: " + std::to_string(Output[0])); + + Value[0] = 0x33330003u; + Require(sequence->DispatchCompute(ShaderWithoutDispatch.data(), Table.data(), 1, 1, 1, nullptr) == 0, "the dispatch of a shader without the dispatch bit failed"); + Require(sequence->Wait() == 0, "the wait after a shader without the dispatch bit failed"); + Require(Output[0] == 0x22220002u, "a shader without the dispatch bit ran"); + + Require(sequence->Dispatch(BlockingJob, nullptr, 1, "blocking") == 0, "the blocking job was not dispatched"); + Require(sequence->SyncPoint() == 0, "the sync point failed"); + Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == 0, "the compute dispatch behind a sync point failed"); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + Require(Output[0] == 0x22220002u, "a compute dispatch ran before the CPU job ahead of its sync point finished"); + Released = true; + Require(sequence->SyncPoint() == 0, "the second sync point failed"); + Require(sequence->Dispatch(ReadOutputJob, nullptr, 1, "reader") == 0, "the reading job was not dispatched"); + Require(sequence->Wait() == 0, "the wait after the ordered dispatches failed"); + Require(Output[0] == 0x33330003u, "the compute dispatch behind a sync point did not run"); + Require(SeenByJob == 0x33330003u, "a CPU job behind a sync point ran before the compute dispatch ahead of it finished"); + + for (int index = 0; index < 4; ++index) Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == 0, "a compute dispatch within the limit failed"); + Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == LimitExceeded, "a compute dispatch past the dispatch limit was accepted"); + Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == Invalid, "a latched compute sequence accepted a dispatch"); + Require(factory->DestroyCompute(sequence) == 0, "the compute sequence was not destroyed"); + + Require(factory->Create(&other, 0x7fff, 0x3ff, 0xff, 1, "nodes") == 0, "the node counting sequence was not created"); + int accepted = 0; + int result = 0; + while (accepted < 0x3ff && (result = other->Dispatch(Nothing, nullptr, 1, nullptr)) == 0) ++accepted; + Require(result == NoNodes && accepted == 43, "the node pool of a compute configuration holds " + std::to_string(accepted) + " dispatches"); + Require(Shutdown(manager) == 0, "shutdown failed"); + ::operator delete(memory, std::align_val_t{256}); +} + +} + +int main() { + try { + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + device.reset(); + BuildShader(); + SizeCase(); + RunCase(); + AgcDriverShutdown_nid_postfix(); + std::cout << "job manager compute test passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { + AgcDriverShutdown_nid_postfix(); + } catch (const std::exception& shutdownError) { + std::cerr << shutdownError.what() << '\n'; + } + return 1; + } +} diff --git a/core/libs/prx/libSceJobManager/CMakeLists.txt b/core/libs/prx/libSceJobManager/CMakeLists.txt index c029f72ce6..8539bbd0c2 100644 --- a/core/libs/prx/libSceJobManager/CMakeLists.txt +++ b/core/libs/prx/libSceJobManager/CMakeLists.txt @@ -1,11 +1,12 @@ add_library(libSceJobManager SHARED EXCLUDE_FROM_ALL Export.cpp) target_include_directories(libSceJobManager PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(libSceJobManager PRIVATE libSceAgc libSceAgcDriver) configure_windows_unwind(libSceJobManager) if(BUILD_TESTING) add_test_executable(guest_job_manager_tests ${LIBS_INCLUDE_DIR}/tests/GuestJobManager.cpp) target_include_directories(guest_job_manager_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_job_manager_tests PRIVATE libSceJobManager libc) + target_link_libraries(guest_job_manager_tests PRIVATE libSceJobManager libSceAgc libSceAgcDriver libkernel libc) configure_windows_unwind(guest_job_manager_tests) foreach(testCase IN ITEMS memory validation dispatch ordering nested limits priority shutdown) add_test(NAME guest_job_manager_${testCase} COMMAND guest_job_manager_tests ${testCase}) diff --git a/core/libs/prx/libSceJobManager/Export.cpp b/core/libs/prx/libSceJobManager/Export.cpp index 94d53929a9..9ed6abee66 100644 --- a/core/libs/prx/libSceJobManager/Export.cpp +++ b/core/libs/prx/libSceJobManager/Export.cpp @@ -22,6 +22,24 @@ int APS5_VABI scePthreadAttrSetstacksize(PthreadAttr* attr, std::size_t stackSiz int APS5_VABI scePthreadAttrSetaffinity(PthreadAttr* attr, KernelCpumask mask); int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); int APS5_VABI scePthreadJoin(Pthread thread, void** result); +int APS5_VABI sceKernelCreateEqueue(KernelEqueue* eq, const char* name); +int APS5_VABI sceKernelDeleteEqueue(KernelEqueue eq); +int APS5_VABI sceKernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, int* out, const KernelUseconds* timo); +int APS5_VABI sceKernelAddUserEventEdge(KernelEqueue eq, int id); +int APS5_VABI sceKernelTriggerUserEvent(KernelEqueue eq, int id, void* udata); +int APS5_VABI sceKernelGetEventFilter(const KernelEvent* ev); +std::uintptr_t APS5_VABI sceKernelGetEventId(const KernelEvent* ev); +int APS5_VABI sceAgcDriverAddEqEvent(KernelEqueue eq, int id, void* udata); +int APS5_VABI sceAgcDriverDeleteEqEvent(KernelEqueue eq, int id); +int APS5_VABI sceAgcDriverSubmitAcb(std::uint32_t queue, const Packet* packet); +std::uint32_t* APS5_VABI sceAgcAcbAcquireMem(CommandBuffer* buf, std::uint32_t gcrControl, const volatile void* base, std::uint64_t sizeBytes, std::uint32_t pollCycles); +std::uint32_t* APS5_VABI sceAgcCbSetShRegistersDirect(CommandBuffer* buf, const volatile ShaderRegister* regs, std::uint32_t numRegs); +std::uint32_t* APS5_VABI sceAgcCbSetShRegisterRangeDirect(CommandBuffer* buf, std::uint32_t offset, const std::uint32_t* values, std::uint32_t numValues); +std::uint32_t* APS5_VABI sceAgcAcbPushMarker(CommandBuffer* buf, const char* str, std::uint32_t color); +std::uint32_t* APS5_VABI sceAgcAcbPopMarker(CommandBuffer* buf); +std::uint32_t* APS5_VABI sceAgcCbDispatch(CommandBuffer* buf, std::uint32_t threadGroupX, std::uint32_t threadGroupY, std::uint32_t threadGroupZ, std::uint32_t modifier); +std::uint32_t* APS5_VABI sceAgcAcbDispatchIndirect(CommandBuffer* buf, const volatile void* indirectArguments, std::uint32_t modifier); +std::uint32_t* APS5_VABI sceAgcCbReleaseMem(CommandBuffer* buf, std::uint8_t action, std::uint16_t gcrControl, std::uint8_t dst, std::uint8_t cachePolicy, const volatile Label* address, std::uint8_t dataSelect, std::uint64_t data, std::uint16_t gdsOffset, std::uint16_t gdsSize, std::uint8_t interrupt, std::uint32_t interruptContextId); } namespace { @@ -50,6 +68,11 @@ constexpr std::uint64_t StateEnd = 0x2d8; constexpr std::uint64_t ThreadRecordBytes = 0x48; constexpr std::uint64_t SequenceBytes = 0x240; constexpr std::uint64_t NodeBytes = 72; +constexpr std::uint64_t ComputeContextBytes = 0x168; +constexpr std::uint64_t ComputeCommandBytes = 0x3ff * 104 + 0xff * 148 + 0x3f * 32; +constexpr std::uint32_t FirstComputeQueue = 0x20; +constexpr std::uint32_t UserDataRegister = 0x240; +constexpr int UserEventFilter = -11; struct MemorySizeQueryParams { std::uint32_t version; @@ -94,7 +117,7 @@ std::uint64_t AlignUp(std::uint64_t value, std::uint64_t alignment) { struct Manager; class Sequence; -enum class ItemType { Dispatch, SyncPoint, Wait }; +enum class ItemType { Dispatch, SyncPoint, Wait, Compute, ComputeIndirect }; struct Item { ItemType type; @@ -105,6 +128,11 @@ struct Item { std::uint32_t handedOut; std::uint32_t completed; bool* passed; + const void* shader = nullptr; + std::array<std::uint32_t, 3> groups{}; + const void* indirectArguments = nullptr; + std::string name; + bool named = false; }; struct Counters { @@ -140,27 +168,26 @@ public: APS5_VABI int Dispatch(JobEntry entry, void* argument, std::uint32_t count, const char* name) override; APS5_VABI int SyncPoint() override; APS5_VABI int Wait() override; - APS5_VABI int DispatchCompute(const void*, void*, std::int32_t, std::int32_t, std::int32_t, const char*) override { - NotImplemented_nid_no_patch("sce::Job compute dispatch"); - return 0; - } - APS5_VABI int DispatchComputeIndirect(const void*, void*, const void*, const char*) override { - NotImplemented_nid_no_patch("sce::Job indirect compute dispatch"); - return 0; - } + APS5_VABI int DispatchCompute(const void* shader, void* userData, std::int32_t x, std::int32_t y, std::int32_t z, const char* name) override; + APS5_VABI int DispatchComputeIndirect(const void* shader, void* userData, const void* indirectArguments, const char* name) override; bool Open(Manager* owner, const Counters& limit, std::uint32_t runQueue); int WaitAndClose(std::uint32_t index); void Advance(); void DispatchCompleted(std::uint32_t invocations); + void CollectCompute(); Manager* manager = nullptr; bool open = false; + bool compute = false; + std::uint32_t queue = 0; private: int Latch(int code); void RequireOpen() const; int WaitLocked(std::unique_lock<std::mutex>& guard); + int AppendCompute(Item item); + void SubmitCompute(const Item& item); Counters limits; Counters submitted; @@ -172,6 +199,11 @@ private: std::deque<Item> items; std::mutex waiters; std::condition_variable passed; + Label label{}; + std::uint64_t labelSeen = 0; + std::uint64_t labelTotal = 0; + bool batchStarted = false; + std::deque<std::vector<std::uint32_t>> commands; }; class Factory final : public ISequenceFactory { @@ -196,6 +228,11 @@ struct Manager { std::uint32_t persistentThreads = 0; std::array<std::deque<Item*>, 2> runQueues; std::vector<Sequence> sequences; + std::deque<Sequence> computeSequences; + std::uint32_t freeComputeSequences = 0; + std::vector<std::uint32_t> computeQueues; + KernelEqueue computeEvents = 0; + Pthread computeHandler = nullptr; std::array<Factory, 2> factories; std::array<Factory, 2> unifiedFactories; std::vector<Pthread> workers; @@ -231,6 +268,8 @@ bool Sequence::Open(Manager* owner, const Counters& limit, std::uint32_t runQueu outstanding = 0; cursor = 0; items.clear(); + commands.clear(); + batchStarted = false; if (!TakeNode(*owner)) return false; open = true; return true; @@ -250,6 +289,12 @@ void Sequence::Advance() { manager->work.notify_all(); continue; } + if (item.type == ItemType::Compute || item.type == ItemType::ComputeIndirect) { + ++outstanding; + ++cursor; + SubmitCompute(item); + continue; + } if (outstanding != 0) return; if (item.type == ItemType::SyncPoint) { ++completed.syncPoints; @@ -260,6 +305,8 @@ void Sequence::Advance() { const auto retired = static_cast<std::uint32_t>(cursor) + 1; items.erase(items.begin(), items.begin() + static_cast<std::ptrdiff_t>(retired)); cursor = 0; + commands.clear(); + batchStarted = false; if (error == 0) { submitted.invocations -= completed.invocations; submitted.dispatches -= completed.dispatches; @@ -296,6 +343,81 @@ int Sequence::Dispatch(JobEntry entry, void* argument, std::uint32_t count, cons return 0; } +void Sequence::SubmitCompute(const Item& item) { + const auto* bytes = static_cast<const std::uint8_t*>(item.shader); + const ShaderRegister* registers = nullptr; + const std::uint8_t* specials = nullptr; + std::memcpy(®isters, bytes + 0x20, sizeof(registers)); + std::memcpy(&specials, bytes + 0x28, sizeof(specials)); + const std::uint32_t registerCount = bytes[0x5c]; + std::uint32_t modifier = 0; + std::memcpy(&modifier, specials + 0x10, sizeof(modifier)); + const auto userData = reinterpret_cast<std::uint64_t>(item.argument); + const std::array<std::uint32_t, 2> userDataWords{static_cast<std::uint32_t>(userData), static_cast<std::uint32_t>(userData >> 32)}; + auto& words = commands.emplace_back(64 + 3 * registerCount + item.name.size() / 4); + CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0}; + bool encoded = sceAgcAcbAcquireMem(&buffer, 0x20380u | (batchStarted ? 0u : 1u), nullptr, 0, 0x190) != nullptr; + encoded = encoded && sceAgcCbSetShRegistersDirect(&buffer, registers, registerCount) != nullptr; + encoded = encoded && sceAgcCbSetShRegisterRangeDirect(&buffer, UserDataRegister, userDataWords.data(), 2) != nullptr; + if (item.named) encoded = encoded && sceAgcAcbPushMarker(&buffer, item.name.c_str(), 0) != nullptr; + if ((modifier & 1) != 0) { + if (item.type == ItemType::Compute) { + encoded = encoded && sceAgcCbDispatch(&buffer, item.groups[0], item.groups[1], item.groups[2], modifier) != nullptr; + } else { + encoded = encoded && sceAgcAcbDispatchIndirect(&buffer, item.indirectArguments, modifier) != nullptr; + } + } + if (item.named) encoded = encoded && sceAgcAcbPopMarker(&buffer) != nullptr; + encoded = encoded && sceAgcCbReleaseMem(&buffer, 0x2f, 0x620, 0, 0, &label, 1, ++labelTotal, 0, 0, 2, 0) != nullptr; + if (!encoded) throw std::runtime_error("libSceJobManager: a compute dispatch did not fit its command buffer"); + batchStarted = true; + const Packet packet{words.data(), static_cast<std::uint32_t>(buffer.cursor_up - words.data()), 0, {}}; + if (sceAgcDriverSubmitAcb(queue, &packet) != 0) throw std::runtime_error("libSceJobManager: sceAgcDriverSubmitAcb failed"); +} + +void Sequence::CollectCompute() { + const std::uint64_t value = label.value; + const auto finished = static_cast<std::uint32_t>(value - labelSeen); + labelSeen = value; + if (finished == 0) return; + completed.dispatches += finished; + outstanding -= finished; + if (outstanding == 0) Advance(); +} + +int Sequence::AppendCompute(Item item) { + const std::lock_guard guard(manager->lock); + RequireOpen(); + if (!compute) throw std::runtime_error("libSceJobManager: a compute dispatch was made on a CPU sequence"); + if (error != 0) return SequenceInvalid; + if (submitted.dispatches++ >= limits.dispatches) return Latch(SequenceLimitExceeded); + if (!TakeNode(*manager)) return Latch(SequenceNoNodes); + items.push_back(std::move(item)); + Advance(); + return 0; +} + +int Sequence::DispatchCompute(const void* shader, void* userData, std::int32_t x, std::int32_t y, std::int32_t z, const char* name) { + if (shader == nullptr) return SequenceNullArgument; + if (x <= 0 || y <= 0 || z <= 0) return SequenceOutOfRange; + Item item{ItemType::Compute, this, nullptr, userData, 0, 0, 0, nullptr}; + item.shader = shader; + item.groups = {static_cast<std::uint32_t>(x), static_cast<std::uint32_t>(y), static_cast<std::uint32_t>(z)}; + item.named = name != nullptr; + if (item.named) item.name = name; + return AppendCompute(std::move(item)); +} + +int Sequence::DispatchComputeIndirect(const void* shader, void* userData, const void* indirectArguments, const char* name) { + if (shader == nullptr || indirectArguments == nullptr || (reinterpret_cast<std::uint64_t>(indirectArguments) & 0x1f) != 0) return SequenceNullArgument; + Item item{ItemType::ComputeIndirect, this, nullptr, userData, 0, 0, 0, nullptr}; + item.shader = shader; + item.indirectArguments = indirectArguments; + item.named = name != nullptr; + if (item.named) item.name = name; + return AppendCompute(std::move(item)); +} + int Sequence::SyncPoint() { const std::lock_guard guard(manager->lock); RequireOpen(); @@ -318,10 +440,11 @@ int Sequence::WaitAndClose(std::uint32_t index) { const int result = WaitLocked(guard); ReturnNodes(*manager, static_cast<std::uint32_t>(items.size()) + 1); items.clear(); + commands.clear(); cursor = 0; outstanding = 0; open = false; - manager->freeSequences |= std::uint32_t{1} << index; + (compute ? manager->freeComputeSequences : manager->freeSequences) |= std::uint32_t{1} << index; return result; } @@ -353,42 +476,63 @@ bool ValidLimits(std::uint32_t invocations, std::uint32_t dispatches, std::uint3 return invocations != 0 || dispatches != 0 || syncPoints != 0 || waits != 0; } -int Factory::Create(ISequence** out, std::uint32_t invocations, std::uint32_t dispatches, std::uint32_t syncPoints, std::uint32_t waits, const char*) { +template <typename TSequences> +int CreateIn(Manager* manager, TSequences& sequences, std::uint32_t& free, std::uint32_t priority, ISequence** out, const Counters& limits) { if (out == nullptr) return SequenceNullArgument; - if (!ValidLimits(invocations, dispatches, syncPoints, waits)) return SequenceOutOfRange; + if (!ValidLimits(limits.invocations, limits.dispatches, limits.syncPoints, limits.waits)) return SequenceOutOfRange; const std::lock_guard guard(manager->lock); - if (manager->freeSequences == 0) return SequenceNoneFree; - const int index = std::countr_zero(manager->freeSequences); - manager->freeSequences &= ~(std::uint32_t{1} << index); - Sequence& sequence = manager->sequences[static_cast<std::size_t>(index)]; - if (!sequence.Open(manager, {invocations, dispatches, syncPoints, waits}, priority)) return SequenceNoNodes; + if (free == 0) return SequenceNoneFree; + const int index = std::countr_zero(free); + free &= ~(std::uint32_t{1} << index); + Sequence& sequence = sequences[static_cast<std::size_t>(index)]; + if (!sequence.Open(manager, limits, priority)) return SequenceNoNodes; *out = &sequence; return 0; } -int Factory::Destroy(ISequence* sequence) { +template <typename TSequences> +int DestroyIn(Manager* manager, TSequences& sequences, const std::uint32_t& free, ISequence* sequence) { Sequence* found = nullptr; std::uint32_t index = 0; { const std::lock_guard guard(manager->lock); - for (; index < manager->sequences.size(); ++index) { - if (&manager->sequences[index] == sequence) break; + for (; index < sequences.size(); ++index) { + if (&sequences[index] == sequence) break; } - if (index == manager->sequences.size() || (manager->freeSequences & (std::uint32_t{1} << index)) != 0) return SequenceInvalid; - found = &manager->sequences[index]; + if (index == sequences.size() || (free & (std::uint32_t{1} << index)) != 0) return SequenceInvalid; + found = &sequences[index]; } static_cast<void>(found->WaitAndClose(index)); return 0; } -int Factory::CreateCompute(ISequence** out, std::uint32_t invocations, std::uint32_t dispatches, std::uint32_t syncPoints, std::uint32_t waits, const char*) { - if (out == nullptr) return SequenceNullArgument; - if (!ValidLimits(invocations, dispatches, syncPoints, waits)) return SequenceOutOfRange; - return SequenceNoneFree; +int Factory::Create(ISequence** out, std::uint32_t invocations, std::uint32_t dispatches, std::uint32_t syncPoints, std::uint32_t waits, const char*) { + return CreateIn(manager, manager->sequences, manager->freeSequences, priority, out, {invocations, dispatches, syncPoints, waits}); } -int Factory::DestroyCompute(ISequence*) { - return SequenceInvalid; +int Factory::Destroy(ISequence* sequence) { + return DestroyIn(manager, manager->sequences, manager->freeSequences, sequence); +} + +int Factory::CreateCompute(ISequence** out, std::uint32_t invocations, std::uint32_t dispatches, std::uint32_t syncPoints, std::uint32_t waits, const char*) { + return CreateIn(manager, manager->computeSequences, manager->freeComputeSequences, priority, out, {invocations, dispatches, syncPoints, waits}); +} + +int Factory::DestroyCompute(ISequence* sequence) { + return DestroyIn(manager, manager->computeSequences, manager->freeComputeSequences, sequence); +} + +void* APS5_VABI ComputeHandlerEntry(void* argument) { + Manager& manager = *static_cast<Manager*>(argument); + for (;;) { + KernelEvent event{}; + int count = 0; + static_cast<void>(sceKernelWaitEqueue(manager.computeEvents, &event, 1, &count, nullptr)); + if (count <= 0) continue; + if (sceKernelGetEventFilter(&event) == UserEventFilter && sceKernelGetEventId(&event) == 0) return nullptr; + const std::lock_guard guard(manager.lock); + for (Sequence& sequence : manager.computeSequences) sequence.CollectCompute(); + } } bool RunOne(Manager& manager, std::unique_lock<std::mutex>& guard) { @@ -421,11 +565,16 @@ void* APS5_VABI WorkerEntry(void* argument) { return nullptr; } -std::uint32_t NodeCount(std::uint64_t memory, std::uint64_t size, std::uint32_t workers, std::uint32_t sequences, bool* fits) { +std::uint32_t NodeCount(std::uint64_t memory, std::uint64_t size, std::uint32_t workers, std::uint32_t sequences, std::uint32_t computeQueues, bool* fits) { const std::uint64_t end = memory + size; std::uint64_t address = AlignUp(memory + StateEnd, 8) + workers * ThreadRecordBytes; for (const std::uint64_t bytes : {std::uint64_t{0x100}, std::uint64_t{0x80}, std::uint64_t{0x80}, std::uint64_t{0x80}}) address = AlignUp(address, 64) + bytes; - address = AlignUp(AlignUp(address, 64) + sequences * SequenceBytes, 64); + address = AlignUp(address, 64) + (sequences + computeQueues) * SequenceBytes; + if (computeQueues != 0) { + address = AlignUp(address + ComputeContextBytes, 0x100) + AlignUp(ComputeCommandBytes * computeQueues, 8); + address += AlignUp(4 * std::uint64_t{computeQueues}, 8) + 8 * std::uint64_t{computeQueues}; + } + address = AlignUp(address, 64); *fits = address <= end; return *fits ? static_cast<std::uint32_t>(((end - address) / NodeBytes) & 0xffff) : 0; } @@ -462,6 +611,30 @@ void StartWorkers(Manager& manager, const ConfigParams& config, std::uint32_t wo } } +void StartCompute(Manager& manager, std::uint32_t computeSequences, std::uint64_t computeQueueMask, std::uint32_t priority, std::uint32_t affinity) { + if (sceKernelCreateEqueue(&manager.computeEvents, "SceJobManagerComputeEventQueue") != 0) throw std::runtime_error("libSceJobManager: sceKernelCreateEqueue failed"); + for (std::uint64_t mask = computeQueueMask; mask != 0; mask &= mask - 1) { + const std::uint32_t queue = FirstComputeQueue + static_cast<std::uint32_t>(std::countr_zero(mask)); + if (sceAgcDriverAddEqEvent(manager.computeEvents, static_cast<int>(queue), nullptr) != 0) throw std::runtime_error("libSceJobManager: sceAgcDriverAddEqEvent failed"); + manager.computeQueues.push_back(queue); + } + manager.computeSequences.resize(computeSequences); + for (std::uint32_t index = 0; index < computeSequences; ++index) { + manager.computeSequences[index].compute = true; + manager.computeSequences[index].queue = manager.computeQueues[index]; + } + manager.freeComputeSequences = computeSequences == MaxSequences ? ~std::uint32_t{0} : (std::uint32_t{1} << computeSequences) - 1; + const KernelSchedParam schedule{static_cast<int>(priority)}; + PthreadAttr attributes = nullptr; + if (scePthreadAttrInit(&attributes) != 0 || scePthreadAttrSetinheritsched(&attributes, 0) != 0 || scePthreadAttrSetschedparam(&attributes, &schedule) != 0 || + scePthreadAttrSetstacksize(&attributes, 0x4000) != 0 || scePthreadAttrSetaffinity(&attributes, affinity) != 0) { + throw std::runtime_error("libSceJobManager: a compute handler thread attribute was rejected"); + } + const int created = scePthreadCreate(&manager.computeHandler, &attributes, ComputeHandlerEntry, &manager, "SceJobManagerComputeEventHdlr"); + scePthreadAttrDestroy(&attributes); + if (created != 0) throw std::runtime_error("libSceJobManager: scePthreadCreate failed for the compute handler"); +} + int Initialize(void* self, void* memory, std::uint64_t size, std::uint32_t workers, std::uint32_t sequences, std::uint32_t computeSequences, std::uint64_t computeQueueMask, const ConfigParams* config, bool unified) { if (Implementation(self) != nullptr) return ManagerAlreadyInitialized; if (memory == nullptr) return ManagerNullMemory; @@ -479,12 +652,16 @@ int Initialize(void* self, void* memory, std::uint64_t size, std::uint32_t worke const BlockHeader header{0x4d4a, static_cast<std::uint16_t>(StateOffset), 0x05000001, static_cast<std::uint32_t>(size), 0}; std::memcpy(memory, &header, sizeof(header)); if (config != nullptr && config->version > 1) return ManagerInvalidArgument; - if (computeSequences != 0) NotImplemented_nid_no_patch("sce::Job::JobManager::initialize with compute sequences"); const ConfigParams& settings = config != nullptr ? *config : DefaultConfig; + const auto computeQueues = computeSequences != 0 ? static_cast<std::uint32_t>(std::popcount(computeQueueMask)) : 0; bool fits = false; - const std::uint32_t nodes = NodeCount(base, size, workers, sequences, &fits); + const std::uint32_t nodes = NodeCount(base, size, workers, sequences, computeQueues, &fits); if (!fits || nodes == 0) return ManagerMemoryTooSmall; if (const int invalid = ValidateWorkers(settings, workers); invalid != 0) return invalid; + const bool ownHandlerSettings = (settings.version & 1) != 0; + const std::uint32_t handlerPriority = ownHandlerSettings ? settings.computePriority : 0x100; + const std::uint32_t handlerAffinity = ownHandlerSettings ? settings.computeAffinity : 0x1fff; + if (computeSequences != 0 && (handlerPriority < 0x100 || handlerPriority > 0x2ff || handlerAffinity == 0 || handlerAffinity > 0x1fff)) return ManagerInvalidArgument; auto* manager = new Manager; manager->running = true; manager->freeNodes = nodes - 1; @@ -496,6 +673,7 @@ int Initialize(void* self, void* memory, std::uint64_t size, std::uint32_t worke manager->unifiedFactories[priority].manager = manager; manager->unifiedFactories[priority].priority = priority; } + if (computeSequences != 0) StartCompute(*manager, computeSequences, computeQueueMask, handlerPriority, handlerAffinity); Implementation(self) = manager; StartWorkers(*manager, settings, workers); return 0; @@ -539,12 +717,19 @@ std::uint64_t APS5_VABI _ZN3sce3Job10JobManager27calculateRequiredMemorySizeEPKN } const std::uint64_t total = params->sequences + computeSequences; if (params->invocations > total * MaxInvocations || params->dispatches > total * MaxDispatches || params->syncPoints > total * MaxSyncPoints || params->waits > total * MaxWaits) return 0; - if (computeSequences != 0) NotImplemented_nid_no_patch("sce::Job::JobManager::calculateRequiredMemorySize with compute sequences"); std::uint64_t bytes = (NodeBytes * params->workers + 0x317) & 0xfc0; bytes = (bytes + 0x13f) & ~std::uint64_t{0x3f}; bytes = (bytes + 0x0bf) & ~std::uint64_t{0x3f}; bytes = (bytes + 0x13f) & ~std::uint64_t{0x3f}; - bytes += params->sequences * SequenceBytes; + if (computeSequences != 0) { + const auto queues = static_cast<std::uint64_t>(std::popcount(params->computeQueueMask)); + bytes = (bytes + (params->sequences + queues) * SequenceBytes + 0x267) & ~std::uint64_t{0xff}; + bytes = (bytes + ComputeCommandBytes * queues + 3) & ~std::uint64_t{3}; + bytes = (bytes + 4 * queues + 7) & ~std::uint64_t{7}; + bytes += 8 * queues; + } else { + bytes += params->sequences * SequenceBytes; + } const std::uint32_t nodes = params->dispatches + params->syncPoints + params->waits + static_cast<std::uint32_t>(total) + 1; return AlignUp(bytes, 64) + std::uint64_t{nodes} * NodeBytes + 0x800; } @@ -569,6 +754,14 @@ int APS5_VABI _ZN3sce3Job10JobManager8shutdownEv(void* self) { for (const Pthread worker : manager->workers) { if (scePthreadJoin(worker, nullptr) != 0) throw std::runtime_error("sce::Job::JobManager::shutdown: scePthreadJoin failed"); } + if (manager->computeHandler != nullptr) { + if (sceKernelAddUserEventEdge(manager->computeEvents, 0) != 0 || sceKernelTriggerUserEvent(manager->computeEvents, 0, nullptr) != 0) { + throw std::runtime_error("sce::Job::JobManager::shutdown: the compute handler could not be signalled"); + } + if (scePthreadJoin(manager->computeHandler, nullptr) != 0) throw std::runtime_error("sce::Job::JobManager::shutdown: scePthreadJoin failed"); + for (const std::uint32_t queue : manager->computeQueues) static_cast<void>(sceAgcDriverDeleteEqEvent(manager->computeEvents, static_cast<int>(queue))); + if (sceKernelDeleteEqueue(manager->computeEvents) != 0) throw std::runtime_error("sce::Job::JobManager::shutdown: sceKernelDeleteEqueue failed"); + } Implementation(self) = nullptr; delete manager; return 0; diff --git a/docs/dev/debt/libSceJobManager.md b/docs/dev/debt/libSceJobManager.md index 023da38c79..f1a6c84fee 100644 --- a/docs/dev/debt/libSceJobManager.md +++ b/docs/dev/debt/libSceJobManager.md @@ -2,4 +2,4 @@ ### Functional -- [sce::Job::JobManager](../../../core/libs/prx/libSceJobManager/Export.cpp) (libSceJobManager) is written from the disassembly of the copies PPSA01474 and PPSA36082 ship in `sce_module`, and `guest_job_manager_tests` passes against both of those modules as well: the memory size formula, the error codes, sync point ordering, the per-sequence limits and the priority order are the original's. Only CPU sequences exist: `initialize` and `calculateRequiredMemorySize` with compute sequences, and both compute dispatches, are not implemented, and `createCompute` and `destroyCompute` return the errors the original returns when no compute sequence was configured. The manager's state lives on the host heap, with only the block header written to the title's memory; the original keeps everything in that block, and a block smaller than 0x2d8 bytes throws here. The worker sleep policy, the spin count and the persistent thread count are validated and stored but change nothing, a factory priority above 1 throws, `shutdown` throws while a thread is blocked in a sequence wait, and using a sequence after its `destroy` throws. `initialize` returns 0 where the original returns the result of its last import (`BG26hBGiNlw`, not resolved to a name), the `sceRazorCpuJobManager*` profiler calls are not made, and a worker thread that cannot be created or configured throws where the original does not check. The order of the sequence and factory virtual functions is read from the modules' vtables only, since no title that calls them has been run +- [sce::Job::JobManager](../../../core/libs/prx/libSceJobManager/Export.cpp) (libSceJobManager) is written from the disassembly of the copies PPSA01474 and PPSA36082 ship in `sce_module`, and `guest_job_manager_tests` passes against both of those modules as well: the memory size formula, the error codes, sync point ordering, the per-sequence limits and the priority order are the original's. Compute sequences submit through `sceAgcDriverSubmitAcb` on the queue their mask bit selects and are checked the same way by `agc_driver_job_manager_compute_tests`: results, ordering against CPU dispatches and sync points, limits and sizes match the two modules. The command stream is simpler than the original's: every compute dispatch is submitted with its own label write and interrupt, where the original batches the label write and rewinds commands it encoded ahead, and the command buffers and labels live on the host heap. The per-queue command size in `calculateRequiredMemorySize` is a constant (104, 148 and 32 bytes per dispatch, sync point and wait), which is what the original measures by running the builders of this project's libSceAgc. A compute dispatch on a CPU sequence throws (the original does not check), and a failing `sceAgcDriverAddEqEvent` throws. The manager's state lives on the host heap, with only the block header written to the title's memory; the original keeps everything in that block, and a block smaller than 0x2d8 bytes throws here. The worker sleep policy, the spin count and the persistent thread count are validated and stored but change nothing, a factory priority above 1 throws, `shutdown` throws while a thread is blocked in a sequence wait, and using a sequence after its `destroy` throws. `initialize` returns 0 where the original returns the result of its last import (`BG26hBGiNlw`, not resolved to a name), the `sceRazorCpuJobManager*` profiler calls are not made, and a worker thread that cannot be created or configured throws where the original does not check. The order of the sequence and factory virtual functions is read from the modules' vtables only, since no title that calls them has been run From 394ac0194688945c7c0774c81753d58c623efaf6 Mon Sep 17 00:00:00 2001 From: fma-rc <188743521+fma-rc@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:37:10 +0200 Subject: [PATCH 084/103] fix(libSceJobManager): refuse shutdown with compute work in flight and check the handler wait --- .../tests/execution/JobManagerCompute.cpp | 18 +++++++++++++++--- core/libs/prx/libSceJobManager/Export.cpp | 8 ++++++-- docs/dev/debt/libSceJobManager.md | 2 +- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp index 79af7f5d54..c149aa41a9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp @@ -152,14 +152,19 @@ void SizeCase() { } void RunCase() { - const std::uint64_t size = SizeFor(1, 1, true); + const std::uint64_t size = SizeFor(1, 2, true); void* memory = ::operator new((size + 255) & ~std::uint64_t{255}, std::align_val_t{256}); Require(memory != nullptr, "allocation failed"); alignas(64) static unsigned char manager[0x100]; Construct(manager); const ConfigParams badHandler{1, 0x00021fff, 0x00010000, 0x2bc, nullptr, 0, 0x1fff, 0x50, 0}; - Require(Initialize(manager, memory, size, 2, 1, 1, 1, &badHandler) == InvalidArgument, "a handler priority below 0x100 was accepted"); - Require(Initialize(manager, memory, size, 2, 1, 1, 1, nullptr) == 0, "initialize with one compute sequence failed"); + const ConfigParams noAffinity{1, 0x00021fff, 0x00010000, 0x2bc, nullptr, 0, 0, 0x100, 0}; + const ConfigParams wideAffinity{1, 0x00021fff, 0x00010000, 0x2bc, nullptr, 0, 0x2000, 0x100, 0}; + Require(Initialize(manager, memory, size, 2, 1, 1, 2, &badHandler) == InvalidArgument, "a handler priority below 0x100 was accepted"); + Require(Initialize(manager, memory, size, 2, 1, 1, 2, &noAffinity) == InvalidArgument, "an empty handler affinity was accepted"); + Require(Initialize(manager, memory, size, 2, 1, 1, 2, &wideAffinity) == InvalidArgument, "a handler affinity beyond 13 cores was accepted"); + Require(Initialize(manager, memory, size, 2, 1, 0xffffffffu, 2, nullptr) == InvalidArgument, "a sequence count that wraps was accepted"); + Require(Initialize(manager, memory, size, 2, 1, 1, 2, nullptr) == 0, "initialize with one compute sequence failed"); SequenceFactory* factory = Factories(manager, 0); Sequence* sequence = nullptr; Sequence* other = nullptr; @@ -203,6 +208,13 @@ void RunCase() { Require(Output[0] == 0x33330003u, "the compute dispatch behind a sync point did not run"); Require(SeenByJob == 0x33330003u, "a CPU job behind a sync point ran before the compute dispatch ahead of it finished"); + Value[0] = 0x44440004u; + Require(sequence->DispatchCompute(ShaderWithoutDispatch.data(), Table.data(), 1, 1, 1, nullptr) == 0, "the leading dispatch failed"); + Require(sequence->SyncPoint() == 0, "the sync point between compute dispatches failed"); + Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, "behind") == 0, "the compute dispatch behind a compute dispatch failed"); + Require(sequence->Wait() == 0, "the wait after chained compute dispatches failed"); + Require(Output[0] == 0x44440004u, "a compute dispatch behind a sync point and a compute dispatch did not run"); + for (int index = 0; index < 4; ++index) Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == 0, "a compute dispatch within the limit failed"); Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == LimitExceeded, "a compute dispatch past the dispatch limit was accepted"); Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == Invalid, "a latched compute sequence accepted a dispatch"); diff --git a/core/libs/prx/libSceJobManager/Export.cpp b/core/libs/prx/libSceJobManager/Export.cpp index 9ed6abee66..4c03e4254e 100644 --- a/core/libs/prx/libSceJobManager/Export.cpp +++ b/core/libs/prx/libSceJobManager/Export.cpp @@ -176,6 +176,7 @@ public: void Advance(); void DispatchCompleted(std::uint32_t invocations); void CollectCompute(); + bool Busy() const { return outstanding != 0; } Manager* manager = nullptr; bool open = false; @@ -527,7 +528,7 @@ void* APS5_VABI ComputeHandlerEntry(void* argument) { for (;;) { KernelEvent event{}; int count = 0; - static_cast<void>(sceKernelWaitEqueue(manager.computeEvents, &event, 1, &count, nullptr)); + if (sceKernelWaitEqueue(manager.computeEvents, &event, 1, &count, nullptr) != 0) throw std::runtime_error("libSceJobManager: sceKernelWaitEqueue failed in the compute handler"); if (count <= 0) continue; if (sceKernelGetEventFilter(&event) == UserEventFilter && sceKernelGetEventId(&event) == 0) return nullptr; const std::lock_guard guard(manager.lock); @@ -642,7 +643,7 @@ int Initialize(void* self, void* memory, std::uint64_t size, std::uint32_t worke if ((base & 0xff) != 0) return ManagerMisalignedMemory; if (size == 0 || workers == 0 || workers > MaxWorkers) return ManagerInvalidArgument; if (unified) { - if ((sequences == 0 && computeSequences == 0) || sequences + computeSequences > MaxSequences || (computeQueueMask >> 56) != 0 || static_cast<std::uint32_t>(std::popcount(computeQueueMask)) < computeSequences) return ManagerInvalidArgument; + if ((sequences == 0 && computeSequences == 0) || std::uint64_t{sequences} + computeSequences > MaxSequences || (computeQueueMask >> 56) != 0 || static_cast<std::uint32_t>(std::popcount(computeQueueMask)) < computeSequences) return ManagerInvalidArgument; } else if (sequences == 0 || sequences > MaxSequences) { return ManagerInvalidArgument; } @@ -748,6 +749,9 @@ int APS5_VABI _ZN3sce3Job10JobManager8shutdownEv(void* self) { { const std::lock_guard guard(manager->lock); if (manager->blockedWaiters != 0) throw std::runtime_error("sce::Job::JobManager::shutdown: a thread is still blocked in a sequence wait"); + for (const Sequence& sequence : manager->computeSequences) { + if (sequence.open && sequence.Busy()) throw std::runtime_error("sce::Job::JobManager::shutdown: a compute sequence still has work in flight"); + } manager->running = false; manager->work.notify_all(); } diff --git a/docs/dev/debt/libSceJobManager.md b/docs/dev/debt/libSceJobManager.md index f1a6c84fee..123977e935 100644 --- a/docs/dev/debt/libSceJobManager.md +++ b/docs/dev/debt/libSceJobManager.md @@ -2,4 +2,4 @@ ### Functional -- [sce::Job::JobManager](../../../core/libs/prx/libSceJobManager/Export.cpp) (libSceJobManager) is written from the disassembly of the copies PPSA01474 and PPSA36082 ship in `sce_module`, and `guest_job_manager_tests` passes against both of those modules as well: the memory size formula, the error codes, sync point ordering, the per-sequence limits and the priority order are the original's. Compute sequences submit through `sceAgcDriverSubmitAcb` on the queue their mask bit selects and are checked the same way by `agc_driver_job_manager_compute_tests`: results, ordering against CPU dispatches and sync points, limits and sizes match the two modules. The command stream is simpler than the original's: every compute dispatch is submitted with its own label write and interrupt, where the original batches the label write and rewinds commands it encoded ahead, and the command buffers and labels live on the host heap. The per-queue command size in `calculateRequiredMemorySize` is a constant (104, 148 and 32 bytes per dispatch, sync point and wait), which is what the original measures by running the builders of this project's libSceAgc. A compute dispatch on a CPU sequence throws (the original does not check), and a failing `sceAgcDriverAddEqEvent` throws. The manager's state lives on the host heap, with only the block header written to the title's memory; the original keeps everything in that block, and a block smaller than 0x2d8 bytes throws here. The worker sleep policy, the spin count and the persistent thread count are validated and stored but change nothing, a factory priority above 1 throws, `shutdown` throws while a thread is blocked in a sequence wait, and using a sequence after its `destroy` throws. `initialize` returns 0 where the original returns the result of its last import (`BG26hBGiNlw`, not resolved to a name), the `sceRazorCpuJobManager*` profiler calls are not made, and a worker thread that cannot be created or configured throws where the original does not check. The order of the sequence and factory virtual functions is read from the modules' vtables only, since no title that calls them has been run +- [sce::Job::JobManager](../../../core/libs/prx/libSceJobManager/Export.cpp) (libSceJobManager) is written from the disassembly of the copies PPSA01474 and PPSA36082 ship in `sce_module`, and `guest_job_manager_tests` passes against both of those modules as well: the memory size formula, the error codes, sync point ordering, the per-sequence limits and the priority order are the original's. Compute sequences submit through `sceAgcDriverSubmitAcb` on the queue their mask bit selects and are checked the same way by `agc_driver_job_manager_compute_tests`: results, ordering against CPU dispatches and sync points, limits and sizes match the two modules. The command stream is simpler than the original's: every compute dispatch is submitted with its own label write and interrupt, where the original batches the label write and rewinds commands it encoded ahead, and the command buffers and labels live on the host heap. The per-queue command size in `calculateRequiredMemorySize` is a constant (104, 148 and 32 bytes per dispatch, sync point and wait), which is what the original measures by running the builders of this project's libSceAgc. A compute dispatch on a CPU sequence throws (the original does not check), and a failing `sceAgcDriverAddEqEvent` throws. `shutdown` throws while a compute sequence has work in flight, a sequence's label total is written as 32 bits, and since the label is host memory no GPU import covers, the driver drains recorded work before each label write. The manager's state lives on the host heap, with only the block header written to the title's memory; the original keeps everything in that block, and a block smaller than 0x2d8 bytes throws here. The worker sleep policy, the spin count and the persistent thread count are validated and stored but change nothing, a factory priority above 1 throws, `shutdown` throws while a thread is blocked in a sequence wait, and using a sequence after its `destroy` throws. `initialize` returns 0 where the original returns the result of its last import (`BG26hBGiNlw`, not resolved to a name), the `sceRazorCpuJobManager*` profiler calls are not made, and a worker thread that cannot be created or configured throws where the original does not check. The order of the sequence and factory virtual functions is read from the modules' vtables only, since no title that calls them has been run From 68355e42698e8d9678eea8dfa836889db67e60da Mon Sep 17 00:00:00 2001 From: MontyMcK <MontyMcK@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:20:55 +1300 Subject: [PATCH 085/103] refactor(libSceHmd2): implement the exports where their stubs were --- core/libs/prx/libSceHmd2/Export.cpp | 125 ++++++++++++++-------------- docs/dev/debt/libSceHmd2.md | 5 +- docs/dev/debt/libScePad.md | 2 +- 3 files changed, 68 insertions(+), 64 deletions(-) diff --git a/core/libs/prx/libSceHmd2/Export.cpp b/core/libs/prx/libSceHmd2/Export.cpp index b23019b0df..55d518e133 100644 --- a/core/libs/prx/libSceHmd2/Export.cpp +++ b/core/libs/prx/libSceHmd2/Export.cpp @@ -42,6 +42,16 @@ std::int32_t GetDeviceInformation(Hmd2::DeviceInformation* info) { extern "C" { +int APS5_VABI sceHmd2ReprojectionGetStatus() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceHmd2ReprojectionQuerySeeThroughBufferSizeAlign() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + std::int32_t APS5_VABI sceHmd2Initialize(const void* param) { std::lock_guard lock(stateMutex); if (initialized) return Hmd2::AlreadyInitialized; @@ -50,6 +60,21 @@ std::int32_t APS5_VABI sceHmd2Initialize(const void* param) { return 0; } +std::int32_t APS5_VABI sceHmd2GetFieldOfView(std::int32_t handle, Hmd2::FieldOfView* fieldOfView) { + std::lock_guard lock(stateMutex); + if (!fieldOfView) return Hmd2::ParameterNull; + if (handle == 0 || handle != openHandle) return Hmd2::InvalidHandle; + if (!initialized) return Hmd2::NotInitialized; + *fieldOfView = DefaultFieldOfView; + return 0; +} + +std::int32_t APS5_VABI sceHmd2GetDeviceInformationByHandle(std::int32_t handle, Hmd2::DeviceInformation* info) { + std::lock_guard lock(stateMutex); + if (handle != openHandle) return Hmd2::InvalidHandle; + return GetDeviceInformation(info); +} + std::int32_t APS5_VABI sceHmd2Terminate() { std::lock_guard lock(stateMutex); if (!initialized) return Hmd2::NotInitialized; @@ -57,6 +82,37 @@ std::int32_t APS5_VABI sceHmd2Terminate() { return 0; } +std::int32_t APS5_VABI sceHmd2Close(std::int32_t handle) { + std::lock_guard lock(stateMutex); + if (!initialized) return Hmd2::NotInitialized; + if (handle != openHandle) return Hmd2::InvalidHandle; + openHandle = 0; + return 0; +} + +int APS5_VABI sceHmd2GazeGetResult() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceHmd2GazeGetResultForFoveatedRendering() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +std::int32_t APS5_VABI sceHmd2GetDeviceInformation(Hmd2::DeviceInformation* info) { + std::lock_guard lock(stateMutex); + return GetDeviceInformation(info); +} + +std::int32_t APS5_VABI sceHmd2GetFieldOfViewWithoutHandle(Hmd2::FieldOfView* fieldOfView) { + std::lock_guard lock(stateMutex); + if (!fieldOfView) return Hmd2::ParameterNull; + if (!initialized) return Hmd2::NotInitialized; + *fieldOfView = DefaultFieldOfView; + return 0; +} + std::int32_t APS5_VABI sceHmd2Open(std::int32_t userId, std::int32_t type, std::int32_t index, const void* param) { std::lock_guard lock(stateMutex); if (!initialized) return Hmd2::NotInitialized; @@ -70,68 +126,6 @@ std::int32_t APS5_VABI sceHmd2Open(std::int32_t userId, std::int32_t type, std:: return DeviceHandle; } -std::int32_t APS5_VABI sceHmd2Close(std::int32_t handle) { - std::lock_guard lock(stateMutex); - if (!initialized) return Hmd2::NotInitialized; - if (handle != openHandle) return Hmd2::InvalidHandle; - openHandle = 0; - return 0; -} - -std::int32_t APS5_VABI sceHmd2GetDeviceInformation(Hmd2::DeviceInformation* info) { - std::lock_guard lock(stateMutex); - return GetDeviceInformation(info); -} - -std::int32_t APS5_VABI sceHmd2GetDeviceInformationByHandle(std::int32_t handle, Hmd2::DeviceInformation* info) { - std::lock_guard lock(stateMutex); - if (handle != openHandle) return Hmd2::InvalidHandle; - return GetDeviceInformation(info); -} - -std::int32_t APS5_VABI sceHmd2GetFieldOfView(std::int32_t handle, Hmd2::FieldOfView* fieldOfView) { - std::lock_guard lock(stateMutex); - if (!fieldOfView) return Hmd2::ParameterNull; - if (handle == 0 || handle != openHandle) return Hmd2::InvalidHandle; - if (!initialized) return Hmd2::NotInitialized; - *fieldOfView = DefaultFieldOfView; - return 0; -} - -std::int32_t APS5_VABI sceHmd2GetFieldOfViewWithoutHandle(Hmd2::FieldOfView* fieldOfView) { - std::lock_guard lock(stateMutex); - if (!fieldOfView) return Hmd2::ParameterNull; - if (!initialized) return Hmd2::NotInitialized; - *fieldOfView = DefaultFieldOfView; - return 0; -} - -std::int32_t APS5_VABI sceHmd2SetVibration(std::int32_t handle, std::uint32_t level) { - (void)level; - std::lock_guard lock(stateMutex); - return handle == openHandle ? 0 : Hmd2::InvalidHandle; -} - -int APS5_VABI sceHmd2ReprojectionGetStatus() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceHmd2ReprojectionQuerySeeThroughBufferSizeAlign() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceHmd2GazeGetResult() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceHmd2GazeGetResultForFoveatedRendering() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI sceHmd2ReprojectionBeginFrame() { NotImplemented_nid_no_patch(__func__); return 0; @@ -222,4 +216,11 @@ int APS5_VABI sceHmd2ReprojectionTerminate() { return 0; } +std::int32_t APS5_VABI sceHmd2SetVibration(std::int32_t handle, std::uint32_t level) { + (void)level; + std::lock_guard lock(stateMutex); + if (handle != openHandle) return Hmd2::InvalidHandle; + return 0; +} + } diff --git a/docs/dev/debt/libSceHmd2.md b/docs/dev/debt/libSceHmd2.md index 70033deefa..050b0f7c9d 100644 --- a/docs/dev/debt/libSceHmd2.md +++ b/docs/dev/debt/libSceHmd2.md @@ -8,8 +8,11 @@ - PPSA29866 imports declared without parameters, signatures unknown, added without an implementation because the title fails to load without them: [sceHmd2ReprojectionQuerySeeThroughBufferSizeAlign](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2), [sceVideoOutSetFlipMaster and sceVideoOutClearFlipMaster](../../../core/libs/prx/libSceVideoOut/Unimplemented.cpp) (libSceVideoOut) - PPSA09030 imports from its `Media/Plugins/psvr2.prx` declared without parameters, signatures unknown, added without an implementation because the title fails to load the plugin without them: [sceHmd2ReprojectionGetStatus](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2), [sceVrTracker2GetCoordinateSystem and sceVrTracker2GetPlayAreaBoundaryGeometry](../../../core/libs/prx/libSceVrTracker2/Export.cpp) (libSceVrTracker2) -- [sceHmd2Initialize](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the firmware also fails with 0x8a72001f when `sceKernelGetAppInfo` fails and with 0x8a72000a for some applications, depending on app info fields at 0x2c, 0x4b and 0x4d whose meaning is unknown, and then calls `sceMbusInit`; none of this is modelled - [sceHmd2Open](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the handle is the value the console's driver returned to the first open; titles built with an SDK before 7.00 may open for any user, and the firmware's `sceMbusAddHandleByUserId` result for a user other than the initial one is unknown, so that case throws. 0x8a720006 for a second open is named `AlreadyOpened` from its use - [sceHmd2SetVibration](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the second parameter is checked against 25 only while a headset is connected; its name and unit are unknown. The field of view member names are assumed: of the firmware's default tangents the first is the widest and the last two are equal - PPSA01317 imports declared without parameters, signatures unknown: the [libSceHmd2](../../../core/libs/prx/libSceHmd2/Export.cpp) exports that throw and the remaining [libSceVrTracker2](../../../core/libs/prx/libSceVrTracker2/Export.cpp) exports. Names from the aerolib NID list - PPSA15878 imports declared without parameters, signatures unknown: the [libSceHmd2](../../../core/libs/prx/libSceHmd2/Export.cpp) reprojection and gaze exports, `sceVrTracker2Finalize`, `sceVrTracker2GetPlayAreaOrientedBoundingBox` and `sceVrTracker2LocateCoordinateSystem` in [libSceVrTracker2](../../../core/libs/prx/libSceVrTracker2/Export.cpp), and the `scePadVrController` trigger effect exports in [libScePad](../../../core/libs/prx/libScePad/Export.cpp). Names from the aerolib NID list + +### Functional + +- [sceHmd2Initialize](../../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the firmware also fails with 0x8a72001f when `sceKernelGetAppInfo` fails and with 0x8a72000a for some applications, depending on app info fields at 0x2c, 0x4b and 0x4d whose meaning is unknown, and then calls `sceMbusInit`; none of this is modelled diff --git a/docs/dev/debt/libScePad.md b/docs/dev/debt/libScePad.md index 291c86d4e2..6ec836b141 100644 --- a/docs/dev/debt/libScePad.md +++ b/docs/dev/debt/libScePad.md @@ -6,9 +6,9 @@ - [scePadSetAngularVelocityDeadbandState](../../../core/libs/prx/libScePad/src/PadState.cpp) (libScePad) - with the deadband on, each angular velocity axis whose magnitude is below 0.017453292 rad/s (1 degree per second) reads as 0, before the orientation is integrated; it starts off. Threshold, per-axis test and default follow [duaLib](https://github.com/WujekFoliarz/duaLib/blob/03bad1bea2a36561b520846776f2a07ced6773e0/src/source/duaLib.cpp#L44), which reimplements the PC `libScePad.dll`; the console's threshold is not measured, and the deadband is kept per library rather than per handle since one pad handle exists - [scePadOpen and scePadOpenExt](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) - special ports (type 2) return `DEVICE_NOT_CONNECTED` (0x80920007) because the host has no special controller, and `scePadGetHandle` has no handle for them; whether the console opens a handle and reports the device as disconnected later is unknown. scePadOpenExt handles only special ports. PPSA01317 passes a 16-byte param that names the device by vendor and product id (`b7 0e 04 0e ...` for 0x0eb7:0x0e04) and skips the user on any negative result. Other port types throw - [scePadVrControllerReadState](../../../core/libs/prx/libScePad/Export.cpp) (libScePad, NID `iA-DdobUen8`) - the name hashes to the imported NID; the `(int handle, void* state)` signature follows [PS5VR](https://github.com/theghostonline/PS5VR/blob/0e408909ce7a39891a2e06eee3438d07a09e11e3/app/src/app_vr.c#L326-L369) (PPSA99177), which passes a PS VR2 Sense controller handle from `scePadOpen` type 3 or 4 and a 0x68-byte state it reads as buttons at 0, stick bytes at 4 and a connected byte at 0x40. The title calls it only after `sceHmd2Initialize` succeeds, which it does not here, so every call throws; the state layout is otherwise unknown +- PPSA07411 PS VR2 imports declared without parameters, signatures unknown, added without an implementation because the title fails to load without them: [scePadVrControllerSetVibration, scePadVrControllerSetVibrationMode and scePadVrControllerReadCombinedState](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) and [sceVrTracker2ResetLocalCoordinate](../../../core/libs/prx/libSceVrTracker2/Export.cpp) (libSceVrTracker2) - [scePadSetVibrationMode](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) - modes 1 to 3 and 5 are accepted and stored. On a PS5 (firmware 13.60) the call returns `0x80920005` before `scePadInit`, then rejects a mode of 0 or below with `INVALID_ARG` before it checks the handle, and returns `INVALID_HANDLE` for a bad handle with any higher mode; whether an open handle accepts modes above 3 was not measured, since the payload could not open a pad; 5 is accepted because PPSA25286 passes it on its open handle at start-up; the meanings of modes 3 and 5 are unknown (PPSA21480 calls it with 3 right after `scePadOpen` and aborts on failure; [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Pad/Pad.cs) names only 1 and 2). The stored mode has no other effect - [scePadIsRemoteController](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) - the name hashes to NID `fCWdlnmB1Ks`; the signature `(int handle, bool* remote)` comes from PPSA16064, which zeroes a byte, passes its address after `scePadGetControllerInformation` and ignores the result. The error codes follow the other pad calls. Controllers are local devices, so the answer is always false - [scePadSetAngularVelocityBiasCorrectionState](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) - inferred signature `(int handle, bool enabled)` follows `scePadSetTiltCorrectionState` and `scePadSetAngularVelocityDeadbandState`; an invalid handle returns `PAD_ERROR_INVALID_HANDLE`, disabling is a no-op, and enabling throws because bias correction is not modelled - [scePadUnknown_zBsgF0q8DIM](../../../core/libs/prx/libScePad/Export.cpp) (libScePad, NID `zBsgF0q8DIM`) - PPSA03671 imports it and fails to load without it. The aerolib list has no name for the NID, and none of the pad names in SharpProspero, shadPS4 or OpenOrbis, nor the trigger effect, motion sensor, VR controller, HD and haptic names tried, hashes to it. It is declared without parameters and throws. The title's executable has an import stub for it but no call, jump or address reference to that stub, so the argument count and whether the result is used are unknown - [scePadUnknown_n3kSX62fgNo](../../../core/libs/prx/libScePad/Export.cpp) (libScePad, NID `n3kSX62fgNo`) - PPSA01530 and PPSA13356 import it and fail to load without it. The real name is unknown; it is declared without parameters and throws -- PPSA07411 PS VR2 imports declared without parameters, signatures unknown, added without an implementation because the title fails to load without them: [scePadVrControllerSetVibration, scePadVrControllerSetVibrationMode and scePadVrControllerReadCombinedState](../../../core/libs/prx/libScePad/Export.cpp) (libScePad) and [sceVrTracker2ResetLocalCoordinate](../../../core/libs/prx/libSceVrTracker2/Export.cpp) (libSceVrTracker2) From 5e340fa54d44a55d6bfab7dc748008e2f60abe1d Mon Sep 17 00:00:00 2001 From: blu <131330772+ycbuzi@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:37:21 +1300 Subject: [PATCH 086/103] fix(relinker): append the test temp directory instead of replacing the environment set_tests_properties replaced the ASAN_OPTIONS that add_asan_test_options appends to guest_module_args, guest_deferred_dependencies and guest_module_fini in ASan builds. Append the TMPDIR, TEMP and TMP settings instead. --- core/relinker/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index dab82857bf..11c7ea959a 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -739,7 +739,7 @@ if(BUILD_TESTING) set(relinkerTestTemp ${CMAKE_CURRENT_BINARY_DIR}/test-temp) file(MAKE_DIRECTORY ${relinkerTestTemp}) get_property(relinkerTests DIRECTORY PROPERTY TESTS) - set_tests_properties(${relinkerTests} PROPERTIES ENVIRONMENT_MODIFICATION + set_property(TEST ${relinkerTests} APPEND PROPERTY ENVIRONMENT_MODIFICATION "TMPDIR=set:${relinkerTestTemp};TEMP=set:${relinkerTestTemp};TMP=set:${relinkerTestTemp}") endif() From 6c47f92b52a70d8f39877b1261f564d9213a3ae8 Mon Sep 17 00:00:00 2001 From: milescoviello <kitslayer29@gmail.com> Date: Sun, 18 Oct 2026 00:43:33 +0000 Subject: [PATCH 087/103] fix(shader): pass clip distances through the rect-list stages as the vertex shader writes them The recompiled vertex shader writes gl_ClipDistance through a variable of its own beside its position block, but the rect-list control and evaluation stages read and wrote it as a second member of their per-vertex block, so the stage interfaces did not match (VUID-RuntimeSpirv-OpVariable-08746). They now take and forward the clip distances through ClipDistance variables of their own. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../prx/libSceAgcDriver/tests/Graphics.cpp | 32 +++++++++++++++++++ .../SpirvBackend/src/RectListShaders.cpp | 23 +++++++++---- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index c4d38792ee..720aa42f40 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -6042,6 +6042,23 @@ void registeredReadableCoverTests() { ::operator delete(memory, std::align_val_t{block}); } +std::size_t VariableBuiltIns(const ShaderRecompiler::RecompileResult& result, spv::BuiltIn builtin) { + std::size_t count = 0; + const auto& words = result.spirv.Words(); + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + if (static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpDecorate && (words[at] >> 16u) == 4u && words[at + 2] == spv::DecorationBuiltIn && words[at + 3] == builtin) ++count; + } + return count; +} + +bool MemberBuiltIn(const ShaderRecompiler::RecompileResult& result, spv::BuiltIn builtin) { + const auto& words = result.spirv.Words(); + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + if (static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpMemberDecorate && words[at + 3] == spv::DecorationBuiltIn && words[at + 4] == builtin) return true; + } + return false; +} + void rectListTests() { using namespace ShaderRecompiler; using namespace AgcDriver::Graphics; @@ -6075,6 +6092,21 @@ void rectListTests() { expectFailure([&] { ValidateShaders(shaders, state, VkPhysicalDeviceSubgroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}, false); }, "stage count"); } } + { + RecompileResult clipped; + clipped.spirv = makeModule({.clipDistanceLength = 2}); + const std::array<std::uint32_t, 3> clipCapabilities{spv::CapabilityShader, spv::CapabilityTessellation, spv::CapabilityClipDistance}; + auto clipTarget = target; + clipTarget.supportedCapabilities = clipCapabilities; + auto clippedRect = BuildRectListShaders(clipped, fragment, clipTarget); + for (const auto* stage : {&clippedRect.control, &clippedRect.lastVertexControl, &clippedRect.evaluation}) { + Require(!MemberBuiltIn(*stage, spv::BuiltInClipDistance) && VariableBuiltIns(*stage, spv::BuiltInClipDistance) == 2u, "a rect-list stage does not take and forward the vertex clip distances through variables of their own, as the vertex shader writes them"); + } + const std::array<CompiledShader, 4> clippedShaders{{{ShaderStage::Vertex, &clipped, 0}, {ShaderStage::TessellationControl, &clippedRect.control, 0}, {ShaderStage::TessellationEvaluation, &clippedRect.evaluation, 0}, {ShaderStage::Fragment, &fragment, 0}}}; + auto queue = makeState(); + queue.userConfig[0x242] = 17; + ValidateShaders(clippedShaders, DecodeState(queue), VkPhysicalDeviceSubgroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}, false, false, false, false, false, false, true); + } vertex.spirv = makeModule({.parameterOutput = true}); fragment.spirv = makeModule({.fragment = true, .rectParameters = true}); vertex.parameterExports = {0}; diff --git a/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp b/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp index 9be097f99f..053a298982 100644 --- a/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp +++ b/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp @@ -42,13 +42,11 @@ public: vec4FloatType = type(spv::OpTypeVector, floatType, 4u); functionType = type(spv::OpTypeFunction, voidType); + perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpMemberDecorate, {0u, spv::DecorationInvariant}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType); if (clipDistances != 0u) { clipArrayType = array(floatType, clipDistances); - perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpMemberDecorate, {0u, spv::DecorationInvariant}}, {spv::OpMemberDecorate, {1u, spv::DecorationBuiltIn, spv::BuiltInClipDistance}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType, clipArrayType); ptrInputClip = pointer(spv::StorageClassInput, clipArrayType); ptrOutputClip = pointer(spv::StorageClassOutput, clipArrayType); - } else { - perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpMemberDecorate, {0u, spv::DecorationInvariant}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType); } ptrInputVec4Float = pointer(spv::StorageClassInput, vec4FloatType); @@ -126,8 +124,8 @@ public: store(access(ptrOutputVec4Float, glOut, invocation, intConstant(0)), position); if (clipDistances != 0u) { std::array<std::uint32_t, 3> clips {}; - for (std::uint32_t i = 0; i < clips.size(); i++) clips[i] = load(clipArrayType, access(ptrInputClip, glIn, intConstant(i), intConstant(1))); - const auto own = load(clipArrayType, access(ptrInputClip, glIn, index, intConstant(1))); + for (std::uint32_t i = 0; i < clips.size(); i++) clips[i] = load(clipArrayType, access(ptrInputClip, clipIn, intConstant(i))); + const auto own = load(clipArrayType, access(ptrInputClip, clipIn, index)); std::vector<std::uint32_t> values; for (std::uint32_t k = 0; k < clipDistances; k++) { auto sum = constant(floatType, 0); @@ -136,7 +134,7 @@ public: } const auto clip = builder.AllocateId(); builder.AddFunction(spv::OpCompositeConstruct, clipArrayType, clip, std::span<const std::uint32_t>(values)); - store(access(ptrOutputClip, glOut, invocation, intConstant(1)), clip); + store(access(ptrOutputClip, clipOut, invocation), clip); } for (std::uint32_t i = 0; i < parameters.size(); i++) { @@ -171,7 +169,7 @@ public: const auto position = load(vec4FloatType, access(ptrInputVec4Float, glIn, index, intConstant(0))); store(access(ptrOutputVec4Float, glOut, intConstant(0)), position); - if (clipDistances != 0u) store(access(ptrOutputClip, glOut, intConstant(1)), load(clipArrayType, access(ptrInputClip, glIn, index, intConstant(1)))); + if (clipDistances != 0u) store(clipOut, load(clipArrayType, access(ptrInputClip, clipIn, index))); for (std::uint32_t i = 0; i < parameters.size(); i++) { store(outputs[i], load(vec4FloatType, access(ptrInputVec4Float, inputs[i], index))); } @@ -287,6 +285,12 @@ private: return variable; } + std::uint32_t clipVariable(spv::StorageClass storage, std::uint32_t type) { + const auto variable = addInterface(storage, type); + decorate(variable, spv::DecorationBuiltIn, spv::BuiltInClipDistance); + return variable; + } + void decorate(std::uint32_t target, spv::Decoration decoration, std::uint32_t value) { builder.AddAnnotation(spv::OpDecorate, target, decoration, value); } @@ -320,6 +324,7 @@ private: decorate(tessCoord, spv::DecorationBuiltIn, spv::BuiltInTessCoord); } glIn = addInterface(spv::StorageClassInput, array(perVertexType, tessControl ? 3u : 4u)); + if (clipDistances != 0u) clipIn = clipVariable(spv::StorageClassInput, array(clipArrayType, tessControl ? 3u : 4u)); inputs.resize(parameters.size()); std::array<std::uint32_t, 32> locations {}; @@ -340,6 +345,7 @@ private: const auto tessControl = model == spv::ExecutionModelTessellationControl; if (tessControl) { glOut = addInterface(spv::StorageClassOutput, array(perVertexType, 4u)); + if (clipDistances != 0u) clipOut = clipVariable(spv::StorageClassOutput, array(clipArrayType, 4u)); tessInner = addInterface(spv::StorageClassOutput, array(floatType, 2u)); decorate(tessInner, spv::DecorationBuiltIn, spv::BuiltInTessLevelInner); builder.AddAnnotation(spv::OpDecorate, tessInner, spv::DecorationPatch); @@ -348,6 +354,7 @@ private: builder.AddAnnotation(spv::OpDecorate, tessOuter, spv::DecorationPatch); } else { glOut = addInterface(spv::StorageClassOutput, perVertexType); + if (clipDistances != 0u) clipOut = clipVariable(spv::StorageClassOutput, clipArrayType); } outputs.resize(parameters.size()); @@ -394,6 +401,8 @@ private: std::uint32_t ptrOutputVec4Float = 0; std::uint32_t glIn = 0; std::uint32_t glOut = 0; + std::uint32_t clipIn = 0; + std::uint32_t clipOut = 0; std::uint32_t tessInner = 0; std::uint32_t tessOuter = 0; std::uint32_t tessCoord = 0; From 087c57ac8419b2fe5835306d07c21eefbea8943a Mon Sep 17 00:00:00 2001 From: MarcZX100 <MarcZX100@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:45:12 +0000 Subject: [PATCH 088/103] test: retry transient Windows filesystem cleanup --- core/libs/tests/GuestFilesystem.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/libs/tests/GuestFilesystem.cpp b/core/libs/tests/GuestFilesystem.cpp index f14e191c80..f474c03570 100644 --- a/core/libs/tests/GuestFilesystem.cpp +++ b/core/libs/tests/GuestFilesystem.cpp @@ -13,6 +13,7 @@ #include <cstring> #include <stdexcept> #include <string> +#include <thread> #include <vector> #ifdef _WIN32 #include <io.h> @@ -596,7 +597,13 @@ int main() { Require(rename_nid_postfix(name(replacement).c_str(), name(dangling).c_str()) == 0); Require(!std::filesystem::is_symlink(dangling) && std::filesystem::is_regular_file(dangling)); #endif - std::filesystem::remove_all(area); + std::error_code cleanupError; + for (int attempt = 0; attempt < 10; ++attempt) { + std::filesystem::remove_all(area, cleanupError); + if (!cleanupError) break; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + Require(!cleanupError); } Require(remove_nid_postfix(file.string().c_str()) == 0); Require(!std::filesystem::exists(file)); From 5c3d21023f924d9854054cfd156f9763e62c716a Mon Sep 17 00:00:00 2001 From: thuanquang <ghoulbulldog@gmail.com> Date: Sun, 18 Oct 2026 07:43:54 +0700 Subject: [PATCH 089/103] fix(agc): keep one trace thread pointer and recording flag across modules PerformanceTrace kept the recording flag and each thread's trace buffer as inline static members, so on Windows every module that includes the header had its own copies: agc_driver_registration_trace read an empty pointer, and libSceVideoOut's timers never saw the trace recording. They are now defined once in the driver. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 3 +++ .../Execution/include/PerformanceTrace.hpp | 18 ++++++++++++++---- .../Execution/src/PerformanceTraceThread.cpp | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/PerformanceTraceThread.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index fcd75d52e0..cd94455327 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -185,6 +185,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Synchronization/WaitStatistics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/WorkerSampler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PerformanceTrace.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PerformanceTraceThread.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/TraceFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ThreadPriority.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp @@ -494,6 +495,7 @@ add_test_executable(agc_shader_memory_tests EXCLUDE_FROM_ALL tests/ShaderMemory.cpp Execution/src/ShaderMemory.cpp Execution/src/GuestMemory.cpp + Execution/src/PerformanceTraceThread.cpp ) target_include_directories(agc_shader_memory_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_shader_memory_tests PRIVATE shader_recompiler_agc_driver libc) @@ -518,6 +520,7 @@ configure_windows_unwind(agc_shared_write_tests) add_test_executable(agc_write_tracking_tests EXCLUDE_FROM_ALL tests/WriteTracking.cpp Execution/src/GuestMemory.cpp + Execution/src/PerformanceTraceThread.cpp ) target_include_directories(agc_write_tracking_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_write_tracking_tests PRIVATE shader_recompiler_agc_driver libc) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp index 2a5314f9ff..5c7fb7d2a6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp @@ -21,6 +21,13 @@ #include <utility> #include <vector> +namespace AgcDriver { +class TraceThread; +} + +extern "C" bool AgcDriverTraceRecording_nid_postfix(); +extern "C" AgcDriver::TraceThread* AgcDriverTraceThread_nid_postfix(); + namespace AgcDriver { class VulkanDevice; @@ -532,23 +539,26 @@ public: } static bool Recording() { - return recording.load(std::memory_order_relaxed); + return AgcDriverTraceRecording_nid_postfix(); } static TraceThread* Thread() { - return local; + return AgcDriverTraceThread_nid_postfix(); } static void StopOutput(); private: + friend bool ::AgcDriverTraceRecording_nid_postfix(); + friend TraceThread* ::AgcDriverTraceThread_nid_postfix(); + static void OpenInterval(std::chrono::steady_clock::time_point now, std::uint64_t present); static void FlushInterval(std::chrono::steady_clock::time_point now, std::uint32_t width, std::uint32_t height, const VulkanDevice* device); - inline static std::atomic<bool> recording{false}; + static std::atomic<bool> recording; inline static std::atomic<bool> buffering{false}; inline static std::atomic<std::uint64_t> presentCount{0}; - inline static thread_local TraceThread* local = nullptr; + static thread_local TraceThread* local; inline static std::mutex gpuMutex; inline static std::vector<TraceGpuRange> gpuRanges; inline static std::uint64_t gpuLost = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/PerformanceTraceThread.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/PerformanceTraceThread.cpp new file mode 100644 index 0000000000..9ce81e5294 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/PerformanceTraceThread.cpp @@ -0,0 +1,16 @@ +#include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp" + +namespace AgcDriver { + +std::atomic<bool> PerformanceTrace::recording{false}; +thread_local TraceThread* PerformanceTrace::local = nullptr; + +} + +extern "C" bool AgcDriverTraceRecording_nid_postfix() { + return AgcDriver::PerformanceTrace::recording.load(std::memory_order_relaxed); +} + +extern "C" AgcDriver::TraceThread* AgcDriverTraceThread_nid_postfix() { + return AgcDriver::PerformanceTrace::local; +} From 563cdd6e99deb462eae7bca333b9af20783846c3 Mon Sep 17 00:00:00 2001 From: milescoviello <kitslayer29@gmail.com> Date: Sun, 18 Oct 2026 01:00:16 +0000 Subject: [PATCH 090/103] test(agc): create fixture command pools with the reset flag Seven test fixtures create their own command pool without VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, while the driver's own pool sets it and Recorder begins a used command buffer again without an explicit reset. The Khronos validation layer reports VUID-vkBeginCommandBuffer-commandBuffer-00050 in depth_storage and partially_resident_snapshot --pending-stores for this. The fixtures now create the pool the way VulkanDevice does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp | 1 + core/libs/prx/libSceAgcDriver/tests/execution/DepthStorage.cpp | 1 + .../libSceAgcDriver/tests/execution/OversizedDescriptorSets.cpp | 1 + .../tests/execution/PartiallyResidentSnapshot.cpp | 1 + .../prx/libSceAgcDriver/tests/execution/PooledBufferRange.cpp | 1 + .../prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp | 1 + .../prx/libSceAgcDriver/tests/execution/SparseBufferHoles.cpp | 1 + 7 files changed, 7 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp index 860643bd27..61a1612f87 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp @@ -102,6 +102,7 @@ public: context.bufferDeviceAddress = true; context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pool.queueFamilyIndex = family; Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorage.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorage.cpp index 237d3f7646..3ab804d307 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorage.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorage.cpp @@ -109,6 +109,7 @@ public: if (hasExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) context.memoryProperties2 = function<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2"); context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pool.queueFamilyIndex = family; Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/OversizedDescriptorSets.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/OversizedDescriptorSets.cpp index be56587bfc..8984302a91 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/OversizedDescriptorSets.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/OversizedDescriptorSets.cpp @@ -84,6 +84,7 @@ public: context.formatProperties = function<PFN_vkGetPhysicalDeviceFormatProperties>("vkGetPhysicalDeviceFormatProperties"); context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pool.queueFamilyIndex = family; Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PartiallyResidentSnapshot.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PartiallyResidentSnapshot.cpp index 005a1e506f..5d747bf85a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PartiallyResidentSnapshot.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PartiallyResidentSnapshot.cpp @@ -110,6 +110,7 @@ public: context.imageFormatProperties = function<PFN_vkGetPhysicalDeviceImageFormatProperties>("vkGetPhysicalDeviceImageFormatProperties"); context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pool.queueFamilyIndex = family; Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PooledBufferRange.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PooledBufferRange.cpp index b8276c8bb1..13e19d7dcb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PooledBufferRange.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PooledBufferRange.cpp @@ -214,6 +214,7 @@ public: context.formatProperties = function<PFN_vkGetPhysicalDeviceFormatProperties>("vkGetPhysicalDeviceFormatProperties"); context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pool.queueFamilyIndex = family; Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp index a1b31188b8..fe436e6b22 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp @@ -130,6 +130,7 @@ public: if (hasExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) context.memoryProperties2 = function<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2"); context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pool.queueFamilyIndex = family; Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SparseBufferHoles.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SparseBufferHoles.cpp index add6ad6737..a7a25f9f88 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SparseBufferHoles.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SparseBufferHoles.cpp @@ -96,6 +96,7 @@ public: function<PFN_vkGetPhysicalDeviceMemoryProperties>("vkGetPhysicalDeviceMemoryProperties")(context.physical, &context.memory); context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pool.queueFamilyIndex = family; Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); } catch (...) { From f27ad681eec266f341d3847db1604fed2c0f6154 Mon Sep 17 00:00:00 2001 From: thibaut <thibaut-leaux-angeli@live.fr> Date: Sun, 18 Oct 2026 03:08:58 +0200 Subject: [PATCH 091/103] test(agc): keep the resource cache growth test short The test now sets a base bound of 32 entries and a first shrink window of 2 (ResourceCache::SetBase, applied at the next Clear()) and turns dispatch recipes off, so every dispatch goes through the cache: grow, shrink, regrow, backoff and the Clear() reset take about 1.2k dispatches instead of about 100k. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XBXM4pgkMHYEtqN1EQni2P --- .../Graphics/include/ShaderResources.hpp | 3 ++ .../Graphics/src/ShaderResources.cpp | 16 ++++++---- .../tests/execution/ResourceCacheGrowth.cpp | 29 ++++++++++++++----- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index ef70d27744..d1c446b69d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -631,6 +631,7 @@ public: void Clear(); std::size_t Size() const; std::size_t Capacity() const; + void SetBase(std::size_t entries, std::size_t shrinkWindow); std::uint64_t TableElements() const; static std::uint64_t TableElementBudget(); static constexpr std::size_t InitialEntries = 1024; @@ -662,6 +663,8 @@ private: std::size_t usesSinceCheck = 0; static constexpr std::size_t InitialShrinkWindow = 8; static constexpr std::size_t MaximumShrinkWindow = 1024; + std::size_t baseEntries = InitialEntries; + std::size_t baseShrinkWindow = InitialShrinkWindow; std::size_t shrinkWindow = InitialShrinkWindow; bool shrankLast = false; }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 907062d951..4089e2702d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -3179,7 +3179,7 @@ void ResourceCache::noteEvicted(const Key& key) { } void ResourceCache::noteUse(const Key& key) { - if (fixedCapacity || capacity <= InitialEntries) return; + if (fixedCapacity || capacity <= baseEntries) return; usedKeys.insert(KeyHash{}(key)); if (++usesSinceCheck < capacity * shrinkWindow) return; const auto used = usedKeys.size(); @@ -3188,7 +3188,7 @@ void ResourceCache::noteUse(const Key& key) { usesSinceCheck = 0; if (used * 4u > capacity) return; const auto previous = capacity; - capacity = std::max(capacity / 2u, InitialEntries); + capacity = std::max(capacity / 2u, baseEntries); shrankLast = true; evictionsSinceCheck = 0; returnsSinceCheck = 0; @@ -3335,7 +3335,7 @@ void ResourceCache::Insert(const Key& key, std::shared_ptr<ShaderResources> reso const char* value = std::getenv("APS5_RESOURCE_CACHE_ENTRIES"); const auto parsed = value ? std::strtoull(value, nullptr, 10) : 0ull; fixedCapacity = parsed != 0; - capacity = fixedCapacity ? static_cast<std::size_t>(parsed) : InitialEntries; + capacity = fixedCapacity ? static_cast<std::size_t>(parsed) : baseEntries; } noteUse(key); while (entries.size() > capacity) { @@ -3410,11 +3410,17 @@ void ResourceCache::Clear() { returnsSinceCheck = 0; usedKeys.clear(); usesSinceCheck = 0; - shrinkWindow = InitialShrinkWindow; + shrinkWindow = baseShrinkWindow; shrankLast = false; if (!fixedCapacity) capacity = 0; } +void ResourceCache::SetBase(std::size_t entries, std::size_t window) { + std::lock_guard lock(mutex); + baseEntries = entries; + baseShrinkWindow = window; +} + std::size_t ResourceCache::Size() const { std::lock_guard lock(mutex); return entries.size(); @@ -3422,7 +3428,7 @@ std::size_t ResourceCache::Size() const { std::size_t ResourceCache::Capacity() const { std::lock_guard lock(mutex); - return capacity == 0 ? InitialEntries : capacity; + return capacity == 0 ? baseEntries : capacity; } std::uint64_t ResourceCache::TableElementBudget() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp index bae94f9150..12242e569c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ResourceCacheGrowth.cpp @@ -29,16 +29,17 @@ namespace { using AgcDriver::Graphics::Require; -constexpr std::uint32_t Dispatches = 1536; +constexpr std::size_t BaseEntries = 32; +constexpr std::size_t BaseShrinkWindow = 2; +constexpr std::uint32_t Dispatches = 48; constexpr std::uint32_t Passes = 4; -constexpr std::uint32_t HotDispatches = 64; -constexpr std::uint32_t HotRepeats = 640; +constexpr std::uint32_t HotDispatches = 2; +constexpr std::uint32_t HotRepeats = 160; constexpr std::size_t SrtBytes = 64; constexpr std::size_t OutputBytes = 256; constexpr std::size_t SrtOffset = 4096; constexpr std::size_t OutputOffset = SrtOffset + static_cast<std::size_t>(Dispatches) * SrtBytes; constexpr std::size_t BlockBytes = (OutputOffset + static_cast<std::size_t>(Dispatches) * OutputBytes + 65535) & ~static_cast<std::size_t>(65535); -constexpr std::size_t InitialEntries = 1024; constexpr std::array<std::uint32_t, 7> StoreCode{0xf4080100u, 0xfa000000u, 0xbf8c0000u, 0x7e020281u, 0xe0700000u, 0x80010100u, 0xbf810000u}; @@ -117,13 +118,13 @@ void WorkingSetLargerThanTheInitialBound(const GuestBlock& block) { } } const auto kept = cache.Size(); - Require(kept >= Dispatches, "resource cache growth: " + std::to_string(Dispatches) + " dispatches with distinct resources repeated " + std::to_string(Passes) + " times in turn, but the resource cache kept only " + std::to_string(kept) + " entries (initial bound " + std::to_string(InitialEntries) + "), so every pass rebuilt every set"); + Require(kept >= Dispatches, "resource cache growth: " + std::to_string(Dispatches) + " dispatches with distinct resources repeated " + std::to_string(Passes) + " times in turn, but the resource cache kept only " + std::to_string(kept) + " entries (initial bound " + std::to_string(BaseEntries) + "), so every pass rebuilt every set"); } void BoundComesBackDown(const GuestBlock& block) { auto& cache = AgcDriver::Graphics::SharedResourceCache(); const auto grown = cache.Capacity(); - Require(grown > InitialEntries, "resource cache growth: the bound stayed at " + std::to_string(grown) + " entries after the large working set"); + Require(grown > BaseEntries, "resource cache growth: the bound stayed at " + std::to_string(grown) + " entries after the large working set"); RunPass(block, HotDispatches, HotRepeats); const auto shrunk = cache.Capacity(); Require(shrunk < grown, "resource cache growth: " + std::to_string(HotDispatches) + " sets served " + std::to_string(HotDispatches * HotRepeats) + " dispatches, but the bound stayed at " + std::to_string(shrunk) + " entries"); @@ -140,20 +141,32 @@ void BoundComesBackDown(const GuestBlock& block) { Require(cache.Capacity() == regrown, "resource cache growth: the bound came down again after the same hot pass, although the last shrink had been undone"); cache.Clear(); - Require(cache.Capacity() == InitialEntries, "resource cache growth: Clear() left the bound at " + std::to_string(cache.Capacity()) + " entries instead of " + std::to_string(InitialEntries)); + Require(cache.Capacity() == BaseEntries, "resource cache growth: Clear() left the bound at " + std::to_string(cache.Capacity()) + " entries instead of " + std::to_string(BaseEntries)); Prepare(block); for (std::uint32_t pass = 0; pass < Passes; ++pass) RunPass(block); - Require(cache.Capacity() > InitialEntries, "resource cache growth: after Clear() the large working set did not grow the bound again"); + Require(cache.Capacity() > BaseEntries, "resource cache growth: after Clear() the large working set did not grow the bound again"); +} + +void UseTheResourceCacheOnEveryDispatch() { +#ifdef _WIN32 + _putenv_s("APS5_NO_DISPATCH_RECIPE", "1"); +#else + setenv("APS5_NO_DISPATCH_RECIPE", "1", 1); +#endif } } int main() { + UseTheResourceCacheOnEveryDispatch(); try { { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; } + auto& cache = AgcDriver::Graphics::SharedResourceCache(); + cache.SetBase(BaseEntries, BaseShrinkWindow); + cache.Clear(); GuestBlock block; Prepare(block); WorkingSetLargerThanTheInitialBound(block); From f96cdc1e7eb872466241b45554f069786677d443 Mon Sep 17 00:00:00 2001 From: MarcZX100 <MarcZX100@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:10:16 +0000 Subject: [PATCH 092/103] fix(libc): mark unimplemented NID handler as non-returning --- core/libs/prx/libc/include/General.hpp | 2 +- core/libs/prx/libc/src/General.cpp | 2 +- core/libs/tests/WindowsExceptions.cpp | 2 +- core/libs/tests/shared_memory/Support.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libc/include/General.hpp b/core/libs/prx/libc/include/General.hpp index df34c7d52b..34cee2743f 100644 --- a/core/libs/prx/libc/include/General.hpp +++ b/core/libs/prx/libc/include/General.hpp @@ -9,7 +9,7 @@ #include "general/VabiMacros.hpp" #include "general/ExportMacros.hpp" -extern "C" void NotImplemented_nid_no_patch(const char* funcName); +extern "C" [[noreturn]] void NotImplemented_nid_no_patch(const char* funcName); extern "C" void CxaFinalize_nid_no_patch(void* dsoHandle); extern "C" bool StopAtPrefixLetter_nid_no_patch(const char* str, char** endptr, int base); #ifndef _WIN32 diff --git a/core/libs/prx/libc/src/General.cpp b/core/libs/prx/libc/src/General.cpp index 0f18419dfd..d96670de58 100644 --- a/core/libs/prx/libc/src/General.cpp +++ b/core/libs/prx/libc/src/General.cpp @@ -614,6 +614,6 @@ extern "C" void KeepStandardHandle_nid_no_patch(int descriptor, void* handle) { } #endif -extern "C" void NotImplemented_nid_no_patch(const char* funcName) { +extern "C" [[noreturn]] void NotImplemented_nid_no_patch(const char* funcName) { throw std::runtime_error(std::string(funcName) + " not implemented"); } diff --git a/core/libs/tests/WindowsExceptions.cpp b/core/libs/tests/WindowsExceptions.cpp index 99a3ca9282..19a4bb2e86 100644 --- a/core/libs/tests/WindowsExceptions.cpp +++ b/core/libs/tests/WindowsExceptions.cpp @@ -57,7 +57,7 @@ static void TestTypeInfoVtables() { if (catchType(&other, &single, &object, 0)) throw std::runtime_error("unrelated RTTI catch succeeded"); } -extern "C" void NotImplemented_nid_no_patch(const char*); +extern "C" [[noreturn]] void NotImplemented_nid_no_patch(const char*); static void RequireHandle(bool value) { if (!value) throw std::runtime_error("child process setup failed"); diff --git a/core/libs/tests/shared_memory/Support.cpp b/core/libs/tests/shared_memory/Support.cpp index 03f202e4ef..e9df457817 100644 --- a/core/libs/tests/shared_memory/Support.cpp +++ b/core/libs/tests/shared_memory/Support.cpp @@ -6,6 +6,6 @@ extern "C" int* APS5_VABI __error_nid_postfix() { return &error; } -extern "C" void NotImplemented_nid_no_patch(const char* name) { +extern "C" [[noreturn]] void NotImplemented_nid_no_patch(const char* name) { throw std::runtime_error(name); } From 304798015b0142e82ac40aab092a5bfe45edc317 Mon Sep 17 00:00:00 2001 From: Nikhil Panchal <37629345+nikhilpanchal1@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:27:02 +0900 Subject: [PATCH 093/103] fix(agc): harden host import construction and mutation waits Guard incomplete imports and preserve mutation waiter errors and progress. Follow-up to #2575; refs #2352. --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 61 ++ .../Graphics/src/GuestBufferMemory.cpp | 183 ++-- .../tests/GuestMemoryLifetimeFixture.cpp | 82 ++ .../tests/GuestMemoryLifetimeSupport.hpp | 34 + .../tests/HostImportLifetime.cpp | 803 ++++++++++++++++++ .../tests/HostImportLifetime.hpp | 10 + .../prx/libSceAgcDriver/tests/Recorder.cpp | 3 + .../tests/execution/GuestMemoryLifetime.cpp | 259 ++++++ .../tests/test_guest_memory_lifetime.py | 143 ++++ core/libs/prx/libc/src/GuestAllocations.cpp | 9 +- 10 files changed, 1500 insertions(+), 87 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/GuestMemoryLifetimeFixture.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/GuestMemoryLifetimeSupport.hpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/HostImportLifetime.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/HostImportLifetime.hpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/GuestMemoryLifetime.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/test_guest_memory_lifetime.py diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 1927c61f6f..951a7c85ed 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1093,6 +1093,7 @@ add_custom_command( add_test_executable(agc_driver_recorder_tests EXCLUDE_FROM_ALL tests/Recorder.cpp + tests/HostImportLifetime.cpp tests/ResidentPresent.cpp ${agcTriangleHeaders} ${agcSampleLodShaderHeader} @@ -1273,6 +1274,22 @@ if(BUILD_TESTING) set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps agc_draw_descriptors PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>") endif() + add_test(NAME agc_driver_host_import_lifetime COMMAND agc_driver_recorder_tests --host-import-lifetime-only) + add_test(NAME agc_driver_host_import_mutation_race COMMAND agc_driver_recorder_tests --host-import-mutation-race-only) + set_tests_properties(agc_driver_host_import_lifetime agc_driver_host_import_mutation_race PROPERTIES + ENVIRONMENT "APS5_PIN_WAIT_MS=2000" SKIP_RETURN_CODE 77 TIMEOUT 30) + add_test(NAME agc_driver_unit_shadow COMMAND agc_driver_recorder_tests --unit-shadow-only) + add_test(NAME agc_driver_unit_shadow_budget COMMAND agc_driver_recorder_tests --unit-shadow-only) + set_tests_properties(agc_driver_unit_shadow PROPERTIES + ENVIRONMENT "APS5_PIN_WAIT_MS=2000" SKIP_RETURN_CODE 77 TIMEOUT 30) + set_tests_properties(agc_driver_unit_shadow_budget PROPERTIES + ENVIRONMENT "APS5_PIN_WAIT_MS=2000;APS5_UNIT_SHADOW_MIB=8" SKIP_RETURN_CODE 77 TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_driver_host_import_lifetime agc_driver_host_import_mutation_race agc_driver_unit_shadow agc_driver_unit_shadow_budget PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>") + endif() + set_property(TEST agc_driver_host_import_lifetime APPEND PROPERTY ENVIRONMENT_MODIFICATION + "APS5_RELEASE_UNDER_LOCK=unset:" "APS5_RELEASE_ON_UNLOCK=unset:") endif() add_test_executable(agc_recorder_kept_bytes_tests EXCLUDE_FROM_ALL tests/RecorderKeptBytes.cpp) @@ -1635,6 +1652,8 @@ target_link_libraries(agc_driver_interpolation_modes_tests PRIVATE shader_recomp target_include_directories(agc_driver_prepared_unregistered_graphics_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>) +target_link_libraries(agc_driver_guest_memory_lifetime_tests PRIVATE libkernel) + target_include_directories(agc_driver_prepared_shaders_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES> ) @@ -1849,6 +1868,48 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_uint8x4_color_target_no_resident PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>") endif() + set(agcGuestMemoryLifetimeTests agc_driver_guest_memory_lifetime) + foreach(mode IN ITEMS copy direct locked heap heap_uncached) + set(testName agc_driver_guest_memory_lifetime_${mode}) + if(mode STREQUAL "heap_uncached") + add_test(NAME ${testName} COMMAND agc_driver_guest_memory_lifetime_tests heap) + else() + add_test(NAME ${testName} COMMAND agc_driver_guest_memory_lifetime_tests ${mode}) + endif() + list(APPEND agcGuestMemoryLifetimeTests ${testName}) + endforeach() + set_tests_properties(${agcGuestMemoryLifetimeTests} PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIN_WAIT_MS=1000;APS5_HEAP_CACHE_MIB=4" + SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + set_tests_properties(agc_driver_guest_memory_lifetime_heap_uncached PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIN_WAIT_MS=1000;APS5_HEAP_CACHE_MIB=0") + if(WIN32) + set_tests_properties(${agcGuestMemoryLifetimeTests} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>") + endif() + if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND Python3_Interpreter_FOUND) + add_library(agc_guest_memory_lifetime_fixture SHARED tests/GuestMemoryLifetimeFixture.cpp) + target_include_directories(agc_guest_memory_lifetime_fixture PRIVATE + ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include + ${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include) + target_link_libraries(agc_guest_memory_lifetime_fixture PRIVATE libSceAgcDriver SDL2-static libc) + add_dependencies(agc_guest_memory_lifetime_fixture relinker libkernel) + set_target_properties(agc_guest_memory_lifetime_fixture PROPERTIES + OUTPUT_NAME GuestMemoryFixture SUFFIX ".prx" CXX_EXTENSIONS OFF + BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH "$ORIGIN") + add_test(NAME agc_driver_guest_memory_lifetime_converted COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_guest_memory_lifetime.py + ${CMAKE_SOURCE_DIR}/core/relinker/relinker/tests + $<TARGET_FILE:relinker> + $<TARGET_FILE:agc_guest_memory_lifetime_fixture> + $<TARGET_FILE:libkernel> + $<TARGET_FILE:libc> + $<TARGET_FILE:libSceAgcDriver>) + set_tests_properties(agc_driver_guest_memory_lifetime_converted PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIN_WAIT_MS=1000" SKIP_RETURN_CODE 77 TIMEOUT 30) + endif() set_tests_properties(agc_driver_flat_store PROPERTIES RUN_SERIAL TRUE) set_target_properties(agc_driver_flat_apertures_tests PROPERTIES EXCLUDE_FROM_ALL OFF) foreach(flatAperturesPart RANGE 13) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index a44b59cb26..a836b30afa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -1036,7 +1036,30 @@ std::uint64_t evictableBytes(const HostImports& state, std::uint64_t begin, std: return bytes; } +struct ImportConstruction { + ImportConstruction(const Context& context, std::uint64_t base, std::uint64_t bytes) + : device(context.device), destroyBuffer(context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")), freeMemory(context.Function<PFN_vkFreeMemory>("vkFreeMemory")), entry{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0} {} + ImportConstruction(const ImportConstruction&) = delete; + ImportConstruction& operator=(const ImportConstruction&) = delete; + ~ImportConstruction() { + if (!armed) return; + if (entry.buffer != VK_NULL_HANDLE) destroyBuffer(device, entry.buffer, nullptr); + if (entry.memory != VK_NULL_HANDLE) freeMemory(device, entry.memory, nullptr); +#ifdef _WIN32 + GuestArena::GuestArenaUnmapAlias_nid_postfix(entry.alias); +#endif + } + VkDevice device; + PFN_vkDestroyBuffer destroyBuffer; + PFN_vkFreeMemory freeMemory; + HostImport entry; + bool armed = true; +}; + +void ensurePinWaiter(); + const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t address, const GuestAllocations::Lease& lease, std::uint64_t end = 0, bool evict = true) { + ensurePinWaiter(); const bool fresh = !importsStale(context, state); if (fresh && end != 0) { if (const auto* current = findImport(state, address, end)) return current; @@ -1113,7 +1136,8 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import it = next; } if (state.liveBytes + bytes > budget) retireLeastRecent(context, state, lease, state.liveBytes + bytes - budget); - HostImport entry{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0}; + ImportConstruction owner(context, base, bytes); + auto& entry = owner.entry; entry.range = run.ranges.front(); entry.ranges = std::move(run.ranges); #ifdef _WIN32 @@ -1208,9 +1232,6 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import #endif entry.unwatched = step == nullptr && !(entry.dmaBuf ? watchedDmaBuf : watchedHostPointer); if (step != nullptr) { -#ifdef _WIN32 - GuestArena::GuestArenaUnmapAlias_nid_postfix(entry.alias); -#endif failures.insert_or_assign(base, ImportFailure{base, bytes, GuestAllocations::Mapped(entry.ranges.begin(), entry.ranges.end())}); std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx failed at %s (%d); falling back to copies\n", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), step, static_cast<int>(result)); #ifdef _WIN32 @@ -1244,7 +1265,9 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import if (trace) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx ok (%zu live, %.1f MiB live, %.1f MiB ever)%s\n", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), state.imports.size() + 1, state.liveBytes / 1048576.0, importedBytes / 1048576.0, joined ? ", joined at bind" : ""); entry.joined = joined; entry.lastUse = ++state.useCounter; - return &state.imports.emplace(base, entry).first->second; + const auto inserted = state.imports.emplace(base, entry); + owner.armed = false; + return &inserted.first->second; } // The readable registered range containing [begin, end), or null. @@ -2078,7 +2101,7 @@ SnapshotStats& Snapshots() { return stats; } -bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) noexcept { +bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) { const auto start = std::chrono::steady_clock::now(); bool synced = false; bool drained = false; @@ -2098,19 +2121,82 @@ bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) noexcept state.stats.contentionMs += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count(); return true; } + if (Recorder::InCompletion()) throw std::runtime_error("guest allocation mutation cannot wait for an active GPU completion"); if (GuestMemory::GpuMutex().HeldByThisThread()) { - try { - auto* recorder = Recorder::Active(); + auto* recorder = Recorder::Active(); + if (recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } + if (recorder != nullptr) recorder->FlushDeferredReleases(); + const auto end = static_cast<std::uint64_t>(address) + bytes; + auto& imports = Imports(); + if (releaseParkedImports(imports, address, end)) retiredImport = true; + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + { + std::lock_guard importsLock(imports.mutex); + for (auto it = imports.imports.begin(); it != imports.imports.end();) { + const auto base = it->first; + if (base >= end || address >= base + it->second.bytes) { + ++it; + continue; + } + const auto next = std::next(it); + retireImport(imports.device, imports, it, lease, false); + it = next; + retiredImport = true; + } + } + if (retiredImport && recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } + if (retiredImport && recorder != nullptr) recorder->FlushDeferredReleases(); + } else { + std::lock_guard lock(GuestMemory::GpuMutex()); + auto& state = Leases(); + auto* recorder = Recorder::Active(); + const auto target = state.newestLeaseSerial; + if (recorder != nullptr && recorder == state.leaseRecorder && target > state.finishedLeaseSerial) { + Recorder::CountSync(4); + recorder->Submit(); + recorder->FinishUpTo(target); + state.finishedLeaseSerial = target; + synced = true; + } else if (recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } else { + std::this_thread::yield(); + } + if (recorder != nullptr) recorder->FlushDeferredReleases(); + + const auto end = static_cast<std::uint64_t>(address) + bytes; + auto& imports = Imports(); + if (releaseParkedImports(imports, address, end)) retiredImport = true; + bool overlapsImport = false; + { + std::lock_guard importsLock(imports.mutex); + for (const auto& [base, entry] : imports.imports) { + if (base < end && address < base + entry.bytes) { + overlapsImport = true; + break; + } + } + } + if (overlapsImport) { if (recorder != nullptr && !recorder->Idle()) { Recorder::CountSync(4); recorder->Sync(); synced = true; drained = true; } - if (recorder != nullptr) recorder->FlushDeferredReleases(); - const auto end = static_cast<std::uint64_t>(address) + bytes; - auto& imports = Imports(); - if (releaseParkedImports(imports, address, end)) retiredImport = true; const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); { std::lock_guard importsLock(imports.mutex); @@ -2132,79 +2218,6 @@ bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) noexcept synced = true; drained = true; } - if (retiredImport && recorder != nullptr) recorder->FlushDeferredReleases(); - } catch (const std::exception& error) { - std::fprintf(stderr, "[gpu] lease wait failed: %s\n", error.what()); - } - } else { - try { - std::lock_guard lock(GuestMemory::GpuMutex()); - auto& state = Leases(); - auto* recorder = Recorder::Active(); - const auto target = state.newestLeaseSerial; - if (recorder != nullptr && recorder == state.leaseRecorder && target > state.finishedLeaseSerial) { - // The open batch may be the target; FinishUpTo waits front to back up to it only. - Recorder::CountSync(4); - recorder->Submit(); - recorder->FinishUpTo(target); - state.finishedLeaseSerial = target; - synced = true; - } else if (recorder != nullptr && !recorder->Idle()) { - Recorder::CountSync(4); - recorder->Sync(); - synced = true; - drained = true; - } else { - // Nothing recorded holds the lease: the holder is a lease-holding object that has - // yet to record (a test's, or a build whose worker gave up the lock); give it time. - std::this_thread::yield(); - } - if (recorder != nullptr) recorder->FlushDeferredReleases(); - - const auto end = static_cast<std::uint64_t>(address) + bytes; - auto& imports = Imports(); - if (releaseParkedImports(imports, address, end)) retiredImport = true; - bool overlapsImport = false; - { - std::lock_guard importsLock(imports.mutex); - for (const auto& [base, entry] : imports.imports) { - if (base < end && address < base + entry.bytes) { - overlapsImport = true; - break; - } - } - } - if (overlapsImport) { - if (recorder != nullptr && !recorder->Idle()) { - Recorder::CountSync(4); - recorder->Sync(); - synced = true; - drained = true; - } - const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); - { - std::lock_guard importsLock(imports.mutex); - for (auto it = imports.imports.begin(); it != imports.imports.end();) { - const auto base = it->first; - if (base >= end || address >= base + it->second.bytes) { - ++it; - continue; - } - const auto next = std::next(it); - retireImport(imports.device, imports, it, lease, false); - it = next; - retiredImport = true; - } - } - if (retiredImport && recorder != nullptr && !recorder->Idle()) { - Recorder::CountSync(4); - recorder->Sync(); - synced = true; - drained = true; - } - } - } catch (const std::exception& error) { - std::fprintf(stderr, "[gpu] lease wait failed: %s\n", error.what()); } } auto& state = Leases(); diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestMemoryLifetimeFixture.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestMemoryLifetimeFixture.cpp new file mode 100644 index 0000000000..465575f5dd --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/GuestMemoryLifetimeFixture.cpp @@ -0,0 +1,82 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/tests/GuestMemoryLifetimeSupport.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include <array> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <memory> +#include <mutex> + +namespace { +std::unique_ptr<AgcDriver::VulkanDevice> device; +constexpr std::array<std::uint32_t, 4> Pattern{0x12345678u, 0x89abcdefu, 0x24681357u, 0xfedcba98u}; +} + +extern "C" int APS5_VABI Begin_nid_no_patch() { + try { + device = OpenVulkanTestDevice(); + if (!device || !GuestMemoryLifetimeHasHostImports(device->Device())) { + device.reset(); + std::fflush(stdout); + std::exit(VulkanTestSkipped); + } + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "converted guest Vulkan initialization failed: %s\n", error.what()); + std::exit(1); + } +} + +extern "C" int APS5_VABI Fill_nid_no_patch(void* pointer, int gpu) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(pointer); + if (!range.readable || !range.writable || range.gpu != (gpu != 0)) { + std::fputs("converted guest mapping does not match its requested CPU and GPU access\n", stderr); + return 1; + } + } + if (!device->FillBuffer(reinterpret_cast<std::uintptr_t>(pointer), 4096, Pattern)) return 1; + AgcDriver::Graphics::Context probe{}; + probe.device = device->Device(); + probe.hostImportAlignment = 1; + return !AgcDriver::Graphics::HostImportCovers(probe, reinterpret_cast<std::uintptr_t>(pointer), 65536); +} + +extern "C" int APS5_VABI Retired_nid_no_patch(const void* pointer) { + AgcDriver::Graphics::Context probe{}; + probe.device = device->Device(); + probe.hostImportAlignment = 1; + const bool imported = AgcDriver::Graphics::HostImportCovers(probe, reinterpret_cast<std::uintptr_t>(pointer), 65536); + if (imported) std::fputs("converted guest observed a stale Vulkan host import\n", stderr); + return imported; +} + +extern "C" int APS5_VABI Verify_nid_no_patch(const void* pointer, int zero) { + const auto* bytes = static_cast<const std::uint8_t*>(pointer); + for (std::size_t offset = 0; offset < 4096; ++offset) { + const auto expected = zero ? std::uint8_t{0} : reinterpret_cast<const std::uint8_t*>(Pattern.data())[offset % sizeof(Pattern)]; + if (bytes[offset] != expected) { + std::fprintf(stderr, "converted guest memory mismatch at byte %zu: expected %u, observed %u\n", + offset, unsigned(expected), unsigned(bytes[offset])); + return 1; + } + } + return 0; +} + +extern "C" [[noreturn]] void APS5_VABI Finish_nid_no_patch(int result) { + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + device->WaitIdle(); + } + device.reset(); + std::printf("converted guest memory/Vulkan lifetime result=%d\n", result); + std::fflush(stdout); + std::exit(result); +} diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestMemoryLifetimeSupport.hpp b/core/libs/prx/libSceAgcDriver/tests/GuestMemoryLifetimeSupport.hpp new file mode 100644 index 0000000000..863b8ca67f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/GuestMemoryLifetimeSupport.hpp @@ -0,0 +1,34 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_GUESTMEMORYLIFETIMESUPPORT_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_GUESTMEMORYLIFETIMESUPPORT_HPP + +#include <SDL_error.h> +#include <SDL_loadso.h> +#include <vulkan/vulkan.h> +#include <cstdio> +#include <cstdlib> +#include <memory> +#include <stdexcept> +#include <string> + +static inline bool GuestMemoryLifetimeHasHostImports(VkDevice device) { + if (std::getenv("APS5_NO_HOST_IMPORT") != nullptr) { + std::puts("skipped, host imports are disabled by APS5_NO_HOST_IMPORT"); + return false; + } +#ifdef _WIN32 + constexpr const char* libraryName = "vulkan-1.dll"; +#else + constexpr const char* libraryName = "libvulkan.so.1"; +#endif + const std::unique_ptr<void, decltype(&SDL_UnloadObject)> library(SDL_LoadObject(libraryName), &SDL_UnloadObject); + if (!library) throw std::runtime_error(std::string("cannot load Vulkan for the host import capability query: ") + SDL_GetError()); + const auto deviceProc = reinterpret_cast<PFN_vkGetDeviceProcAddr>(SDL_LoadFunction(library.get(), "vkGetDeviceProcAddr")); + if (deviceProc == nullptr) throw std::runtime_error(std::string("cannot resolve vkGetDeviceProcAddr for the host import capability query: ") + SDL_GetError()); + if (deviceProc(device, "vkGetMemoryHostPointerPropertiesEXT") == nullptr) { + std::puts("skipped, the device has no VK_EXT_external_memory_host"); + return false; + } + return true; +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/tests/HostImportLifetime.cpp b/core/libs/prx/libSceAgcDriver/tests/HostImportLifetime.cpp new file mode 100644 index 0000000000..c507b67de3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/HostImportLifetime.cpp @@ -0,0 +1,803 @@ +#include "HostImportLifetime.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include <algorithm> +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <future> +#include <cstdio> +#include <cstring> +#include <exception> +#include <map> +#include <memory> +#include <mutex> +#include <new> +#include <stdexcept> +#include <string> +#include <thread> +#include <vector> + +namespace { + +using namespace AgcDriver::Graphics; +using AgcDriver::GuestMemory::GpuMutex; + +struct ImportState { + VkBuffer buffer; + VkDeviceMemory memory; + std::uint64_t address; + std::size_t bytes; + std::atomic<bool> destroyed{false}; + std::atomic<bool> freed{false}; + std::atomic<unsigned> frees{0}; + std::atomic<bool> outOfOrder{false}; + std::atomic<bool> inaccessible{false}; + std::mutex mutex; + std::condition_variable wake; + bool blockFree = false; + bool freeing = false; + bool allowFree = false; + std::thread::id freeingThread; +}; + +struct Tracking { + PFN_vkGetDeviceProcAddr resolve; + VkDevice device; + std::mutex mutex; + std::map<VkDeviceMemory, std::shared_ptr<ImportState>> imports; + std::atomic<bool> failAddressLookup{false}; + std::atomic<bool> failQueueSubmit{false}; + std::atomic<unsigned> rejectedSubmissions{0}; + std::atomic<unsigned> acceptedSubmissions{0}; + std::vector<VkFence> rejectedFences; + std::shared_ptr<ImportState> lastAllocated; + + void CleanupRejectedFences() { + std::vector<VkFence> fences; + { + std::lock_guard lock(mutex); + fences.swap(rejectedFences); + } + const auto destroy = reinterpret_cast<PFN_vkDestroyFence>(resolve(device, "vkDestroyFence")); + for (const auto fence : fences) destroy(device, fence, nullptr); + } + + std::shared_ptr<ImportState> Add(const HostImport& import) { + auto state = std::make_shared<ImportState>(); + state->buffer = import.buffer; + state->memory = import.memory; + state->address = import.base; + state->bytes = import.bytes; + std::lock_guard lock(mutex); + imports[import.memory] = state; + return state; + } +}; + +Tracking* tracking = nullptr; + +VKAPI_ATTR void VKAPI_CALL destroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks* allocator) { + const auto original = reinterpret_cast<PFN_vkDestroyBuffer>(tracking->resolve(device, "vkDestroyBuffer")); + original(device, buffer, allocator); + std::lock_guard lock(tracking->mutex); + for (const auto& [memory, state] : tracking->imports) { + if (state->buffer == buffer && !state->freed.load()) state->destroyed.store(true); + } +} + +VKAPI_ATTR void VKAPI_CALL freeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks* allocator) { + std::shared_ptr<ImportState> state; + { + std::lock_guard lock(tracking->mutex); + const auto found = tracking->imports.find(memory); + if (found != tracking->imports.end()) state = found->second; + } + if (state != nullptr) { + if (!state->destroyed.load()) state->outOfOrder.store(true); + if (!RegisteredReadableCovers(state->address, state->bytes) || !AgcDriver::GuestMemory::Accessible(reinterpret_cast<const void*>(state->address), state->bytes, false)) state->inaccessible.store(true); + std::unique_lock lock(state->mutex); + if (state->blockFree) { + state->freeing = true; + state->freeingThread = std::this_thread::get_id(); + state->wake.notify_all(); + if (!state->wake.wait_for(lock, std::chrono::seconds(2), [&] { return state->allowFree; })) state->outOfOrder.store(true); + } + } + const auto original = reinterpret_cast<PFN_vkFreeMemory>(tracking->resolve(device, "vkFreeMemory")); + original(device, memory, allocator); + if (state != nullptr) { + if (!RegisteredReadableCovers(state->address, state->bytes) || !AgcDriver::GuestMemory::Accessible(reinterpret_cast<const void*>(state->address), state->bytes, false)) state->inaccessible.store(true); + state->frees.fetch_add(1); + state->freed.store(true); + } +} + +VKAPI_ATTR VkResult VKAPI_CALL allocateMemory(VkDevice device, const VkMemoryAllocateInfo* allocation, const VkAllocationCallbacks* allocator, VkDeviceMemory* memory) { + const auto original = reinterpret_cast<PFN_vkAllocateMemory>(tracking->resolve(device, "vkAllocateMemory")); + const auto result = original(device, allocation, allocator, memory); + if (result != VK_SUCCESS) return result; + for (auto* next = static_cast<const VkBaseInStructure*>(allocation->pNext); next != nullptr; next = next->pNext) { + if (next->sType != VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT) continue; + const auto* host = reinterpret_cast<const VkImportMemoryHostPointerInfoEXT*>(next); + auto state = std::make_shared<ImportState>(); + state->buffer = VK_NULL_HANDLE; + state->memory = *memory; + state->address = reinterpret_cast<std::uint64_t>(host->pHostPointer); + state->bytes = static_cast<std::size_t>(allocation->allocationSize); + std::lock_guard lock(tracking->mutex); + tracking->imports[*memory] = state; + tracking->lastAllocated = state; + } + return result; +} + +VKAPI_ATTR VkResult VKAPI_CALL bindBufferMemory(VkDevice device, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize offset) { + { + std::lock_guard lock(tracking->mutex); + if (const auto found = tracking->imports.find(memory); found != tracking->imports.end()) found->second->buffer = buffer; + } + const auto original = reinterpret_cast<PFN_vkBindBufferMemory>(tracking->resolve(device, "vkBindBufferMemory")); + return original(device, buffer, memory, offset); +} + +VKAPI_ATTR void VKAPI_CALL destroyFence(VkDevice device, VkFence fence, const VkAllocationCallbacks* allocator) { + const auto original = reinterpret_cast<PFN_vkDestroyFence>(tracking->resolve(device, "vkDestroyFence")); + original(device, fence, allocator); + std::lock_guard lock(tracking->mutex); + std::erase(tracking->rejectedFences, fence); +} + +VKAPI_ATTR VkResult VKAPI_CALL queueSubmit(VkQueue queue, std::uint32_t count, const VkSubmitInfo* submissions, VkFence fence) { + if (tracking->failQueueSubmit.exchange(false)) { + std::lock_guard lock(tracking->mutex); + tracking->rejectedFences.push_back(fence); + tracking->rejectedSubmissions.fetch_add(1); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + const auto original = reinterpret_cast<PFN_vkQueueSubmit>(tracking->resolve(tracking->device, "vkQueueSubmit")); + const auto result = original(queue, count, submissions, fence); + if (result == VK_SUCCESS) { + std::lock_guard lock(tracking->mutex); + std::erase(tracking->rejectedFences, fence); + tracking->acceptedSubmissions.fetch_add(1); + } + return result; +} + +PFN_vkVoidFunction VKAPI_CALL deviceProc(VkDevice device, const char* name) { + if (std::strcmp(name, "vkDestroyFence") == 0) return reinterpret_cast<PFN_vkVoidFunction>(destroyFence); + if (std::strcmp(name, "vkQueueSubmit") == 0) return reinterpret_cast<PFN_vkVoidFunction>(queueSubmit); + if (std::strcmp(name, "vkDestroyBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(destroyBuffer); + if (std::strcmp(name, "vkFreeMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(freeMemory); + if (std::strcmp(name, "vkAllocateMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(allocateMemory); + if (std::strcmp(name, "vkBindBufferMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(bindBufferMemory); + if (std::strcmp(name, "vkGetBufferDeviceAddressKHR") == 0 && tracking->failAddressLookup.exchange(false)) return nullptr; + return tracking->resolve(device, name); +} + +void requireFreed(const std::shared_ptr<ImportState>& state) { + Require(state->freed.load(), "guest mutation reached host apply before vkFreeMemory completed"); + Require(state->destroyed.load() && !state->outOfOrder.load() && state->frees.load() == 1, "host import Vulkan objects were destroyed out of order or more than once"); + Require(!state->inaccessible.load(), "guest pages became inaccessible before vkFreeMemory returned"); +} + +class Fixture { +public: + explicit Fixture(const Context& original) : context(original), calls{original.deviceProc, original.device} { + Require(tracking == nullptr, "host import tracking is already active"); + tracking = &calls; + context.deviceProc = deviceProc; + context.functions = nullptr; + std::lock_guard gpu(GpuMutex()); + recorder = std::make_unique<Recorder>(context); + recorder->Activate(); + } + + ~Fixture() { + std::lock_guard gpu(GpuMutex()); + recorder.reset(); + calls.CleanupRejectedFences(); + ClearHostImports(context.device); + context.bufferPool.reset(); + tracking = nullptr; + } + + std::shared_ptr<ImportState> Import(std::uint64_t address, std::size_t bytes) { + std::lock_guard gpu(GpuMutex()); + const auto* imported = HostImportFor(context, address, bytes); + Require(imported != nullptr, "host pointer import was refused for the lifetime test"); + return calls.Add(*imported); + } + + void Cleanup() { + std::lock_guard gpu(GpuMutex()); + recorder->Sync(); + ClearHostImports(context.device); + } + + Context context; + Tracking calls; + std::unique_ptr<Recorder> recorder; +}; + +class Allocation { +public: + Allocation(Fixture& fixture, std::size_t units = 1) : fixture(fixture) { + alignment = static_cast<std::size_t>(std::max<VkDeviceSize>(65536, fixture.context.hostImportAlignment)); + bytes = alignment * units; + data = ::operator new(bytes, std::align_val_t{alignment}); + std::memset(data, 0x37, bytes); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true, true); + } + + ~Allocation() { + try { + if (registered) { + fixture.Cleanup(); + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } + Release(); + } catch (const std::exception& error) { + std::fprintf(stderr, "host import test cleanup failed: %s\n", error.what()); + std::terminate(); + } + } + + std::uint64_t Address() const { return reinterpret_cast<std::uint64_t>(data); } + + void Release() { + if (released) return; + ::operator delete(data, std::align_val_t{alignment}); + released = true; + } + + Fixture& fixture; + void* data = nullptr; + std::size_t alignment = 0; + std::size_t bytes = 0; + bool registered = true; + bool released = false; +}; + +void descriptorFirstImport(Fixture& fixture) { + Require(fixture.calls.lastAllocated == nullptr, "descriptor-only lifetime test did not run before the first host import"); + Allocation allocation(fixture); + std::shared_ptr<ImportState> imported; + { + std::lock_guard gpu(GpuMutex()); + GuestBufferMemory memory(fixture.context); + memory.AddReadable(allocation.Address(), allocation.bytes); + memory.Upload(false); + imported = fixture.calls.lastAllocated; + Require(imported != nullptr && imported->address == allocation.Address(), "descriptor-only upload did not import its allocation"); + std::uint32_t adjustment = 0; + const auto descriptor = memory.Descriptor(allocation.Address(), allocation.bytes, adjustment); + Require(descriptor.buffer == imported->buffer && descriptor.offset == 0 && adjustment == 0, "descriptor-only upload did not bind the tracked import"); + } + Require(HostImportCovers(fixture.context, allocation.Address(), allocation.bytes), "descriptor-only import did not persist after its build ended"); + { + GuestAllocations::Mutation mutation; + mutation.Unmap(allocation.data, allocation.bytes, [&](const void*, std::size_t, const void*, bool) { + requireFreed(imported); + allocation.Release(); + }); + allocation.registered = false; + } + Require(!HostImportCovers(fixture.context, allocation.Address(), allocation.bytes), "descriptor-only import survived allocation removal"); +} + +class PendingRetirement { +public: + explicit PendingRetirement(Fixture& fixture) : fixture(fixture), first(fixture) { + std::lock_guard gpu(GpuMutex()); + build = std::make_unique<GuestBufferMemory>(fixture.context); + build->AcquireRegistered(); + build->UploadPrepare(true); + retry = std::make_unique<GuestBufferMemory>(fixture.context); + retry->AcquireRegistered(); + retry->UploadPrepare(true); + allocation = std::make_unique<Allocation>(fixture); + imported = fixture.Import(allocation->Address(), allocation->bytes); + result = std::make_unique<Buffer>(fixture.context, allocation->bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto commands = fixture.recorder->Commands(); + RecordMemoryBarrier(fixture.context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(fixture.context, commands, imported->buffer, 0, result->Handle(), 0, allocation->bytes); + RecordMemoryBarrier(fixture.context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + fixture.recorder->NotePendingRead(allocation->Address(), allocation->bytes, Recorder::ReadKind::AddressBased); + fixture.recorder->Submit(); + Check(fixture.context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle")(fixture.context.queue), "vkQueueWaitIdle retirement fixture"); + generation = std::make_unique<Allocation>(fixture); + Require(!fixture.recorder->Idle(), "retirement fixture has no submitted batch"); + } + + ~PendingRetirement() { + if (imported != nullptr) { + { + std::lock_guard lock(imported->mutex); + imported->allowFree = true; + } + imported->wake.notify_all(); + } + build.reset(); + retry.reset(); + fixture.Cleanup(); + } + + void VerifyCopy() { + result->Invalidate(); + const auto bytes = result->Bytes(); + Require(std::all_of(bytes.begin(), bytes.end(), [](std::byte value) { return value == std::byte{0x37}; }), "retirement lost the submitted import read"); + } + + Fixture& fixture; + Allocation first; + std::unique_ptr<Buffer> result; + std::unique_ptr<Allocation> allocation; + std::unique_ptr<Allocation> generation; + std::unique_ptr<GuestBufferMemory> build; + std::unique_ptr<GuestBufferMemory> retry; + std::shared_ptr<ImportState> imported; +}; + +void deferredRetirement(Fixture& fixture) { + PendingRetirement pending(fixture); + pending.imported->blockFree = true; + { + std::lock_guard gpu(GpuMutex()); + pending.build->UploadFinish(true); + Require(!HostImportCovers(fixture.context, pending.allocation->Address(), pending.allocation->bytes), "deferred retirement left its import cached"); + Require(!pending.imported->freed.load(), "retirement did not keep its import through the pending batch"); + fixture.recorder->Sync(); + fixture.recorder->FlushDeferredReleases(); + } + bool entered = false; + bool onReleaseThread = false; + { + std::unique_lock lock(pending.imported->mutex); + entered = pending.imported->wake.wait_for(lock, std::chrono::seconds(2), [&] { return pending.imported->freeing; }); + onReleaseThread = pending.imported->freeingThread != std::this_thread::get_id(); + } + std::atomic<bool> applied{false}; + const auto waitsBefore = LeaseCounters().contentionWaits; + std::promise<void> starting; + auto started = starting.get_future(); + auto mutation = std::async(std::launch::async, [&] { + starting.set_value(); + GuestAllocations::Mutation mutation; + mutation.Unmap(pending.allocation->data, pending.allocation->bytes, [&](const void*, std::size_t, const void*, bool) { + requireFreed(pending.imported); + applied.store(true); + pending.allocation->Release(); + }); + pending.allocation->registered = false; + }); + started.wait(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(1); + while (LeaseCounters().contentionWaits == waitsBefore && mutation.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout && std::chrono::steady_clock::now() < deadline) std::this_thread::yield(); + const bool reachedWaiter = LeaseCounters().contentionWaits > waitsBefore; + const bool blocked = mutation.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout; + const bool premature = applied.load(); + const bool registered = RegisteredReadableCovers(pending.allocation->Address(), pending.allocation->bytes); + { + std::lock_guard lock(pending.imported->mutex); + pending.imported->allowFree = true; + } + pending.imported->wake.notify_all(); + mutation.get(); + Require(entered && onReleaseThread, "retired import destruction did not run on the recorder release thread"); + Require(reachedWaiter && blocked && !premature && registered, "guest mutation passed a retired import whose Vulkan free was still running"); + Require(applied.load(), "guest mutation did not resume after deferred Vulkan free"); + requireFreed(pending.imported); + pending.VerifyCopy(); +} + +void idleUnmap(Fixture& fixture) { + Allocation allocation(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + { + GuestAllocations::Mutation mutation; + mutation.Unmap(allocation.data, allocation.bytes, [&](const void*, std::size_t, const void*, bool) { + requireFreed(imported); + allocation.Release(); + }); + allocation.registered = false; + } + Require(!HostImportCovers(fixture.context, allocation.Address(), allocation.bytes), "an unmapped range still has a live host import"); +} + +void pendingRead(Fixture& fixture, bool submitted, bool locked) { + Allocation allocation(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + Buffer result(fixture.context, allocation.bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + std::atomic<bool> completed{false}; + struct Finish { + Fixture& fixture; + ~Finish() { fixture.Cleanup(); } + } finish{fixture}; + std::unique_lock gpu(GpuMutex()); + const auto commands = fixture.recorder->Commands(); + RecordMemoryBarrier(fixture.context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(fixture.context, commands, imported->buffer, 0, result.Handle(), 0, allocation.bytes); + RecordMemoryBarrier(fixture.context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + fixture.recorder->NotePendingRead(allocation.Address(), allocation.bytes, Recorder::ReadKind::AddressBased); + fixture.recorder->Keep(std::make_shared<GuestAllocations::Lease>(GuestAllocations::GuestAllocationsAcquire_nid_postfix())); + fixture.recorder->OnComplete([&] { completed.store(true); }); + if (submitted) fixture.recorder->Submit(); + if (!locked) gpu.unlock(); + { + GuestAllocations::Mutation mutation; + mutation.Unmap(allocation.data, allocation.bytes, [&](const void*, std::size_t, const void*, bool) { + requireFreed(imported); + Require(completed.load(), "guest unmap passed a pending GPU read"); + result.Invalidate(); + const auto copied = result.Bytes(); + Require(std::all_of(copied.begin(), copied.end(), [](std::byte value) { return value == std::byte{0x37}; }), "GPU read lost the source contents before guest unmap"); + allocation.Release(); + }); + allocation.registered = false; + } +} + +void productionWaiterFailure(Fixture& fixture) { + Allocation allocation(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + Buffer result(fixture.context, allocation.bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + bool completed = false; + struct Finish { + Fixture& fixture; + ~Finish() { fixture.Cleanup(); } + } finish{fixture}; + const auto recordCopy = [&] { + const auto commands = fixture.recorder->Commands(); + RecordMemoryBarrier(fixture.context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(fixture.context, commands, imported->buffer, 0, result.Handle(), 0, allocation.bytes); + RecordMemoryBarrier(fixture.context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + fixture.recorder->NotePendingRead(allocation.Address(), allocation.bytes, Recorder::ReadKind::AddressBased); + fixture.recorder->Keep(std::make_shared<GuestAllocations::Lease>(GuestAllocations::GuestAllocationsAcquire_nid_postfix())); + }; + { + std::lock_guard gpu(GpuMutex()); + recordCopy(); + } + const auto rejectedBefore = fixture.calls.rejectedSubmissions.load(); + const auto acceptedBefore = fixture.calls.acceptedSubmissions.load(); + fixture.calls.failQueueSubmit.store(true); + bool applied = false; + std::string failure; + { + GuestAllocations::Mutation mutation; + try { + mutation.Protect(allocation.data, allocation.bytes, true, true, true, [&] { applied = true; }); + } catch (const std::runtime_error& error) { + failure = error.what(); + } + const auto range = mutation.Find(allocation.data); + Require(range.readable && range.writable && range.gpu, "failed production waiter changed the guest registration"); + } + Require(!fixture.calls.failQueueSubmit.load() && fixture.calls.rejectedSubmissions.load() == rejectedBefore + 1, "production pin waiter did not reach the injected queue submission failure"); + Require(fixture.calls.acceptedSubmissions.load() == acceptedBefore, "failed production waiter accepted GPU work before reporting failure"); + Require(failure == "AGC graphics: vkQueueSubmit recorder: Vulkan result -1" && !applied, "production pin waiter did not propagate the original Vulkan error before host apply"); + Require(!imported->destroyed.load() && !imported->freed.load(), "failed production waiter retired its host import"); + Require(HostImportCovers(fixture.context, allocation.Address(), allocation.bytes), "failed production waiter removed the live import"); + { + std::lock_guard gpu(GpuMutex()); + recordCopy(); + fixture.recorder->OnComplete([&] { completed = true; }); + } + { + GuestAllocations::Mutation mutation; + mutation.Unmap(allocation.data, allocation.bytes, [&](const void*, std::size_t, const void*, bool) { + requireFreed(imported); + Require(completed, "production waiter retry passed a pending GPU read"); + result.Invalidate(); + const auto copied = result.Bytes(); + Require(std::all_of(copied.begin(), copied.end(), [](std::byte value) { return value == std::byte{0x37}; }), "production waiter retry lost its real GPU copy"); + allocation.Release(); + }); + allocation.registered = false; + } + Require(fixture.calls.acceptedSubmissions.load() == acceptedBefore + 1, "production waiter retry did not submit exactly its new copy"); + requireFreed(imported); +} + +void partialUnmap(Fixture& fixture) { + Allocation allocation(fixture, 3); + Allocation unrelated(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + const auto other = fixture.Import(unrelated.Address(), unrelated.bytes); + auto* middle = static_cast<std::byte*>(allocation.data) + allocation.alignment; + { + GuestAllocations::Mutation mutation; + mutation.Unmap(middle, allocation.alignment, [&](const void* piece, std::size_t bytes, const void* original, bool last) { + requireFreed(imported); + Require(!other->freed.load(), "partial unmap retired a disjoint host import"); + Require(piece == middle && bytes == allocation.alignment && original == allocation.data && !last, "partial unmap changed the wrong guest extent"); + }); + } + Require(!RegisteredReadableCovers(reinterpret_cast<std::uint64_t>(middle), allocation.alignment), "the middle of a partial unmap remains registered"); + const auto low = fixture.Import(allocation.Address(), allocation.alignment); + const auto high = fixture.Import(allocation.Address() + 2 * allocation.alignment, allocation.alignment); + Require(!low->freed.load() && !high->freed.load() && !other->freed.load(), "remaining partial mappings could not be imported"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(allocation.data); + allocation.registered = false; + } + requireFreed(low); + requireFreed(high); + Require(!other->freed.load(), "removing a split allocation retired a disjoint import"); +} + +void protectRollback(Fixture& fixture) { + Allocation allocation(fixture); + auto imported = fixture.Import(allocation.Address(), allocation.bytes); + bool called = false; + try { + GuestAllocations::Mutation mutation; + mutation.Protect(allocation.data, allocation.bytes, true, false, true, [&] { + requireFreed(imported); + called = true; + throw std::runtime_error("expected host protection failure"); + }); + throw std::runtime_error("host protection failure was ignored"); + } catch (const std::runtime_error& error) { + Require(called && std::string(error.what()) == "expected host protection failure", "unexpected protection failure"); + } + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(allocation.data); + Require(range.readable && range.writable && range.gpu, "failed host protection changed the guest registration"); + } + imported = fixture.Import(allocation.Address(), allocation.bytes); + { + GuestAllocations::Mutation mutation; + mutation.Protect(allocation.data, allocation.bytes, true, false, true, [&] { requireFreed(imported); }); + } + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(allocation.data); + Require(range.readable && !range.writable && range.gpu, "successful protection did not update the guest registration"); + mutation.Protect(allocation.data, allocation.bytes, true, true, true, [] {}); + } + imported = fixture.Import(allocation.Address(), allocation.bytes); + { + GuestAllocations::Mutation mutation; + mutation.Remove(allocation.data); + allocation.registered = false; + } + requireFreed(imported); +} + +void blockedFree(Fixture& fixture) { + Allocation allocation(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + imported->blockFree = true; + std::atomic<bool> applied{false}; + auto mutation = std::async(std::launch::async, [&] { + GuestAllocations::Mutation mutation; + mutation.Unmap(allocation.data, allocation.bytes, [&](const void*, std::size_t, const void*, bool) { + requireFreed(imported); + applied.store(true); + allocation.Release(); + }); + allocation.registered = false; + }); + bool entered = false; + { + std::unique_lock lock(imported->mutex); + entered = imported->wake.wait_for(lock, std::chrono::seconds(2), [&] { return imported->freeing; }); + } + const bool premature = applied.load(); + const bool stillRegistered = RegisteredReadableCovers(allocation.Address(), allocation.bytes); + { + std::lock_guard lock(imported->mutex); + imported->allowFree = true; + } + imported->wake.notify_all(); + mutation.get(); + Require(entered, "guest unmap never reached Vulkan import destruction"); + Require(!premature && stillRegistered, "guest unmap released its registration while vkFreeMemory was blocked"); + Require(applied.load(), "guest unmap did not resume after vkFreeMemory returned"); +} + +void failedConstruction(Fixture& fixture) { + Allocation allocation(fixture); + fixture.calls.failAddressLookup.store(true); + bool failed = false; + try { + fixture.Import(allocation.Address(), allocation.bytes); + } catch (const std::runtime_error& error) { + failed = std::string(error.what()).find("vkGetBufferDeviceAddressKHR") != std::string::npos; + } + Require(failed && !fixture.calls.failAddressLookup.load(), "import construction did not reach the injected Vulkan resolver failure"); + const auto allocated = fixture.calls.lastAllocated; + Require(allocated != nullptr && allocated->address == allocation.Address(), "resolver failure happened before host memory was imported"); + if (!allocated->freed.load()) { + destroyBuffer(fixture.context.device, allocated->buffer, nullptr); + freeMemory(fixture.context.device, allocated->memory, nullptr); + throw std::runtime_error("failed host import construction leaked its Vulkan memory"); + } + requireFreed(allocated); + Require(!HostImportCovers(fixture.context, allocation.Address(), allocation.bytes), "failed host import construction remained cached"); + const auto retried = fixture.Import(allocation.Address(), allocation.bytes); + { + GuestAllocations::Mutation mutation; + mutation.Remove(allocation.data); + allocation.registered = false; + } + requireFreed(retried); + Require(allocated->frees.load() == 1, "retry destroyed the failed import a second time"); +} + +void mutationDuringCompletion(Fixture& fixture) { + Allocation allocation(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + Buffer result(fixture.context, allocation.bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + bool rejected = false; + bool applied = false; + bool laterCompletion = false; + bool copied = false; + std::lock_guard gpu(GpuMutex()); + const auto commands = fixture.recorder->Commands(); + RecordMemoryBarrier(fixture.context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(fixture.context, commands, imported->buffer, 0, result.Handle(), 0, allocation.bytes); + RecordMemoryBarrier(fixture.context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + fixture.recorder->OnComplete([&] { + try { + GuestAllocations::Mutation mutation; + mutation.Protect(allocation.data, allocation.bytes, false, false, false, [&] { applied = true; }); + } catch (const std::runtime_error& error) { + rejected = std::string(error.what()).find("active GPU completion") != std::string::npos; + } + }); + fixture.recorder->OnComplete([&] { + laterCompletion = true; + result.Invalidate(); + const auto bytes = result.Bytes(); + copied = std::all_of(bytes.begin(), bytes.end(), [](std::byte value) { return value == std::byte{0x37}; }); + }); + fixture.recorder->Sync(); + Require(rejected && !applied, "an active completion allowed guest memory mutation before later completions"); + Require(laterCompletion && copied, "rejecting a completion mutation lost later GPU results"); + Require(!imported->freed.load(), "rejecting a completion mutation retired its host import"); + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(allocation.data); + Require(range.readable && range.writable && range.gpu, "rejecting a completion mutation changed its guest range"); + mutation.Remove(allocation.data); + allocation.registered = false; + } + requireFreed(imported); +} + +void completedLeaseUnderLock(Fixture& fixture) { + Allocation allocation(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + std::lock_guard gpu(GpuMutex()); + fixture.recorder->Keep(std::make_shared<GuestAllocations::Lease>(GuestAllocations::GuestAllocationsAcquire_nid_postfix())); + fixture.recorder->Sync(); + Require(fixture.recorder->Idle(), "completed lease test recorder is not idle"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(allocation.data); + allocation.registered = false; + } + requireFreed(imported); +} + + +struct MutationRace { + Fixture* fixture; + Allocation* allocation; + std::shared_ptr<ImportState> replacement; + unsigned calls = 0; +}; + +MutationRace* mutationRace = nullptr; + +bool raceWaiter(std::uintptr_t address, std::size_t bytes) { + auto& race = *mutationRace; + Require(address == race.allocation->Address() && bytes == race.allocation->bytes, "pin waiter received the wrong mutation extent"); + std::lock_guard gpu(GpuMutex()); + race.fixture->Cleanup(); + if (++race.calls == 1) race.replacement = race.fixture->Import(address, bytes); + return true; +} + +bool failingWaiter(std::uintptr_t, std::size_t) { + throw std::runtime_error("expected pin waiter failure"); +} + +void waiterFailure(Fixture& fixture) { + Allocation allocation(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + GuestAllocations::GuestAllocationsSetPinWaiter_nid_postfix(&failingWaiter); + bool failed = false; + bool excluded = false; + std::future<void> reader; + { + GuestAllocations::Mutation mutation; + try { + mutation.Protect(allocation.data, allocation.bytes, false, false, false, [] { throw std::runtime_error("host apply ran after a pin waiter failure"); }); + } catch (const std::runtime_error& error) { + failed = std::string(error.what()) == "expected pin waiter failure"; + } + std::promise<void> starting; + auto ready = starting.get_future(); + reader = std::async(std::launch::async, [&] { + starting.set_value(); + static_cast<void>(GuestAllocations::GuestAllocationsAcquire_nid_postfix()); + }); + ready.wait(); + excluded = reader.wait_for(std::chrono::milliseconds(20)) == std::future_status::timeout; + const auto range = mutation.Find(allocation.data); + Require(range.readable && range.writable && range.gpu, "a pin waiter failure changed the registered permissions"); + } + reader.get(); + Require(failed, "a pin waiter error did not propagate"); + Require(excluded, "a failed pin waiter did not restore the registry lock"); + Require(!imported->freed.load(), "a failed pin waiter retired an import"); +} + +void reimportBeforeRelock(Fixture& fixture) { + Allocation allocation(fixture); + const auto imported = fixture.Import(allocation.Address(), allocation.bytes); + MutationRace race{&fixture, &allocation}; + mutationRace = ∽̱ + GuestAllocations::GuestAllocationsSetPinWaiter_nid_postfix(&raceWaiter); + { + GuestAllocations::Mutation mutation; + mutation.Unmap(allocation.data, allocation.bytes, [&](const void*, std::size_t, const void*, bool) { + Require(race.calls == 2 && race.replacement != nullptr, "guest mutation did not recheck an import acquired before registry relock"); + requireFreed(imported); + requireFreed(race.replacement); + allocation.Release(); + }); + allocation.registered = false; + } + mutationRace = nullptr; +} + +} + +int RunHostImportLifetimeTests(const Context& context) { + if (context.hostImportAlignment == 0) { + std::puts("skipped, the device has no VK_EXT_external_memory_host"); + return 77; + } + Fixture fixture(context); + descriptorFirstImport(fixture); + idleUnmap(fixture); + pendingRead(fixture, false, false); + pendingRead(fixture, true, false); + pendingRead(fixture, false, true); + pendingRead(fixture, true, true); + partialUnmap(fixture); + protectRollback(fixture); + completedLeaseUnderLock(fixture); + mutationDuringCompletion(fixture); + blockedFree(fixture); + failedConstruction(fixture); + productionWaiterFailure(fixture); + deferredRetirement(fixture); + std::puts("host import lifetime tests passed"); + return 0; +} + +int RunHostImportMutationRaceTests(const Context& context) { + if (context.hostImportAlignment == 0) { + std::puts("skipped, the device has no VK_EXT_external_memory_host"); + return 77; + } + Fixture fixture(context); + waiterFailure(fixture); + reimportBeforeRelock(fixture); + std::puts("host import mutation race tests passed"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/HostImportLifetime.hpp b/core/libs/prx/libSceAgcDriver/tests/HostImportLifetime.hpp new file mode 100644 index 0000000000..9e28ea9da5 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/HostImportLifetime.hpp @@ -0,0 +1,10 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_HOSTIMPORTLIFETIME_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_HOSTIMPORTLIFETIME_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" + +int RunHostImportLifetimeTests(const AgcDriver::Graphics::Context& context); + +int RunHostImportMutationRaceTests(const AgcDriver::Graphics::Context& context); + +#endif diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 64c2075bf5..b7adb1d449 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -29,6 +29,7 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "ResidentPresent.hpp" #include "ImageTableAbi.hpp" +#include "HostImportLifetime.hpp" #include "SampleLod_spv.h" #include "BufferUpdate_spv.h" #include "SampleArray_spv.h" @@ -8300,6 +8301,8 @@ int main(int argc, char** argv) { return 77; } Device& device = *created; + if (argc == 2 && std::string_view(argv[1]) == "--host-import-lifetime-only") return RunHostImportLifetimeTests(device.GetContext()); + if (argc == 2 && std::string_view(argv[1]) == "--host-import-mutation-race-only") return RunHostImportMutationRaceTests(device.GetContext()); if (argc == 2 && std::string_view(argv[1]) == "--import-watch-only") { importWatchTests(device); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GuestMemoryLifetime.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GuestMemoryLifetime.cpp new file mode 100644 index 0000000000..9e9e22d814 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GuestMemoryLifetime.cpp @@ -0,0 +1,259 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/tests/GuestMemoryLifetimeSupport.hpp" +#include <array> +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <cstring> +#include <memory> +#include <mutex> +#include <stdexcept> +#include <string_view> + +extern "C" { +void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept; +int APS5_VABI munmap_nid_postfix(void*, std::size_t) noexcept; +int APS5_VABI mprotect_nid_postfix(void*, std::size_t, int) noexcept; +void* APS5_VABI memalign_nid_postfix(std::size_t, std::size_t); +void* APS5_VABI realloc_nid_postfix(void*, std::size_t); +void APS5_VABI free_nid_postfix(void*); +int APS5_VABI sceKernelMapFlexibleMemory(void**, std::size_t, int, int); +int APS5_VABI sceKernelAllocateDirectMemory(std::int64_t, std::int64_t, std::size_t, std::size_t, int, std::int64_t*); +int APS5_VABI sceKernelMapDirectMemory(void**, std::size_t, int, int, std::int64_t, std::size_t); +int APS5_VABI sceKernelReleaseDirectMemory(std::int64_t, std::size_t); +} + +namespace { + +constexpr std::size_t BlockBytes = 65536; +constexpr std::size_t FillBytes = 4096; +constexpr std::array<std::uint32_t, 4> Pattern{0x12345678u, 0x89abcdefu, 0x24681357u, 0xfedcba98u}; + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +std::uint64_t Address(const void* pointer) { + return reinterpret_cast<std::uintptr_t>(pointer); +} + +bool Imported(const AgcDriver::VulkanDevice& device, const void* pointer, std::size_t bytes = BlockBytes) { + AgcDriver::Graphics::Context probe{}; + probe.device = device.Device(); + probe.hostImportAlignment = 1; + return AgcDriver::Graphics::HostImportCovers(probe, Address(pointer), bytes); +} + +void* Map(std::size_t bytes = BlockBytes) { + auto* result = mmap_nid_postfix(nullptr, bytes, 3, 0x1002, -1, 0); + Require(result != reinterpret_cast<void*>(~std::uintptr_t{0}), "guest mmap failed"); + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(result); + Require(range.readable && range.writable && !range.gpu, "guest mmap did not register CPU-only read/write access"); + } + std::memset(result, 0, bytes); + return result; +} + +void CheckPattern(const void* pointer) { + const auto* data = static_cast<const std::byte*>(pointer); + for (std::size_t offset = 0; offset < FillBytes; offset += sizeof(Pattern)) + Require(std::memcmp(data + offset, Pattern.data(), sizeof(Pattern)) == 0, "guest mutation lost the pending GPU fill"); +} + +void Fill(AgcDriver::VulkanDevice& device, void* pointer) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + Require(device.FillBuffer(Address(pointer), FillBytes, Pattern), "Vulkan device did not import the guest allocation"); + Require(Imported(device, pointer), "GPU fill did not create a host import"); +} + +void CheckMappings(AgcDriver::VulkanDevice& device) { + auto* mapping = Map(); + Fill(device, mapping); + Require(mprotect_nid_postfix(mapping, BlockBytes, 3) == 0, "guest writable protection refresh failed"); + Require(!Imported(device, mapping), "host import survived guest writable protection refresh"); + CheckPattern(mapping); + Fill(device, mapping); + Require(mprotect_nid_postfix(mapping, BlockBytes, 1) == 0, "guest read-only protection failed"); + Require(!Imported(device, mapping), "host import survived guest protection change"); + CheckPattern(mapping); + Require(mprotect_nid_postfix(mapping, BlockBytes, 0) == 0, "guest inaccessible protection failed"); + Require(mprotect_nid_postfix(mapping, BlockBytes, 3) == 0, "guest writable protection restore failed"); + CheckPattern(mapping); + Fill(device, mapping); + void* replaced = mapping; + Require(sceKernelMapFlexibleMemory(&replaced, BlockBytes, 0x33, 0x10) == 0 && replaced == mapping, "guest fixed mapping replacement failed"); + Require(!Imported(device, mapping), "host import survived guest fixed mapping replacement"); + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(mapping); + Require(range.readable && range.writable && range.gpu, "guest fixed mapping did not register its requested CPU and GPU access"); + } + const auto* data = static_cast<const std::uint8_t*>(mapping); + for (std::size_t offset = 0; offset < BlockBytes; ++offset) + Require(data[offset] == 0, "GPU work reached replacement guest pages"); + Fill(device, mapping); + Require(munmap_nid_postfix(mapping, BlockBytes) == 0, "guest munmap failed"); + Require(!Imported(device, mapping), "host import survived guest munmap"); + + mapping = Map(3 * BlockBytes); + Fill(device, mapping); + auto* middle = static_cast<std::uint8_t*>(mapping) + BlockBytes; + Require(mprotect_nid_postfix(middle, BlockBytes, 1) == 0, "guest partial protection failed"); + Require(!Imported(device, mapping), "whole host import survived partial protection change"); + CheckPattern(mapping); + Fill(device, mapping); + Require(munmap_nid_postfix(middle, BlockBytes) == 0, "guest middle unmap failed"); + Require(Imported(device, mapping), "unrelated prefix import was retired by middle unmap"); + Require(munmap_nid_postfix(mapping, BlockBytes) == 0, "guest prefix unmap failed"); + Require(!Imported(device, mapping), "prefix host import survived guest munmap"); + Require(munmap_nid_postfix(middle + BlockBytes, BlockBytes) == 0, "guest suffix unmap failed"); +} + +void CheckHeap(AgcDriver::VulkanDevice& device) { + const std::array<void*, 10> api{}; + ApplicationHeapRegister_nid_no_patch(api.data()); + auto* allocation = memalign_nid_postfix(BlockBytes, BlockBytes); + Require(allocation != nullptr, "guest aligned allocation failed"); + std::memset(allocation, 0, BlockBytes); + Fill(device, allocation); + auto* replacement = realloc_nid_postfix(allocation, 2 * BlockBytes); + Require(replacement != nullptr, "guest realloc failed"); + Require(!Imported(device, allocation), "host import survived guest realloc"); + CheckPattern(replacement); + free_nid_postfix(replacement); + for (int iteration = 0; iteration < 3; ++iteration) { + allocation = memalign_nid_postfix(BlockBytes, BlockBytes); + Require(allocation != nullptr, "guest heap reuse allocation failed"); + std::memset(allocation, 0, BlockBytes); + Fill(device, allocation); + free_nid_postfix(allocation); + Require(!Imported(device, allocation), "host import survived guest free or heap cache eviction"); + } +} + +void CheckLockedMutation(AgcDriver::VulkanDevice& device) { + auto* mapping = Map(); + { + std::lock_guard outer(AgcDriver::GuestMemory::GpuMutex()); + Fill(device, mapping); + Require(mprotect_nid_postfix(mapping, BlockBytes, 3) == 0, "writable protection refresh under the GPU lock did not finish"); + Require(!Imported(device, mapping), "locked protection refresh retained the import"); + CheckPattern(mapping); + Fill(device, mapping); + Require(mprotect_nid_postfix(mapping, BlockBytes, 1) == 0, "protection change under the GPU lock did not finish"); + Require(!Imported(device, mapping), "locked protection change retained the import"); + CheckPattern(mapping); + Require(munmap_nid_postfix(mapping, BlockBytes) == 0, "unmap under the GPU lock did not finish"); + } + + mapping = Map(); + { + std::lock_guard outer(AgcDriver::GuestMemory::GpuMutex()); + auto lease = std::make_shared<GuestAllocations::Lease>(GuestAllocations::GuestAllocationsAcquire_nid_postfix()); + auto* recorder = AgcDriver::Graphics::Recorder::Active(); + Require(recorder != nullptr, "Vulkan device has no active recorder"); + recorder->Keep(std::move(lease)); + Require(munmap_nid_postfix(mapping, BlockBytes) == 0, "completed batch kept its lease until the outer GPU lock was released"); + } +} + +void CheckCopy(AgcDriver::VulkanDevice& device) { + auto* source = Map(); + auto* destination = Map(); + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + Fill(device, source); + const auto copied = device.CopyBuffer(Address(destination), Address(source), FillBytes, 0, BlockBytes, 0, 0, 0, [](std::span<const std::byte>, std::uint64_t) {}); + Require(copied.path == 1, "guest source was not read by a Vulkan host-import transfer"); + Require(Imported(device, source) && Imported(device, destination), "GPU copy did not import both guest ranges"); + } + Require(mprotect_nid_postfix(source, BlockBytes, 3) == 0, "copied source protection refresh failed"); + Require(!Imported(device, source), "GPU copy source remained imported after guest protection refresh"); + CheckPattern(destination); + Require(Imported(device, destination), "source retirement unnecessarily retired the copy destination"); + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + Fill(device, source); + const auto copied = device.CopyBuffer(Address(destination), Address(source), FillBytes, 0, BlockBytes, 0, 0, 0, [](std::span<const std::byte>, std::uint64_t) {}); + Require(copied.path == 1, "second source transfer did not use host imports"); + } + Require(munmap_nid_postfix(source, BlockBytes) == 0, "copied source unmap failed"); + Require(!Imported(device, source), "GPU copy source remained imported after guest unmap"); + CheckPattern(destination); + Require(Imported(device, destination), "source unmap retired the unrelated copy destination"); + Require(munmap_nid_postfix(destination, BlockBytes) == 0, "copy destination unmap failed"); + Require(!Imported(device, destination), "GPU copy destination remained imported after guest unmap"); +} + +void CheckDirectRelease(AgcDriver::VulkanDevice& device) { + std::int64_t physical = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, BlockBytes, BlockBytes, 0, &physical) == 0, "guest direct allocation failed"); + void* first = nullptr; + void* second = nullptr; + Require(sceKernelMapDirectMemory(&first, BlockBytes, 0x33, 0, physical, BlockBytes) == 0, "first direct view mapping failed"); + Require(sceKernelMapDirectMemory(&second, BlockBytes, 0x33, 0, physical, BlockBytes) == 0 && second != first, "second direct view mapping failed"); + { + GuestAllocations::Mutation mutation; + const auto firstRange = mutation.Find(first); + const auto secondRange = mutation.Find(second); + Require(firstRange.readable && firstRange.writable && firstRange.gpu && secondRange.readable && secondRange.writable && secondRange.gpu, "direct views did not register their requested CPU and GPU access"); + } + std::memset(first, 0, BlockBytes); + static_cast<std::uint8_t*>(first)[BlockBytes - 1] = 0x5a; + Require(static_cast<const std::uint8_t*>(second)[BlockBytes - 1] == 0x5a, "direct views do not share their physical backing"); + auto* destination = Map(); + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + Fill(device, first); + Require(device.FillBuffer(Address(second) + FillBytes, FillBytes, Pattern), "second direct view was not imported for its GPU fill"); + const auto firstCopy = device.CopyBuffer(Address(destination), Address(first), FillBytes, 0, BlockBytes, 0, 0, 0, [](std::span<const std::byte>, std::uint64_t) {}); + const auto secondCopy = device.CopyBuffer(Address(destination) + FillBytes, Address(second) + FillBytes, FillBytes, 0, BlockBytes, 0, 0, 0, [](std::span<const std::byte>, std::uint64_t) {}); + Require(firstCopy.path == 1 && secondCopy.path == 1, "direct views were not read through GPU import transfers"); + Require(Imported(device, first) && Imported(device, second), "both direct view imports were not retained before release"); + } + Require(sceKernelReleaseDirectMemory(physical, BlockBytes) == 0, "guest direct release failed"); + Require(!Imported(device, first) && !Imported(device, second), "direct release retained an imported virtual view"); + Require(!AgcDriver::Graphics::RegisteredReadableCovers(Address(first), BlockBytes) && !AgcDriver::Graphics::RegisteredReadableCovers(Address(second), BlockBytes), "direct release left a virtual view registered"); + CheckPattern(destination); + CheckPattern(static_cast<const std::byte*>(destination) + FillBytes); + Require(Imported(device, destination), "direct release retired the unrelated copy destination"); + Require(munmap_nid_postfix(destination, BlockBytes) == 0, "direct release copy destination unmap failed"); + Require(!Imported(device, destination), "direct release copy destination remained imported after unmap"); +} + +} + +int main(int argc, char** argv) { + try { + Require(argc <= 2, "expected mappings, heap, locked, copy or direct test mode"); + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!GuestMemoryLifetimeHasHostImports(device->Device())) return VulkanTestSkipped; + const std::string_view mode = argc == 2 ? argv[1] : "mappings"; + if (mode == "mappings") CheckMappings(*device); + else if (mode == "heap") CheckHeap(*device); + else if (mode == "locked") CheckLockedMutation(*device); + else if (mode == "copy") CheckCopy(*device); + else if (mode == "direct") CheckDirectRelease(*device); + else throw std::runtime_error("unknown guest memory lifetime test mode"); + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + device->WaitIdle(); + } + device.reset(); + std::puts("guest memory host import lifetime passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "guest memory host import lifetime: %s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/test_guest_memory_lifetime.py b/core/libs/prx/libSceAgcDriver/tests/test_guest_memory_lifetime.py new file mode 100644 index 0000000000..e5cc5d7467 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/test_guest_memory_lifetime.py @@ -0,0 +1,143 @@ +from pathlib import Path +import struct +import subprocess +import sys +import tempfile + +sys.path.insert(0, sys.argv[1]) +from test_guest_symbol_names import importing_executable, string_table, DYNAMIC, STRINGS, SYMBOLS, RELOCATIONS, GOT + +NAMES = ['Begin_nid_no_patch', 'mmap_nid_postfix', 'Fill_nid_no_patch', 'mprotect_nid_postfix', + 'Retired_nid_no_patch', 'Verify_nid_no_patch', 'sceKernelMapFlexibleMemory', + 'munmap_nid_postfix', 'Finish_nid_no_patch'] + + +def fixture(): + image = importing_executable(NAMES) + struct.pack_into('<I', image, 64 + 4, 7) + libraries = ['libGuestMemoryFixture.prx', 'libkernel.prx', 'libc.prx'] + strings, offsets = string_table(NAMES + libraries) + assert len(strings) < SYMBOLS - STRINGS + image[STRINGS:STRINGS + len(strings)] = strings + tags = [(5, STRINGS), (10, len(strings)), (6, SYMBOLS), (11, 24), + (7, RELOCATIONS), (8, len(NAMES) * 24), (9, 24)] + tags.extend((1, offset) for offset in offsets[len(NAMES):]) + tags.append((0, 0)) + for index, tag in enumerate(tags): + struct.pack_into('<qQ', image, DYNAMIC + index * 16, *tag) + struct.pack_into('<QQQQQ', image, 120 + 8, DYNAMIC, DYNAMIC, DYNAMIC, len(tags) * 16, len(tags) * 16) + code = bytearray(bytes.fromhex('4883ec18')) + checks = [] + + def emit(value): + code.extend(bytes.fromhex(value)) + + def call(name): + address = 0x4000 + len(code) + code.extend(b'\xff\x15' + struct.pack('<i', GOT + NAMES.index(name) * 8 - address - 6)) + + def check(): + emit('85c00f85') + checks.append(len(code)) + code.extend(bytes(4)) + + def pointer(): + emit('4c89e7') + + def fill(gpu): + pointer() + emit('be') + code.extend(struct.pack('<I', gpu)) + call('Fill_nid_no_patch') + check() + + def retired(): + pointer() + call('Retired_nid_no_patch') + check() + + def protect(value): + pointer() + emit('be00000100ba') + code.extend(struct.pack('<I', value)) + call('mprotect_nid_postfix') + check() + + def verify(zero): + pointer() + emit('be') + code.extend(struct.pack('<I', zero)) + call('Verify_nid_no_patch') + check() + + call('Begin_nid_no_patch') + check() + emit('31ffbe00000100ba03000000b90210000041b8ffffffff4531c9') + call('mmap_nid_postfix') + emit('4989c44883f8ff0f84') + checks.append(len(code)) + code.extend(bytes(4)) + fill(0) + protect(3) + retired() + verify(0) + fill(0) + protect(1) + retired() + verify(0) + protect(0) + protect(3) + verify(0) + fill(0) + emit('4c8924244889e7be00000100ba33000000b910000000') + call('sceKernelMapFlexibleMemory') + check() + retired() + verify(1) + fill(1) + pointer() + emit('be00000100') + call('munmap_nid_postfix') + check() + retired() + emit('31ff') + call('Finish_nid_no_patch') + emit('0f0b') + failed = len(code) + emit('bf01000000') + call('Finish_nid_no_patch') + emit('0f0b') + for position in checks: + struct.pack_into('<i', code, position, failed - position - 4) + assert len(code) < 0x600 + image[0x4000:0x4000 + len(code)] = code + return image + + +def main(): + relinker, fixture_library, kernel, libc, driver = (Path(value).resolve() for value in sys.argv[2:]) + with tempfile.TemporaryDirectory(prefix="anyps5-guest-memory-lifetime-") as directory: + work = Path(directory) + libraries = work / "libs" + libraries.mkdir() + for path in (fixture_library, kernel, libc, driver): + (libraries / path.name).symlink_to(path) + source = work / "guest.elf" + source.write_bytes(fixture()) + output = work / "native.elf" + converted = subprocess.run( + [str(relinker), "--skip-sce-module", "--registry", str(source), str(output)], + cwd=work, capture_output=True, text=True, timeout=10) + assert converted.returncode == 0, (converted.stdout, converted.stderr) + output.chmod(0o755) + result = subprocess.run([str(output)], cwd=work, capture_output=True, text=True, timeout=15) + if result.returncode == 77: + print(result.stdout, end="") + sys.exit(77) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + assert "converted guest memory/Vulkan lifetime result=0" in result.stdout, result.stdout + print("Converted guest memory/Vulkan lifetime tests passed") + + +if __name__ == "__main__": + main() diff --git a/core/libs/prx/libc/src/GuestAllocations.cpp b/core/libs/prx/libc/src/GuestAllocations.cpp index 438a819c8f..a931983d75 100644 --- a/core/libs/prx/libc/src/GuestAllocations.cpp +++ b/core/libs/prx/libc/src/GuestAllocations.cpp @@ -385,8 +385,13 @@ void GuestAllocationsRequireUnpinned_nid_postfix(void* mutation, const void* poi bool progressed = false; if (state != nullptr && state->lock.owns_lock()) { state->lock.unlock(); - if (waiter != nullptr) progressed = waiter(address, bytes); - else std::this_thread::yield(); + try { + if (waiter != nullptr) progressed = waiter(address, bytes); + else std::this_thread::yield(); + } catch (...) { + state->lock.lock(); + throw; + } state->lock.lock(); } else { std::this_thread::yield(); From 66247cef5cebd87468ba22090261954024cdacb1 Mon Sep 17 00:00:00 2001 From: T3cm0 <srodriguez7604@gmail.com> Date: Sat, 17 Oct 2026 20:40:58 -0500 Subject: [PATCH 094/103] feat(libc): implement the sceLibcPafMspace allocation exports Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libc/src/Mspace.cpp | 42 ++++++++++++++++++++++++++++++ core/libs/tests/Mspace.cpp | 43 +++++++++++++++++++++++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 86 insertions(+) diff --git a/core/libs/prx/libc/src/Mspace.cpp b/core/libs/prx/libc/src/Mspace.cpp index dbd74b1203..5292c9fe89 100644 --- a/core/libs/prx/libc/src/Mspace.cpp +++ b/core/libs/prx/libc/src/Mspace.cpp @@ -9,6 +9,7 @@ #include <memory> #include <mutex> #include <set> +#include <stdexcept> #include <utility> extern "C" int* APS5_VABI __error_nid_postfix(); @@ -309,4 +310,45 @@ std::size_t APS5_VABI sceLibcMspaceMallocUsableSize_nid_postfix(const void* poin return 0; } +void* APS5_VABI sceLibcPafMspaceCreate_nid_postfix(const char* name, void* base, std::size_t size, unsigned flags, const void* options) { + if (options) throw std::runtime_error("sceLibcPafMspaceCreate: creation options are not supported"); + return sceLibcMspaceCreate_nid_postfix(name, base, size, flags); +} + +int APS5_VABI sceLibcPafMspaceDestroy_nid_postfix(void* handle) { + return sceLibcMspaceDestroy_nid_postfix(handle); +} + +void* APS5_VABI sceLibcPafMspaceMalloc_nid_postfix(void* handle, std::size_t size, const void* owner) { + (void)owner; + return sceLibcMspaceMalloc_nid_postfix(handle, size); +} + +void APS5_VABI sceLibcPafMspaceFree_nid_postfix(void* handle, void* pointer, const void* owner) { + (void)owner; + sceLibcMspaceFree_nid_postfix(handle, pointer); +} + +void* APS5_VABI sceLibcPafMspaceCalloc_nid_postfix(void* handle, std::size_t count, std::size_t size, const void* owner) { + (void)owner; + return sceLibcMspaceCalloc_nid_postfix(handle, count, size); +} + +void* APS5_VABI sceLibcPafMspaceRealloc_nid_postfix(void* handle, void* pointer, std::size_t size, const void* owner) { + (void)owner; + return sceLibcMspaceRealloc_nid_postfix(handle, pointer, size); +} + +void* APS5_VABI sceLibcPafMspaceReallocalign_nid_postfix(void* handle, void* pointer, std::size_t alignment, std::size_t size, const void* owner) { + (void)owner; + return sceLibcMspaceReallocalign_nid_postfix(handle, pointer, alignment, size); +} + +int APS5_VABI sceLibcPafMspaceMallocStats_nid_postfix(void* handle, MallocManagedSize* stats) { + return sceLibcMspaceMallocStats_nid_postfix(handle, stats); +} + +int APS5_VABI sceLibcPafMspaceMallocStatsFast_nid_postfix(void* handle, MallocManagedSize* stats) { + return sceLibcMspaceMallocStatsFast_nid_postfix(handle, stats); +} } diff --git a/core/libs/tests/Mspace.cpp b/core/libs/tests/Mspace.cpp index 8a3571292f..2224e2b7a1 100644 --- a/core/libs/tests/Mspace.cpp +++ b/core/libs/tests/Mspace.cpp @@ -5,6 +5,7 @@ #include <cstdlib> #include <cstring> #include <limits> +#include <stdexcept> #include <thread> extern "C" { @@ -22,6 +23,15 @@ void* APS5_VABI sceLibcMspaceAlignedAlloc_nid_postfix(void*, std::size_t, std::s void* APS5_VABI sceLibcMspaceReallocalign_nid_postfix(void*, void*, std::size_t, std::size_t); int APS5_VABI sceLibcMspaceMallocStats_nid_postfix(void*, void*); int APS5_VABI sceLibcMspaceMallocStatsFast_nid_postfix(void*, void*); +void* APS5_VABI sceLibcPafMspaceCreate_nid_postfix(const char*, void*, std::size_t, unsigned, const void*); +int APS5_VABI sceLibcPafMspaceDestroy_nid_postfix(void*); +void* APS5_VABI sceLibcPafMspaceMalloc_nid_postfix(void*, std::size_t, const void*); +void APS5_VABI sceLibcPafMspaceFree_nid_postfix(void*, void*, const void*); +void* APS5_VABI sceLibcPafMspaceCalloc_nid_postfix(void*, std::size_t, std::size_t, const void*); +void* APS5_VABI sceLibcPafMspaceRealloc_nid_postfix(void*, void*, std::size_t, const void*); +void* APS5_VABI sceLibcPafMspaceReallocalign_nid_postfix(void*, void*, std::size_t, std::size_t, const void*); +int APS5_VABI sceLibcPafMspaceMallocStats_nid_postfix(void*, void*); +int APS5_VABI sceLibcPafMspaceMallocStatsFast_nid_postfix(void*, void*); } struct MallocManagedSize { std::uint16_t size; @@ -35,6 +45,38 @@ struct MallocManagedSize { static void Require(bool value) { if (!value) { std::fputs("Mspace check failed\n", stderr); std::abort(); } } +static void CheckPaf() { + alignas(64) static std::array<unsigned char, 65536> storage{}; + bool rejected = false; + int options = 0; + try { sceLibcPafMspaceCreate_nid_postfix("paf", storage.data(), storage.size(), 0, &options); } + catch (const std::runtime_error&) { rejected = true; } + Require(rejected); + void* arena = sceLibcPafMspaceCreate_nid_postfix("paf", storage.data(), storage.size(), 0, nullptr); + Require(arena == storage.data()); + const int owner = 0; + auto* block = static_cast<unsigned char*>(sceLibcPafMspaceMalloc_nid_postfix(arena, 40, &owner)); + Require(block != nullptr && sceLibcMspaceMallocUsableSize_nid_postfix(block) >= 40); + std::memset(block, 0x5a, 40); + auto* zeroed = static_cast<unsigned char*>(sceLibcPafMspaceCalloc_nid_postfix(arena, 4, 8, arena)); + Require(zeroed != nullptr && zeroed[0] == 0 && zeroed[31] == 0); + block = static_cast<unsigned char*>(sceLibcPafMspaceRealloc_nid_postfix(arena, block, 4000, &owner)); + Require(block != nullptr && block[0] == 0x5a && block[39] == 0x5a); + block = static_cast<unsigned char*>(sceLibcPafMspaceReallocalign_nid_postfix(arena, block, 256, 64, &owner)); + Require(block != nullptr && reinterpret_cast<std::uintptr_t>(block) % 256 == 0 && block[39] == 0x5a); + MallocManagedSize viaPaf{}, viaMspace{}, fastPaf{}, fastMspace{}; + viaPaf.size = viaMspace.size = fastPaf.size = fastMspace.size = sizeof(MallocManagedSize); + viaPaf.version = viaMspace.version = fastPaf.version = fastMspace.version = 1; + Require(sceLibcPafMspaceMallocStats_nid_postfix(arena, &viaPaf) == sceLibcMspaceMallocStats_nid_postfix(arena, &viaMspace)); + Require(sceLibcPafMspaceMallocStatsFast_nid_postfix(arena, &fastPaf) == sceLibcMspaceMallocStatsFast_nid_postfix(arena, &fastMspace)); + Require(viaPaf.currentInuseSize == viaMspace.currentInuseSize && viaPaf.currentInuseSize > 0); + Require(fastPaf.currentInuseSize == fastMspace.currentInuseSize); + sceLibcPafMspaceFree_nid_postfix(arena, block, &owner); + sceLibcPafMspaceFree_nid_postfix(arena, zeroed, arena); + sceLibcPafMspaceFree_nid_postfix(arena, nullptr, &owner); + Require(sceLibcPafMspaceDestroy_nid_postfix(arena) == 0); + Require(sceLibcPafMspaceDestroy_nid_postfix(arena) == -1); +} static void CheckAlignment() { alignas(32) static std::array<unsigned char, 65536 + 16> storage{}; void* arena = sceLibcMspaceCreate_nid_postfix("aligned", storage.data() + 16, 65536, 0); @@ -107,6 +149,7 @@ static void CheckMovedBlockKeepsUsableBytes() { Require(sceLibcMspaceDestroy_nid_postfix(arena) == 0); } int main() { + CheckPaf(); CheckAlignment(); CheckAdjacentBlocks(); CheckMovedBlockKeepsUsableBytes(); diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 497130f640..b077f88597 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -15,6 +15,7 @@ - [Guest arena](../../../core/libs/prx/libc/src/GuestArena.cpp) (libc) - the application map area is assumed to end at 0xFC_0000_0000, as `SCE_KERNEL_APP_MAP_AREA_END_ADDR` does on the PS4. On Windows, host allocations made in the arena before libc loads stay, and a fixed guest mapping over one throws - [sceLibcInternalBacktraceForGame](../../../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - `(const char* heapName)` and its return follow the `libc.prx` ASTRO BOT (PPSA21564 01.007.000) bundles: its four calls (0x6801, 0x7F46, 0xE75E, 0xEE02) are in the `malloc_stats` and `sceLibcMspaceMallocStats` paths, each after a `[%s] ...` line printed with the same heap name buffer, and the caller ignores the result and continues. The console's backtrace output is not reproduced: the heap name and the guest return address are written to the host standard error - [sceLibcInternalHeapErrorReportForGame](../../../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - `(void* heap, void* block, uint32 error)` follows the same `libc.prx` (call at 0xF634): it passes the heap, the block and the nonzero result of its block check, and then calls a function that never returns, so the report only has to return. The meaning of the error values is unknown; the three values and the guest return address are written to the host standard error +- [sceLibcPafMspaceCreate, Destroy, Malloc, Free, Calloc, Realloc, Reallocalign, MallocStats and MallocStatsFast](../../../core/libs/prx/libc/src/Mspace.cpp) (libSceLibcInternal, implemented in libc) follow the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x5D680 to 0x5D770): each calls the same internal routine as its `sceLibcMspace` counterpart without the malloc callback hook those add, and `Reallocalign` takes the alignment before the size as `sceLibcMspaceReallocalign` does. `Create` takes a fifth argument, a pointer to a creation options block (a version word, a type of 1 or 2 at offset 2, a field at 0x10) that the `Mspace` form passes as null; a non-null block throws, as its meaning is unknown. `Malloc`, `Free`, `Calloc`, `Realloc` and `Reallocalign` take a trailing owner value that the console XORs with its block magic and stores in each block's footer, where the `Mspace` forms pass the heap itself. Footers are not modelled here, so the owner is ignored. Measured on a PS5 (firmware 13.60) from a homebrew payload, the calls with a matching owner behave as here, but the console ends the process when `Reallocalign` is given a block whose owner differs and when it resizes a block in place with an owner other than the heap, and `MallocStats` returns 1 instead of 0 while the heap holds blocks owned by something other than the heap; none of this is reproduced. `PafMspaceMemalign`, `PosixMemalign`, `MallocUsableSize`, `Trim`, `IsHeapEmpty`, `CheckMemoryBounds`, `ReportMemoryBlocks` and `GetFooterValue` are not implemented - [malloc_usable_size](../../../core/libs/prx/libc/src/ApplicationHeap.cpp) (libc) - with the default allocator it returns the size the block was requested with (the range registered for it), not the rounded-up chunk size Sony's allocator reports; both satisfy "at least the requested size", and bytes past the request would not be write-watched. With an application allocator it calls entry 9 of the replacement table (`malloc_usable_size` in KytyPS5's `MallocReplace` order) and throws when the table leaves it null, like `malloc_stats_fast` - [malloc, calloc, realloc, memalign, aligned_alloc and reallocalign](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) - follow FreeBSD 11.4 `contrib/jemalloc/src/jemalloc.c` for failures, not measured on the console: a size the default heap rejects or the guest arena cannot place (including a `calloc` product that overflows) returns null with `errno` `ENOMEM`, and `memalign` and `aligned_alloc` return null with `EINVAL` for an alignment that is zero or not a power of two. FreeBSD's libc has no `memalign`, so setting `errno` there is assumed. A host commit failure for a block that fits in the arena (Windows `ERROR_NOT_ENOUGH_MEMORY`, `ERROR_OUTOFMEMORY` or `ERROR_COMMITMENT_LIMIT`, Linux `ENOMEM`) resets and releases the reserved range and returns null with `ENOMEM`; any other commit failure releases the range and throws. `reallocalign` returns null with `EINVAL` for an invalid alignment and `ENOMEM` like the others, which is an assumption as it is not in FreeBSD. `realloc(ptr, 0)` frees the block and returns null, where FreeBSD 11.4's jemalloc 4.5 resizes it to one byte and returns a block - [reallocf](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libSceLibcInternal, implemented in libc) - follows FreeBSD 11 `lib/libc/stdlib/reallocf.c` on top of this libc's `realloc`, not measured on the console: size 0 frees the block and returns null as `realloc` does here, so the block is not freed twice; when `realloc` fails because the application allocator returns null or the default heap rejects the size, the block is freed through the same allocator, `errno` is `ENOMEM` and null is returned. Other failures (a commit failure that is not out of memory, an unknown pointer) throw, as they do for `realloc` From f9b7b230fc5b7576eff55eeb355dfe83b6463a69 Mon Sep 17 00:00:00 2001 From: T3cm0 <srodriguez7604@gmail.com> Date: Sat, 17 Oct 2026 20:41:26 -0500 Subject: [PATCH 095/103] feat(libSceLibcInternal): implement the remaining posix_spawnattr accessors Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libSceLibcInternal/Spawn.cpp | 66 ++++++++++++++++++++++ core/libs/tests/GuestPosixSpawn.cpp | 33 +++++++++++ docs/dev/debt/libSceLibcInternal.md | 1 + 3 files changed, 100 insertions(+) diff --git a/core/libs/prx/libSceLibcInternal/Spawn.cpp b/core/libs/prx/libSceLibcInternal/Spawn.cpp index c4571fb64e..c81a8fd0cb 100644 --- a/core/libs/prx/libSceLibcInternal/Spawn.cpp +++ b/core/libs/prx/libSceLibcInternal/Spawn.cpp @@ -60,6 +60,72 @@ int APS5_VABI posix_spawnattr_setsigdefault_nid_postfix(PosixSpawnAttributes** a return 0; } +int APS5_VABI posix_spawnattr_getflags_nid_postfix(PosixSpawnAttributes* const* attributes, std::int16_t* flags) { + const auto& source = Attributes(attributes, "posix_spawnattr_getflags"); + if (flags == nullptr) throw std::invalid_argument("posix_spawnattr_getflags: null flags"); + *flags = source.flags; + return 0; +} + +int APS5_VABI posix_spawnattr_getpgroup_nid_postfix(PosixSpawnAttributes* const* attributes, std::int32_t* processGroup) { + const auto& source = Attributes(attributes, "posix_spawnattr_getpgroup"); + if (processGroup == nullptr) throw std::invalid_argument("posix_spawnattr_getpgroup: null process group"); + *processGroup = source.processGroup; + return 0; +} + +int APS5_VABI posix_spawnattr_getschedparam_nid_postfix(PosixSpawnAttributes* const* attributes, std::int32_t* schedulingParameter) { + const auto& source = Attributes(attributes, "posix_spawnattr_getschedparam"); + if (schedulingParameter == nullptr) throw std::invalid_argument("posix_spawnattr_getschedparam: null scheduling parameter"); + *schedulingParameter = source.schedulingPriority; + return 0; +} + +int APS5_VABI posix_spawnattr_getschedpolicy_nid_postfix(PosixSpawnAttributes* const* attributes, std::int32_t* schedulingPolicy) { + const auto& source = Attributes(attributes, "posix_spawnattr_getschedpolicy"); + if (schedulingPolicy == nullptr) throw std::invalid_argument("posix_spawnattr_getschedpolicy: null scheduling policy"); + *schedulingPolicy = source.schedulingPolicy; + return 0; +} + +int APS5_VABI posix_spawnattr_getsigdefault_nid_postfix(PosixSpawnAttributes* const* attributes, GuestSignalSet* signalDefault) { + const auto& source = Attributes(attributes, "posix_spawnattr_getsigdefault"); + if (signalDefault == nullptr) throw std::invalid_argument("posix_spawnattr_getsigdefault: null signal set"); + *signalDefault = source.signalDefault; + return 0; +} + +int APS5_VABI posix_spawnattr_getsigmask_nid_postfix(PosixSpawnAttributes* const* attributes, GuestSignalSet* signalMask) { + const auto& source = Attributes(attributes, "posix_spawnattr_getsigmask"); + if (signalMask == nullptr) throw std::invalid_argument("posix_spawnattr_getsigmask: null signal set"); + *signalMask = source.signalMask; + return 0; +} + +int APS5_VABI posix_spawnattr_setpgroup_nid_postfix(PosixSpawnAttributes** attributes, std::int32_t processGroup) { + Attributes(attributes, "posix_spawnattr_setpgroup").processGroup = processGroup; + return 0; +} + +int APS5_VABI posix_spawnattr_setschedparam_nid_postfix(PosixSpawnAttributes** attributes, const std::int32_t* schedulingParameter) { + auto& target = Attributes(attributes, "posix_spawnattr_setschedparam"); + if (schedulingParameter == nullptr) throw std::invalid_argument("posix_spawnattr_setschedparam: null scheduling parameter"); + target.schedulingPriority = *schedulingParameter; + return 0; +} + +int APS5_VABI posix_spawnattr_setschedpolicy_nid_postfix(PosixSpawnAttributes** attributes, std::int32_t schedulingPolicy) { + Attributes(attributes, "posix_spawnattr_setschedpolicy").schedulingPolicy = schedulingPolicy; + return 0; +} + +int APS5_VABI posix_spawnattr_setsigmask_nid_postfix(PosixSpawnAttributes** attributes, const GuestSignalSet* signalMask) { + auto& target = Attributes(attributes, "posix_spawnattr_setsigmask"); + if (signalMask == nullptr) throw std::invalid_argument("posix_spawnattr_setsigmask: null signal set"); + target.signalMask = *signalMask; + return 0; +} + int APS5_VABI posix_spawn_file_actions_init_nid_postfix(PosixSpawnFileActions** actions) { if (actions == nullptr) throw std::invalid_argument("posix_spawn_file_actions_init: null file actions"); auto* created = new (std::nothrow) PosixSpawnFileActions{}; diff --git a/core/libs/tests/GuestPosixSpawn.cpp b/core/libs/tests/GuestPosixSpawn.cpp index 8fe49f711d..33ae4ebd0f 100644 --- a/core/libs/tests/GuestPosixSpawn.cpp +++ b/core/libs/tests/GuestPosixSpawn.cpp @@ -12,6 +12,16 @@ int APS5_VABI posix_spawnattr_init_nid_postfix(PosixSpawnAttributes**); int APS5_VABI posix_spawnattr_destroy_nid_postfix(PosixSpawnAttributes**); int APS5_VABI posix_spawnattr_setflags_nid_postfix(PosixSpawnAttributes**, std::int16_t); int APS5_VABI posix_spawnattr_setsigdefault_nid_postfix(PosixSpawnAttributes**, const GuestSignalSet*); +int APS5_VABI posix_spawnattr_getflags_nid_postfix(PosixSpawnAttributes* const*, std::int16_t*); +int APS5_VABI posix_spawnattr_getpgroup_nid_postfix(PosixSpawnAttributes* const*, std::int32_t*); +int APS5_VABI posix_spawnattr_getschedparam_nid_postfix(PosixSpawnAttributes* const*, std::int32_t*); +int APS5_VABI posix_spawnattr_getschedpolicy_nid_postfix(PosixSpawnAttributes* const*, std::int32_t*); +int APS5_VABI posix_spawnattr_getsigdefault_nid_postfix(PosixSpawnAttributes* const*, GuestSignalSet*); +int APS5_VABI posix_spawnattr_getsigmask_nid_postfix(PosixSpawnAttributes* const*, GuestSignalSet*); +int APS5_VABI posix_spawnattr_setpgroup_nid_postfix(PosixSpawnAttributes**, std::int32_t); +int APS5_VABI posix_spawnattr_setschedparam_nid_postfix(PosixSpawnAttributes**, const std::int32_t*); +int APS5_VABI posix_spawnattr_setschedpolicy_nid_postfix(PosixSpawnAttributes**, std::int32_t); +int APS5_VABI posix_spawnattr_setsigmask_nid_postfix(PosixSpawnAttributes**, const GuestSignalSet*); int APS5_VABI posix_spawn_file_actions_init_nid_postfix(PosixSpawnFileActions**); int APS5_VABI posix_spawn_file_actions_destroy_nid_postfix(PosixSpawnFileActions**); int APS5_VABI posix_spawn_file_actions_addopen_nid_postfix(PosixSpawnFileActions**, int, const char*, int, std::uint16_t); @@ -67,7 +77,30 @@ static void CheckAttributes() { Check(std::memcmp(&attributes->signalDefault, &defaults, sizeof(defaults)) == 0); Check(std::memcmp(&attributes->signalMask, &empty, sizeof(empty)) == 0); + std::int16_t flags = 1; + std::int32_t value = 1; + GuestSignalSet read{{9u, 9u, 9u, 9u}}; + Check(posix_spawnattr_getflags_nid_postfix(&attributes, &flags) == 0 && flags == 0); + Check(posix_spawnattr_getsigdefault_nid_postfix(&attributes, &read) == 0 && std::memcmp(&read, &defaults, sizeof(read)) == 0); + Check(posix_spawnattr_getsigmask_nid_postfix(&attributes, &read) == 0 && std::memcmp(&read, &empty, sizeof(read)) == 0); + Check(posix_spawnattr_setpgroup_nid_postfix(&attributes, 77) == 0 && attributes->processGroup == 77); + Check(posix_spawnattr_getpgroup_nid_postfix(&attributes, &value) == 0 && value == 77); + const std::int32_t priority = 450; + Check(posix_spawnattr_setschedparam_nid_postfix(&attributes, &priority) == 0 && attributes->schedulingPriority == 450); + Check(posix_spawnattr_getschedparam_nid_postfix(&attributes, &value) == 0 && value == 450); + Check(posix_spawnattr_setschedpolicy_nid_postfix(&attributes, 2) == 0 && attributes->schedulingPolicy == 2); + Check(posix_spawnattr_getschedpolicy_nid_postfix(&attributes, &value) == 0 && value == 2); + const GuestSignalSet mask{{1u, 2u, 3u, 4u}}; + Check(posix_spawnattr_setsigmask_nid_postfix(&attributes, &mask) == 0 && std::memcmp(&attributes->signalMask, &mask, sizeof(mask)) == 0); + Check(posix_spawnattr_getsigmask_nid_postfix(&attributes, &read) == 0 && std::memcmp(&read, &mask, sizeof(read)) == 0); + Check(std::memcmp(&attributes->signalDefault, &defaults, sizeof(defaults)) == 0); + PosixSpawnAttributes* uninitialized = nullptr; + Check(ThrowsInvalidArgument([&] { posix_spawnattr_getflags_nid_postfix(&uninitialized, &flags); })); + Check(ThrowsInvalidArgument([&] { posix_spawnattr_getpgroup_nid_postfix(&attributes, nullptr); })); + Check(ThrowsInvalidArgument([&] { posix_spawnattr_setpgroup_nid_postfix(&uninitialized, 1); })); + Check(ThrowsInvalidArgument([&] { posix_spawnattr_setschedparam_nid_postfix(&attributes, nullptr); })); + Check(ThrowsInvalidArgument([&] { posix_spawnattr_setsigmask_nid_postfix(&attributes, nullptr); })); Check(ThrowsInvalidArgument([&] { posix_spawnattr_setflags_nid_postfix(&uninitialized, 1); })); Check(ThrowsInvalidArgument([&] { posix_spawnattr_setsigdefault_nid_postfix(&uninitialized, &defaults); })); Check(ThrowsInvalidArgument([&] { posix_spawnattr_setsigdefault_nid_postfix(&attributes, nullptr); })); diff --git a/docs/dev/debt/libSceLibcInternal.md b/docs/dev/debt/libSceLibcInternal.md index 7cc67d0d95..be978908f4 100644 --- a/docs/dev/debt/libSceLibcInternal.md +++ b/docs/dev/debt/libSceLibcInternal.md @@ -14,4 +14,5 @@ - [fstatvfs](../../../core/libs/prx/libSceLibcInternal/FilesystemStatistics.cpp) (libSceLibcInternal) converts `_fstatfs` to the 88-byte `struct statvfs` as FreeBSD 9.2's `sfs2svfs` does (`f_bsize` from the transfer size, `f_frsize` from the block size, `f_favail` equal to `f_ffree`, `f_fsid` 0, `ST_RDONLY` and `ST_NOSUID` only). Differences: `f_namemax` is the `statfs` value, not `fpathconf(_PC_NAME_MAX)`; negative `f_bavail` and `f_ffree` wrap to unsigned without `EOVERFLOW`, as the same-width copy in FreeBSD does; inode counts are 0 on Windows, inherited from `_fstatfs`; a null output returns `EFAULT` after the descriptor is validated, where FreeBSD would fault. Layout and conversion are unverified against the PS5 library (PS5MC imports `fstatvfs` but was not run) - [posix_spawn and posix_spawnp](../../../core/libs/prx/libSceLibcInternal/Spawn.cpp) (libSceLibcInternal) throw: creating a child process is not supported. PPSA99305 (Dusklight) and PPSA99309 (Wind Waker) stop on `main` at unresolved import `44hlATrd47U` (`posix_spawnattr_init`); a run with this PR has not established how far either title gets, and process creation still throws. The attribute and file action calls keep FreeBSD 11.4's objects, so a real `posix_spawn` would have to apply them in the child. Null or uninitialized objects, which FreeBSD dereferences, throw. - [getopt, getopt_long](../../../core/libs/prx/libSceLibcInternal/Getopt.cpp) (libSceLibcInternal) follow FreeBSD 11 [getopt.c](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/stdlib/getopt.c) and [getopt_long.c](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/stdlib/getopt_long.c) with `GNU_COMPATIBLE`: `getopt` does not permute, `getopt_long` permutes unless `POSIXLY_CORRECT` is set in the guest environment or the option string starts with `+`, and the exported `opterr`, `optind`, `optopt`, `optreset` and `optarg` start at 1, 1, 0, 0 and null. Warnings go to the guest's current `stderr` prefixed with `__progname`, where FreeBSD's `warnx` keeps the `stderr` of its first call. A null `getopt` option string, a null argument vector or argument, a negative `optind` and an `optind` moved inside a pending permutation throw where FreeBSD faults. `getopt_long_only` is not exported. That the console's `libSceLibcInternal` uses this FreeBSD code is unverified +- [posix_spawnattr_getflags, getpgroup, getschedparam, getschedpolicy, getsigdefault, getsigmask, setpgroup, setschedparam, setschedpolicy and setsigmask](../../../core/libs/prx/libSceLibcInternal/Spawn.cpp) (libSceLibcInternal) read and write the 0x30-byte attribute object at the offsets of the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x8D860 to 0x8D910: flags at 0, process group at 4, scheduling parameter at 8, policy at 0xC, default and masked signal sets at 0x10 and 0x20), which never fail there and return 0; the layout matched on a PS5 (firmware 13.60). Uninitialized attributes and null pointers, which the console dereferences, throw as the existing setters do - [sys_siglist and sys_nsig](../../../core/libs/prx/libSceLibcInternal/SignalList.cpp) (libSceLibcInternal) hold the 32 descriptions and the count of [FreeBSD 11.4](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/gen/siglist.c#L73-L107) `siglist.c` (`NSIG` 32, signal 0 to `SIGUSR2`); the console's table was not read From 48b394d1ecf8eb61dac140ba10cf263dff519432 Mon Sep 17 00:00:00 2001 From: thuanquang <ghoulbulldog@gmail.com> Date: Sun, 18 Oct 2026 08:42:00 +0700 Subject: [PATCH 096/103] fix(agc): keep robustBufferAccess2 off on AMD's proprietary driver Since #4048 enabled robustBufferAccess2, AMD's Windows driver draws nothing in agc_driver_layered_volume_draw and agc_driver_layered_volume_vertex, and the selected loads make the high half of agc_driver_subvector_loop's wave64 skip its loop. Until the cause is found, that driver does not get the feature, and its loads keep their selections. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 9 +++++ .../Execution/include/RobustBufferAccess.hpp | 14 ++++++++ .../Execution/src/VulkanDevice.cpp | 10 +++++- .../tests/RobustBufferAccess.cpp | 33 +++++++++++++++++++ docs/dev/debt/shader.md | 2 +- 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/RobustBufferAccess.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index fdf3f37b5c..16c629d635 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -610,6 +610,15 @@ if(BUILD_TESTING) add_test(NAME agc_pipeline_link_optimization COMMAND agc_pipeline_link_optimization_tests) endif() +add_test_executable(agc_robust_buffer_access_tests EXCLUDE_FROM_ALL tests/RobustBufferAccess.cpp) +target_include_directories(agc_robust_buffer_access_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) +set_target_properties(agc_robust_buffer_access_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_robust_buffer_access_tests) +if(BUILD_TESTING) + set_target_properties(agc_robust_buffer_access_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_robust_buffer_access COMMAND agc_robust_buffer_access_tests) +endif() + add_test_executable(agc_device_fault_tests EXCLUDE_FROM_ALL tests/DeviceFault.cpp) target_include_directories(agc_device_fault_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) set_target_properties(agc_device_fault_tests PROPERTIES CXX_EXTENSIONS OFF) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp new file mode 100644 index 0000000000..a180b23104 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp @@ -0,0 +1,14 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_ROBUSTBUFFERACCESS_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_ROBUSTBUFFERACCESS_HPP + +#include <vulkan/vulkan.h> + +namespace AgcDriver { + +inline bool RobustBufferAccess2Usable(VkDriverId driver) { + return driver != VK_DRIVER_ID_AMD_PROPRIETARY; +} + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index e1e3b027f6..8f1b3bc7db 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -15,6 +15,7 @@ #include "prx/libSceAgcDriver/Execution/include/DeviceFault.hpp" #include "prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp" #include "prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp" +#include "prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp" #include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp" #include "prx/libSceAgcDriver/Execution/include/PresentMode.hpp" @@ -1836,7 +1837,14 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : VkPhysicalDeviceFeatures2 queried{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness2}; state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &queried); } - state->robustBufferAccess2 = robustness2.robustBufferAccess2 == VK_TRUE && enabled.robustBufferAccess == VK_TRUE; + VkDriverId robustnessDriver = VK_DRIVER_ID_MAX_ENUM; + if (hasExtension(VK_KHR_DRIVER_PROPERTIES_EXTENSION_NAME)) { + VkPhysicalDeviceDriverProperties driver{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; + VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver}; + state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties); + robustnessDriver = driver.driverID; + } + state->robustBufferAccess2 = robustness2.robustBufferAccess2 == VK_TRUE && enabled.robustBufferAccess == VK_TRUE && RobustBufferAccess2Usable(robustnessDriver); if (robustness2.nullDescriptor == VK_TRUE || state->robustBufferAccess2) { const bool nullDescriptor = robustness2.nullDescriptor == VK_TRUE; robustness2 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT}; diff --git a/core/libs/prx/libSceAgcDriver/tests/RobustBufferAccess.cpp b/core/libs/prx/libSceAgcDriver/tests/RobustBufferAccess.cpp new file mode 100644 index 0000000000..6b0331109f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/RobustBufferAccess.cpp @@ -0,0 +1,33 @@ +#include "prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp" +#include <cstdio> + +namespace { + +struct Case { + VkDriverId driver; + bool usable; +}; + +constexpr Case Cases[]{ + {VK_DRIVER_ID_AMD_PROPRIETARY, false}, + {VK_DRIVER_ID_AMD_OPEN_SOURCE, true}, + {VK_DRIVER_ID_MESA_RADV, true}, + {VK_DRIVER_ID_NVIDIA_PROPRIETARY, true}, + {VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS, true}, + {VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA, true}, + {VK_DRIVER_ID_MESA_LLVMPIPE, true}, + {VK_DRIVER_ID_MOLTENVK, true}, +}; + +} + +int main() { + int failures = 0; + for (const auto& item : Cases) { + if (AgcDriver::RobustBufferAccess2Usable(item.driver) != item.usable) { + std::fprintf(stderr, "driver %d: expected robustBufferAccess2 to be %s\n", static_cast<int>(item.driver), item.usable ? "usable" : "unusable"); + ++failures; + } + } + return failures == 0 ? 0 : 1; +} diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index d4129c7e2f..768c21eace 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -155,4 +155,4 @@ - [Flag structurization](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a graph that every structurizer attempt refuses, irreducible and non-terminating graphs included, is rebuilt with one reach flag per block and one again flag per loop (`structurizeWithFlags`): each block keeps one copy and runs behind a guard on its flag, and each strongly connected component becomes a loop whose single latch repeats it while a back edge set its flag. The output has about 8 times the input's blocks and one route variable per block, and ends in an empty Return that never runs. Only blocks that start the top-level order unconditionally run unguarded, so resource planning also sees the static paths that skip a guarded block and can plan extra descriptor candidates for them. The attempts still run first. Shared merge splitting refuses an attempt once the graph has grown past 16 times its block count at the start of splitting (at least 64 blocks); succeeding attempts grew at most 10.6 times on random graphs of up to 128 blocks, and an attempt that would succeed only beyond the budget leaves the graph to a later attempt or to this construction. Splitting that adds one block per round stays within that budget until its 4 rounds per block run out, so the attempts can still take more than 10 s (9 of 2,000 random graphs of up to 128 blocks, 63 s at most). `APS5_FORCE_STRUCTURIZER_FALLBACK=1` builds every graph this way, for tests - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) use only the base address, stride and record count for resources accessed exclusively by scalar loads; their fourth word is normalized to zero, following AMD RDNA2 ISA section 7.2.2. Vector and mixed accesses retain descriptor validation. Native Vulkan readbacks cover ignored fourth-word fields on Windows/NVIDIA; their console behavior has not been measured. - [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - `APS5_SPANNED_WAVE=1`, off by default, runs a single-wave wave64 compute program at one lane per invocation on a 32-lane host subgroup and exchanges ballots, shuffles and reductions between the two host subgroups through workgroup memory and barriers. The driver turns it on only where compute subgroups can be pinned at 32 full lanes (`subgroupSizeControl` and `computeFullSubgroups`), passes it as `SpirvTarget::spannedWaves` (part of the cache key and the serialized request), and creates the pipelines of spanned programs with `REQUIRE_FULL_SUBGROUPS`, since the lane number `SubgroupId * 32 + SubgroupLocalInvocationId` needs two full subgroups. Those barriers are correct only where every invocation of the workgroup reaches them, which holds for the guest's wave-uniform control flow. The emulated 64-bit LDS atomics ([LockedLdsUpdate](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp)) run only on the lanes the exec mask keeps, so their lock loop ballots within its own host subgroup, as its elect already does. Any other cross-lane exchange emitted inside per-lane control flow would hang a spanned wave in the same way, and the emitter does not reject one: there is no uniformity check. The only check so far is manual: disassemble the module (`spirv-dis`), list the selection and loop constructs that enclose each `OpControlBarrier`, and read the condition of each enclosing selection. In Ghost of Yōtei's program 0x80003af600 (PPSA26344), 108 of 111 barriers sat only in loops on wave-uniform conditions and 3 inside per-invocation conditionals, all in the lock loop. Other programs and titles have not been scanned. An earlier, cheaper ballot (one barrier instead of three) also lost the device for a reason that is not understood, which may be another barrier placement problem. Measured on an RTX 4090 (Windows): an offline replay of that program and two Yōtei runs that previously lost the device (NVIDIA TDR) now run past it, and [LdsMaskWaterfall.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp) runs both modes and covers a half-inactive wave. Stray (PPSA02101) on Intel ANV and a Demon's Souls (PPSA01342) request replay showed no fault, before this fix -- [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it) those loads keep their selections; LDS loads, stores, atomics, formatted and physical-address (BDA) accesses always do. LDS keeps the branch because NVIDIA's compiler predicates a branched LDS load at no cost, while the select form costs about 2.5 instructions per dword and splits a `ds_read_b128` into four loads (RTX 4060 measurements by @karanvora2599 in #4048); Intel's static counts favour the select only for wide reads, and run time was not measured on either. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. +- [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it) those loads keep their selections; LDS loads, stores, atomics, formatted and physical-address (BDA) accesses always do. LDS keeps the branch because NVIDIA's compiler predicates a branched LDS load at no cost, while the select form costs about 2.5 instructions per dword and splits a `ds_read_b128` into four loads (RTX 4060 measurements by @karanvora2599 in #4048); Intel's static counts favour the select only for wide reads, and run time was not measured on either. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. AMD's proprietary driver does not get `robustBufferAccess2` ([RobustBufferAccess.hpp](../../../core/libs/prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp)), so its loads keep their selections. On an RX 6700 XT with driver 32.0.21045.5002, enabling the feature broke `agc_driver_layered_volume_draw` and `agc_driver_layered_volume_vertex`: the vertex draw left the volume untouched even with the selections kept. `agc_driver_subvector_loop` failed too, with the selected loads: the high half of a wave64 skipped its loop. The cause was not found. From 300b7ff34c49896d51d17e45a9f1d1cb724bd884 Mon Sep 17 00:00:00 2001 From: T3cm0 <srodriguez7604@gmail.com> Date: Sat, 17 Oct 2026 20:42:20 -0500 Subject: [PATCH 097/103] feat(libSceLibcInternal): implement inet_aton, inet_addr and inet_ntoa Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../prx/libSceLibcInternal/CMakeLists.txt | 11 ++ core/libs/prx/libSceLibcInternal/Inet.cpp | 106 ++++++++++++++++++ core/libs/tests/GuestInetAddress.cpp | 68 +++++++++++ docs/dev/debt/libSceLibcInternal.md | 1 + 4 files changed, 186 insertions(+) create mode 100644 core/libs/prx/libSceLibcInternal/Inet.cpp create mode 100644 core/libs/tests/GuestInetAddress.cpp diff --git a/core/libs/prx/libSceLibcInternal/CMakeLists.txt b/core/libs/prx/libSceLibcInternal/CMakeLists.txt index b5dde2fe3f..1780e1d720 100644 --- a/core/libs/prx/libSceLibcInternal/CMakeLists.txt +++ b/core/libs/prx/libSceLibcInternal/CMakeLists.txt @@ -4,6 +4,7 @@ target_link_libraries(libSceLibcInternal PRIVATE ${CMAKE_DL_LIBS}) target_sources(libSceLibcInternal PRIVATE Directory.cpp) configure_windows_unwind(libSceLibcInternal) target_sources(libSceLibcInternal PRIVATE FilesystemStatistics.cpp PathStatistics.cpp) +target_sources(libSceLibcInternal PRIVATE Inet.cpp) if(BUILD_TESTING) add_test_executable(guest_statvfs_tests ${LIBS_INCLUDE_DIR}/tests/GuestStatvfs.cpp) @@ -95,6 +96,16 @@ if(BUILD_TESTING) set_tests_properties(guest_getopt PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceLibcInternal>") endif() + add_test_executable(guest_inet_address_tests ${LIBS_INCLUDE_DIR}/tests/GuestInetAddress.cpp) + target_include_directories(guest_inet_address_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_inet_address_tests PRIVATE libSceLibcInternal libkernel libc) + configure_windows_unwind(guest_inet_address_tests) + add_test(NAME guest_inet_address COMMAND guest_inet_address_tests) + set_tests_properties(guest_inet_address PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_inet_address PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceLibcInternal>") + endif() add_test_executable(guest_force_tls_destructor_tests ${LIBS_INCLUDE_DIR}/tests/GuestForceTlsDestructor.cpp) target_include_directories(guest_force_tls_destructor_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_force_tls_destructor_tests PRIVATE libSceLibcInternal libkernel libc ${CMAKE_DL_LIBS}) diff --git a/core/libs/prx/libSceLibcInternal/Inet.cpp b/core/libs/prx/libSceLibcInternal/Inet.cpp new file mode 100644 index 0000000000..90996aacf6 --- /dev/null +++ b/core/libs/prx/libSceLibcInternal/Inet.cpp @@ -0,0 +1,106 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include <cstdint> +#include <cstring> +#include <stdexcept> + +extern "C" { +const char* APS5_VABI __inet_ntop_nid_postfix(int family, const void* source, char* destination, std::uint32_t capacity); +} + +namespace { + +constexpr int AddressFamilyInet = 2; +char g_ntoaBuffer[18]; + +bool IsDigit(unsigned char c) { return c >= '0' && c <= '9'; } +bool IsHexDigit(unsigned char c) { return IsDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } +bool IsSpace(unsigned char c) { return c == ' ' || (c >= '\t' && c <= '\r'); } + +} + +extern "C" { + +int APS5_VABI inet_aton_nid_postfix(const char* text, std::uint32_t* address) { + if (text == nullptr) throw std::invalid_argument("inet_aton: null text"); + std::uint64_t value = 0; + std::uint8_t parts[4] = {}; + std::uint8_t* part = parts; + bool digit = false; + auto c = static_cast<unsigned char>(*text); + for (;;) { + if (!IsDigit(c)) return 0; + value = 0; + unsigned base = 10; + digit = false; + if (c == '0') { + c = static_cast<unsigned char>(*++text); + if (c == 'x' || c == 'X') { + base = 16; + c = static_cast<unsigned char>(*++text); + } else { + base = 8; + digit = true; + } + } + for (;;) { + if (IsDigit(c)) { + if (base == 8 && (c == '8' || c == '9')) return 0; + value = value * base + (c - '0'); + c = static_cast<unsigned char>(*++text); + digit = true; + } else if (base == 16 && IsHexDigit(c)) { + value = (value << 4) | static_cast<std::uint64_t>(c + 10 - (c >= 'a' ? 'a' : 'A')); + c = static_cast<unsigned char>(*++text); + digit = true; + } else { + break; + } + } + if (c != '.') break; + if (part >= parts + 3 || value > 0xff) return 0; + *part++ = static_cast<std::uint8_t>(value); + c = static_cast<unsigned char>(*++text); + } + if (c != 0 && (c >= 0x80 || !IsSpace(c))) return 0; + if (!digit) return 0; + switch (part - parts + 1) { + case 2: + if (value > 0xffffff) return 0; + value |= static_cast<std::uint64_t>(parts[0]) << 24; + break; + case 3: + if (value > 0xffff) return 0; + value |= (static_cast<std::uint64_t>(parts[0]) << 24) | (static_cast<std::uint64_t>(parts[1]) << 16); + break; + case 4: + if (value > 0xff) return 0; + value |= (static_cast<std::uint64_t>(parts[0]) << 24) | (static_cast<std::uint64_t>(parts[1]) << 16) | (static_cast<std::uint64_t>(parts[2]) << 8); + break; + default: + break; + } + if (address != nullptr) { + const auto host = static_cast<std::uint32_t>(value); + const std::uint8_t network[4] = {static_cast<std::uint8_t>(host >> 24), static_cast<std::uint8_t>(host >> 16), static_cast<std::uint8_t>(host >> 8), static_cast<std::uint8_t>(host)}; + std::memcpy(address, network, sizeof(network)); + } + return 1; +} + +std::uint32_t APS5_VABI inet_addr_nid_postfix(const char* text) { + std::uint32_t address = 0; + return inet_aton_nid_postfix(text, &address) ? address : 0xffffffffu; +} + +char* APS5_VABI inet_ntoa_r_nid_postfix(std::uint32_t address, char* buffer, std::uint32_t size) { + __inet_ntop_nid_postfix(AddressFamilyInet, &address, buffer, size); + return buffer; +} + +char* APS5_VABI inet_ntoa_nid_postfix(std::uint32_t address) { + std::memcpy(g_ntoaBuffer, "[inet_ntoa error]", sizeof(g_ntoaBuffer)); + __inet_ntop_nid_postfix(AddressFamilyInet, &address, g_ntoaBuffer, sizeof(g_ntoaBuffer)); + return g_ntoaBuffer; +} + +} diff --git a/core/libs/tests/GuestInetAddress.cpp b/core/libs/tests/GuestInetAddress.cpp new file mode 100644 index 0000000000..80447cad5d --- /dev/null +++ b/core/libs/tests/GuestInetAddress.cpp @@ -0,0 +1,68 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <string> + +extern "C" { +int APS5_VABI inet_aton_nid_postfix(const char* text, std::uint32_t* address); +std::uint32_t APS5_VABI inet_addr_nid_postfix(const char* text); +char* APS5_VABI inet_ntoa_nid_postfix(std::uint32_t address); +char* APS5_VABI inet_ntoa_r_nid_postfix(std::uint32_t address, char* buffer, std::uint32_t size); +} + +namespace { + +void Require(bool value, const char* what) { + if (value) return; + std::fprintf(stderr, "failed: %s\n", what); + std::abort(); +} + +std::uint32_t Network(std::uint8_t a, std::uint8_t b, std::uint8_t c, std::uint8_t d) { + const std::uint8_t bytes[4] = {a, b, c, d}; + std::uint32_t value = 0; + std::memcpy(&value, bytes, sizeof(value)); + return value; +} + +bool Parses(const char* text, std::uint32_t expected) { + std::uint32_t address = 0x5a5a5a5a; + return inet_aton_nid_postfix(text, &address) == 1 && address == expected; +} + +bool Rejects(const char* text) { + std::uint32_t address = 0x5a5a5a5a; + return inet_aton_nid_postfix(text, &address) == 0 && address == 0x5a5a5a5a; +} + +} + +int main() { + Require(Parses("192.168.1.2", Network(192, 168, 1, 2)), "dotted quad"); + Require(Parses("10.1", Network(10, 0, 0, 1)), "two parts"); + Require(Parses("10.1.300", Network(10, 1, 1, 44)), "three parts"); + Require(Parses("0x7f.1", Network(127, 0, 0, 1)), "hex part"); + Require(Parses("017.0.0.1", Network(15, 0, 0, 1)), "octal part"); + Require(Parses("3232235777", Network(192, 168, 1, 1)), "single number"); + Require(Parses("1.2.3.4 trailing", Network(1, 2, 3, 4)), "trailing after space"); + Require(Parses("0", Network(0, 0, 0, 0)), "zero"); + Require(Parses("0xffffffff", Network(255, 255, 255, 255)), "broadcast"); + Require(Rejects("256.1.1.1") && Rejects("1.2.3.256") && Rejects("1.2.65536"), "part out of range"); + Require(Rejects("1.2.3.4.5") && Rejects("08") && Rejects("0x") && Rejects("1.2.3.4x"), "malformed"); + Require(Rejects("") && Rejects(".1") && Rejects("1..2") && Rejects("-1"), "missing digits"); + Require(inet_aton_nid_postfix("1.2.3.4", nullptr) == 1 && inet_aton_nid_postfix("x", nullptr) == 0, "null destination"); + + Require(inet_addr_nid_postfix("127.0.0.1") == Network(127, 0, 0, 1), "inet_addr"); + Require(inet_addr_nid_postfix("bad") == 0xffffffffu && inet_addr_nid_postfix("255.255.255.255") == 0xffffffffu, "inet_addr none"); + + char* const text = inet_ntoa_nid_postfix(Network(10, 20, 30, 40)); + Require(std::string(text) == "10.20.30.40", "inet_ntoa"); + Require(inet_ntoa_nid_postfix(Network(255, 255, 255, 255)) == text && std::string(text) == "255.255.255.255", "inet_ntoa static buffer"); + char buffer[16]; + Require(inet_ntoa_r_nid_postfix(Network(1, 2, 3, 4), buffer, sizeof(buffer)) == buffer && std::string(buffer) == "1.2.3.4", "inet_ntoa_r"); + char tiny[4] = {'k', 'e', 'e', 'p'}; + Require(inet_ntoa_r_nid_postfix(Network(1, 2, 3, 4), tiny, sizeof(tiny)) == tiny, "inet_ntoa_r small buffer returns it"); + return 0; +} diff --git a/docs/dev/debt/libSceLibcInternal.md b/docs/dev/debt/libSceLibcInternal.md index 7cc67d0d95..14f0726d0a 100644 --- a/docs/dev/debt/libSceLibcInternal.md +++ b/docs/dev/debt/libSceLibcInternal.md @@ -15,3 +15,4 @@ - [posix_spawn and posix_spawnp](../../../core/libs/prx/libSceLibcInternal/Spawn.cpp) (libSceLibcInternal) throw: creating a child process is not supported. PPSA99305 (Dusklight) and PPSA99309 (Wind Waker) stop on `main` at unresolved import `44hlATrd47U` (`posix_spawnattr_init`); a run with this PR has not established how far either title gets, and process creation still throws. The attribute and file action calls keep FreeBSD 11.4's objects, so a real `posix_spawn` would have to apply them in the child. Null or uninitialized objects, which FreeBSD dereferences, throw. - [getopt, getopt_long](../../../core/libs/prx/libSceLibcInternal/Getopt.cpp) (libSceLibcInternal) follow FreeBSD 11 [getopt.c](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/stdlib/getopt.c) and [getopt_long.c](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/stdlib/getopt_long.c) with `GNU_COMPATIBLE`: `getopt` does not permute, `getopt_long` permutes unless `POSIXLY_CORRECT` is set in the guest environment or the option string starts with `+`, and the exported `opterr`, `optind`, `optopt`, `optreset` and `optarg` start at 1, 1, 0, 0 and null. Warnings go to the guest's current `stderr` prefixed with `__progname`, where FreeBSD's `warnx` keeps the `stderr` of its first call. A null `getopt` option string, a null argument vector or argument, a negative `optind` and an `optind` moved inside a pending permutation throw where FreeBSD faults. `getopt_long_only` is not exported. That the console's `libSceLibcInternal` uses this FreeBSD code is unverified - [sys_siglist and sys_nsig](../../../core/libs/prx/libSceLibcInternal/SignalList.cpp) (libSceLibcInternal) hold the 32 descriptions and the count of [FreeBSD 11.4](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/gen/siglist.c#L73-L107) `siglist.c` (`NSIG` 32, signal 0 to `SIGUSR2`); the console's table was not read +- [inet_aton, inet_addr, inet_ntoa and inet_ntoa_r](../../../core/libs/prx/libSceLibcInternal/Inet.cpp) (libSceLibcInternal) follow the FreeBSD code in the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x8ADE0 to 0x8B3D0) and matched it on a PS5 (firmware 13.60) for 20 `inet_aton` inputs; a null text, which the console dereferences, throws From 24a999c49947cafe5ba14c29930bdb5d420a8653 Mon Sep 17 00:00:00 2001 From: T3cm0 <srodriguez7604@gmail.com> Date: Sat, 17 Oct 2026 20:43:13 -0500 Subject: [PATCH 098/103] feat(libc): implement long double, Bessel and integer math exports Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libc/src/Math.cpp | 60 +++++++++++++++++++++ core/libs/tests/GuestMath.cpp | 92 +++++++++++++++++++++++++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 153 insertions(+) diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index 029440fc53..ddeea98f8c 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -126,6 +126,23 @@ struct ExtendedParts { std::uint16_t signExponent; }; +template <typename TValue, typename THypot> +TValue Hypot3(TValue x, TValue y, TValue z, THypot hypot) { + if (std::isnan(x)) return x; + if (std::isnan(y)) return y; + if (std::isnan(z)) return z; + if (std::isinf(x) || std::isinf(y) || std::isinf(z)) return std::numeric_limits<TValue>::infinity(); + if (x == 0) return hypot(y, z); + if (y == 0) return hypot(x, z); + if (z == 0) return hypot(x, y); + x = std::fabs(x); + y = std::fabs(y); + z = std::fabs(z); + if (z >= x && z >= y) return hypot(z, hypot(x, y)); + if (y >= x && y >= z) return hypot(y, hypot(x, z)); + return hypot(x, hypot(y, z)); +} + ExtendedParts SplitExtended(long double value) { unsigned char bytes[sizeof(long double)]; std::memcpy(bytes, &value, sizeof(value)); @@ -199,6 +216,22 @@ std::lldiv_t APS5_VABI ldiv_nid_postfix(std::int64_t numerator, std::int64_t den return std::lldiv(numerator, denominator); } +std::lldiv_t APS5_VABI imaxdiv_nid_postfix(std::int64_t numerator, std::int64_t denominator) { + return std::lldiv(numerator, denominator); +} + +std::int64_t APS5_VABI labs_nid_postfix(std::int64_t x) { + return x < 0 ? static_cast<std::int64_t>(0u - static_cast<std::uint64_t>(x)) : x; +} + +std::int64_t APS5_VABI llabs_nid_postfix(std::int64_t x) { + return labs_nid_postfix(x); +} + +std::int64_t APS5_VABI imaxabs_nid_postfix(std::int64_t x) { + return labs_nid_postfix(x); +} + float APS5_VABI fmodf_nid_postfix(float x, float y) { return std::fmod(x, y); } float APS5_VABI asinf_nid_postfix(float x) { const int saved_errno = errno; @@ -242,6 +275,8 @@ double APS5_VABI exp2_nid_postfix(double x) { double APS5_VABI ldexp_nid_postfix(double x, int exponent) { return std::ldexp(x, exponent); } double APS5_VABI scalbn_nid_postfix(double x, int exponent) { return std::scalbn(x, exponent); } float APS5_VABI scalbnf_nid_postfix(float x, int exponent) { return std::scalbn(x, exponent); } +long double APS5_VABI scalbnl_nid_postfix(long double x, int exponent) { return std::scalbn(x, exponent); } +float APS5_VABI significandf_nid_postfix(float x) { return x == 0.0f || !std::isfinite(x) ? x : std::scalbn(x, -std::ilogb(x)); } double APS5_VABI frexp_nid_postfix(double x, int* exponent) { return std::frexp(x, exponent); } float APS5_VABI frexpf_nid_postfix(float x, int* exponent) { return std::frexp(x, exponent); } // Guest long is 64-bit, including on Windows where native long is 32-bit. @@ -285,6 +320,31 @@ double APS5_VABI fmax_nid_postfix(double x, double y) { return GuestFmax(x, y); float APS5_VABI fmaxf_nid_postfix(float x, float y) { return GuestFmax(x, y); } long double APS5_VABI fmaxl_nid_postfix(long double x, long double y) { return GuestFmax(x, y); } long double APS5_VABI acosl_nid_postfix(long double x) { return std::acos(x); } +long double APS5_VABI acoshl_nid_postfix(long double x) { return std::acosh(x); } +long double APS5_VABI asinhl_nid_postfix(long double x) { return std::asinh(x); } +long double APS5_VABI atanhl_nid_postfix(long double x) { return std::atanh(x); } +long double APS5_VABI atan2l_nid_postfix(long double y, long double x) { return std::atan2(y, x); } +long double APS5_VABI cbrtl_nid_postfix(long double x) { return std::cbrt(x); } +long double APS5_VABI erfl_nid_postfix(long double x) { return std::erf(x); } +long double APS5_VABI erfcl_nid_postfix(long double x) { return std::erfc(x); } +long double APS5_VABI expm1l_nid_postfix(long double x) { return std::expm1(x); } +long double APS5_VABI log1pl_nid_postfix(long double x) { return std::log1p(x); } +long double APS5_VABI log2l_nid_postfix(long double x) { return std::log2(x); } +long double APS5_VABI lgammal_nid_postfix(long double x) { return std::lgamma(x); } +long double APS5_VABI tgammal_nid_postfix(long double x) { return std::tgamma(x); } +long double APS5_VABI hypotl_nid_postfix(long double x, long double y) { + if (std::isinf(x) || std::isinf(y)) return std::numeric_limits<long double>::infinity(); + return std::hypot(x, y); +} +float APS5_VABI hypot3f_nid_postfix(float x, float y, float z) { return Hypot3(x, y, z, hypotf_nid_postfix); } +double APS5_VABI hypot3_nid_postfix(double x, double y, double z) { return Hypot3(x, y, z, hypot_nid_postfix); } +long double APS5_VABI hypot3l_nid_postfix(long double x, long double y, long double z) { return Hypot3(x, y, z, hypotl_nid_postfix); } +float APS5_VABI j0f_nid_postfix(float x) { return static_cast<float>(::j0(x)); } +float APS5_VABI j1f_nid_postfix(float x) { return static_cast<float>(::j1(x)); } +float APS5_VABI jnf_nid_postfix(int order, float x) { return static_cast<float>(::jn(order, x)); } +float APS5_VABI y0f_nid_postfix(float x) { return static_cast<float>(::y0(x)); } +float APS5_VABI y1f_nid_postfix(float x) { return static_cast<float>(::y1(x)); } +float APS5_VABI ynf_nid_postfix(int order, float x) { return static_cast<float>(::yn(order, x)); } long double APS5_VABI asinl_nid_postfix(long double x) { const ExtendedParts parts = SplitExtended(x); diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index 3ab7ee3685..d095eda0ff 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -117,6 +117,34 @@ int APS5_VABI __isinf_nid_postfix(double); int APS5_VABI isnan_nid_postfix(double); int APS5_VABI isinf_nid_postfix(double); std::lldiv_t APS5_VABI lldiv_nid_postfix(long long, long long); +std::lldiv_t APS5_VABI imaxdiv_nid_postfix(std::int64_t, std::int64_t); +std::int64_t APS5_VABI labs_nid_postfix(std::int64_t); +std::int64_t APS5_VABI llabs_nid_postfix(std::int64_t); +std::int64_t APS5_VABI imaxabs_nid_postfix(std::int64_t); +long double APS5_VABI scalbnl_nid_postfix(long double, int); +float APS5_VABI significandf_nid_postfix(float); +long double APS5_VABI acoshl_nid_postfix(long double); +long double APS5_VABI asinhl_nid_postfix(long double); +long double APS5_VABI atanhl_nid_postfix(long double); +long double APS5_VABI atan2l_nid_postfix(long double, long double); +long double APS5_VABI cbrtl_nid_postfix(long double); +long double APS5_VABI erfl_nid_postfix(long double); +long double APS5_VABI erfcl_nid_postfix(long double); +long double APS5_VABI expm1l_nid_postfix(long double); +long double APS5_VABI log1pl_nid_postfix(long double); +long double APS5_VABI log2l_nid_postfix(long double); +long double APS5_VABI lgammal_nid_postfix(long double); +long double APS5_VABI tgammal_nid_postfix(long double); +long double APS5_VABI hypotl_nid_postfix(long double, long double); +float APS5_VABI hypot3f_nid_postfix(float, float, float); +double APS5_VABI hypot3_nid_postfix(double, double, double); +long double APS5_VABI hypot3l_nid_postfix(long double, long double, long double); +float APS5_VABI j0f_nid_postfix(float); +float APS5_VABI j1f_nid_postfix(float); +float APS5_VABI jnf_nid_postfix(int, float); +float APS5_VABI y0f_nid_postfix(float); +float APS5_VABI y1f_nid_postfix(float); +float APS5_VABI ynf_nid_postfix(int, float); std::lldiv_t APS5_VABI ldiv_nid_postfix(std::int64_t, std::int64_t); float APS5_VABI nanf_nid_postfix(const char*); double APS5_VABI nan_nid_postfix(const char*); @@ -755,6 +783,67 @@ static void CheckFloatMathExports() { Require(isinf_nid_postfix(nan) == 0 && isinf_nid_postfix(1.0) == 0); } +static void CheckIntegerAbsAndDivision() { + for (auto function : {labs_nid_postfix, llabs_nid_postfix, imaxabs_nid_postfix}) { + Require(function(-5) == 5 && function(7) == 7 && function(0) == 0); + Require(function(-0x100000000ll) == 0x100000000ll); + Require(function(INT64_MAX) == INT64_MAX && function(INT64_MIN) == INT64_MIN); + } + const auto small = imaxdiv_nid_postfix(-7, 2); + Require(small.quot == -3 && small.rem == -1); + const auto large = imaxdiv_nid_postfix(0x123456789abll, -0x1000); + Require(large.quot == -0x12345678ll && large.rem == 0x9abll); +} + +static void CheckScaling() { + Require(scalbnl_nid_postfix(3.0L, 4) == 48.0L && scalbnl_nid_postfix(1.0L, -16445) == 0x1p-16445L); + Require(scalbnl_nid_postfix(1.0L, 16384) == std::numeric_limits<long double>::infinity()); + Require(significandf_nid_postfix(12.0f) == 1.5f && significandf_nid_postfix(-0.375f) == -1.5f); + Require(significandf_nid_postfix(0x1p-149f) == 1.0f); + Require(std::signbit(significandf_nid_postfix(-0.0f)) && significandf_nid_postfix(-0.0f) == 0.0f); + Require(significandf_nid_postfix(std::numeric_limits<float>::infinity()) == std::numeric_limits<float>::infinity()); + Require(std::isnan(significandf_nid_postfix(std::numeric_limits<float>::quiet_NaN()))); +} + +static bool Near(long double actual, long double expected) { + return std::fabs(actual - expected) <= std::fabs(expected) * 1e-17L; +} + +static void CheckLongDoubleFunctions() { + const long double infinity = std::numeric_limits<long double>::infinity(); + const long double nan = std::numeric_limits<long double>::quiet_NaN(); + Require(acoshl_nid_postfix(1.0L) == 0.0L && std::isnan(acoshl_nid_postfix(0.5L))); + Require(asinhl_nid_postfix(-0.0L) == 0.0L && std::signbit(asinhl_nid_postfix(-0.0L))); + Require(atanhl_nid_postfix(1.0L) == infinity && Near(atanhl_nid_postfix(0.5L), 0.5493061443340548456976226L)); + Require(Near(atan2l_nid_postfix(1.0L, 1.0L), 0.7853981633974483096156608L) && Near(atan2l_nid_postfix(0.0L, -1.0L), 3.1415926535897932384626434L)); + Require(cbrtl_nid_postfix(-27.0L) == -3.0L && Near(cbrtl_nid_postfix(0x1p-16443L) * cbrtl_nid_postfix(0x1p-16443L) * cbrtl_nid_postfix(0x1p-16443L), 0x1p-16443L)); + Require(erfl_nid_postfix(0.0L) == 0.0L && erfl_nid_postfix(infinity) == 1.0L && erfcl_nid_postfix(0.0L) == 1.0L); + Require(erfcl_nid_postfix(infinity) == 0.0L && erfcl_nid_postfix(-infinity) == 2.0L); + Require(expm1l_nid_postfix(0.0L) == 0.0L && expm1l_nid_postfix(-infinity) == -1.0L); + Require(log1pl_nid_postfix(0.0L) == 0.0L && log1pl_nid_postfix(-1.0L) == -infinity); + Require(log2l_nid_postfix(8.0L) == 3.0L && log2l_nid_postfix(0x1p-16445L) == -16445.0L); + Require(lgammal_nid_postfix(1.0L) == 0.0L && lgammal_nid_postfix(2.0L) == 0.0L && lgammal_nid_postfix(0.0L) == infinity); + Require(tgammal_nid_postfix(5.0L) == 24.0L && Near(tgammal_nid_postfix(0.5L), 1.7724538509055160272981675L)); + Require(hypotl_nid_postfix(3.0L, -4.0L) == 5.0L && hypotl_nid_postfix(nan, -infinity) == infinity); + + Require(hypot3f_nid_postfix(3.0f, -4.0f, 12.0f) == 13.0f && hypot3f_nid_postfix(0.0f, 3.0f, -4.0f) == 5.0f); + Require(hypot3f_nid_postfix(2.0f, 0.0f, 0.0f) == 2.0f && hypot3f_nid_postfix(-2.0f, 3.0f, 6.0f) == 7.0f); + Require(hypot3_nid_postfix(1.0, 2.0, 2.0) == 3.0 && hypot3_nid_postfix(12.0, 3.0, 4.0) == 13.0); + Require(hypot3l_nid_postfix(2.0L, -3.0L, 6.0L) == 7.0L); + const double payload = std::bit_cast<double>(0x7ff8000000000123ull); + Require(std::bit_cast<std::uint64_t>(hypot3_nid_postfix(1.0, payload, std::numeric_limits<double>::infinity())) == 0x7ff8000000000123ull); + Require(std::isnan(hypot3f_nid_postfix(std::numeric_limits<float>::infinity(), 1.0f, std::numeric_limits<float>::quiet_NaN()))); + Require(hypot3l_nid_postfix(-infinity, 1.0L, 2.0L) == infinity && hypot3_nid_postfix(1.0, -std::numeric_limits<double>::infinity(), 0.0) == std::numeric_limits<double>::infinity()); + Require(hypot3_nid_postfix(0x1p1000, 0x1p1000, 0x1p1000) == std::hypot(0x1p1000, std::hypot(0x1p1000, 0x1p1000))); + + Require(j0f_nid_postfix(0.0f) == 1.0f && j1f_nid_postfix(0.0f) == 0.0f && jnf_nid_postfix(2, 0.0f) == 0.0f); + Require(std::fabs(j0f_nid_postfix(2.5f) - static_cast<float>(std::cyl_bessel_j(0.0, 2.5))) < 1e-6f); + Require(std::fabs(jnf_nid_postfix(3, 4.0f) - static_cast<float>(std::cyl_bessel_j(3.0, 4.0))) < 1e-6f); + Require(y0f_nid_postfix(0.0f) == -std::numeric_limits<float>::infinity() && y1f_nid_postfix(0.0f) == -std::numeric_limits<float>::infinity()); + Require(std::fabs(ynf_nid_postfix(2, 3.0f) - static_cast<float>(std::cyl_neumann(2.0, 3.0))) < 1e-6f); + Require(std::isnan(y0f_nid_postfix(-1.0f))); +} + static void CheckDoubleClassification() { const struct { std::uint64_t bits; short code; } cases[] = { {0x0000000000000000ull, 0}, {0x8000000000000000ull, 0}, @@ -1093,4 +1182,7 @@ int main() { CheckHypotSpecialValue(); CheckFmodErrno(); CheckDoubleClassification(); + CheckIntegerAbsAndDivision(); + CheckScaling(); + CheckLongDoubleFunctions(); } diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 497130f640..c2dc13eb6d 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -56,6 +56,7 @@ - [nan and nanf](../../../core/libs/prx/libc/src/Math.cpp) (libc) return fixed quiet NaN values (`0x7fc00000` and `0x7ff8000000000000`), ignoring the tag argument as observed in the console's `libc.prx` (both exports load a global constant via `mov rax,[rip+..]; vmovsd xmm0,[rax]; ret`); VLC (PPSA85300) stops on nanf at start-up after strsep. - [lgamma and signgam](../../../core/libs/prx/libc/src/Math.cpp) (libc) - `lgamma` returns the host's `lgamma` and stores the sign of Γ(x) in the exported `signgam` (NID `5gOOC0kzW0c`, starting at 0 as in FreeBSD 11's [`s_signgam.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/msun/src/s_signgam.c)) by the rules of [`e_lgamma_r.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/msun/src/e_lgamma_r.c): -1 for -0 and for a negative non-integer whose floor is odd, 1 otherwise, including poles, infinities and NaN. Taken from the FreeBSD source, not measured on the console. The NID is in the aerolib symbol list; no title is known to read `signgam`. PPSA99004 imports `lgamma` - [sinhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `sinhf` of the `libc.prx` bundled with PPSA28319 (disassembly): FreeBSD msun's [`e_sinhf.c`](https://github.com/freebsd/freebsd-src/blob/5aa554c7e5b6fd9042986df07f12ac5687e41392/lib/msun/src/e_sinhf.c) from before `__ldexp_expf`, which squares `expf(|x| / 2)` between `logf(FLT_MAX)` and the overflow threshold, calling msun's [`expm1f`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/s_expm1f.c) and [`expf`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_expf.c), whose paths for the arguments it passes are inlined here. Past the threshold Sony's error helper sets `errno` to `ERANGE` before returning `x * 1e37`; msun itself sets no `errno`. That code, run on a Windows host, gives the same bits as this implementation for every finite argument up to the threshold, of both signs; it was not run on Linux. That the system `libSceLibcInternal`, which Ruffle (PPSA68091) and RetroArch (PPSA99169) import while bundling a zero-return `libc.prx`, has the same code is assumed +- [acoshl, asinhl, atanhl, atan2l, cbrtl, erfl, erfcl, expm1l, hypotl, log1pl, log2l, lgammal, tgammal, j0f, j1f, jnf, y0f, y1f and ynf](../../../core/libs/prx/libc/src/Math.cpp) (libc) return the host C library's results (the float Bessel functions through its double ones), not those of the console's own code, so the last bits can differ. As in the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x56410), `lgammal` does not store the sign in `signgam`; `hypotl` returns infinity for an infinite argument even when the other is a NaN, as C11 F.10.4.3 requires. [hypot3, hypot3f and hypot3l](../../../core/libs/prx/libc/src/Math.cpp) follow that module (0x51D30, 0x520D0, 0x52420): the first NaN of x, y and z is returned unchanged, any infinity gives positive infinity, a zero argument gives the two-argument hypot of the other two, and otherwise the largest magnitude is combined with the hypot of the other two - [asinl](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `asinl` of the `libc.prx` bundled with PPSA28319 (disassembly), FreeBSD msun's [`e_asinl.c`](https://github.com/freebsd/freebsd-src/blob/9f0432041b8fb69832f10c32da50a67f34feeb61/lib/msun/src/e_asinl.c) with the ld80 [`invtrig.c`](https://github.com/freebsd/freebsd-src/blob/9f0432041b8fb69832f10c32da50a67f34feeb61/lib/msun/ld80/invtrig.c) coefficients, which the binary holds as the same 80-bit constants. For |x| > 1, the infinities and NaN, Sony's error helper raises `FE_INVALID` and sets `errno` to `EDOM` before returning `(x - x) / (x - x)`, so a NaN argument also sets `EDOM`; msun itself sets no `errno`. 20 million random arguments in (-1, 1) give the same bits as that code run on a Windows host. The result needs the x87 precision control at 64 bits, as FreeBSD amd64 starts threads (control word `0x37f`), but guest threads do not set it: on Windows a thread started by `_beginthreadex` runs at 53 bits (`0x27f`), where `asinl(0.75)` returns mantissa `0xd91a98ae3406e000` instead of `0xd91a98ae3406e041`. The system `libSceLibcInternal` that RetroArch (PPSA99169) imports is assumed to have the same code - [atanhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `atanhf` of the `libc.prx` bundled with PPSA28319 (disassembly): FreeBSD msun's [`e_atanhf.c`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_atanhf.c) calling msun's [`log1pf`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/s_log1pf.c), whose paths for the positive arguments it passes are inlined here; the exported `log1pf` is still the host's. For |x| > 1, the infinities and NaN, Sony's error helper raises `FE_INVALID` and sets `errno` to `EDOM` before returning `(x - x) / (x - x)`, and for |x| = 1 it raises `FE_DIVBYZERO` and sets `errno` to `ERANGE` before returning `x / 0`; msun itself sets no `errno`. That code, run on a Windows host, gives the same bits, `errno` and floating-point exceptions as this implementation for all 2^32 arguments; it was not run on Linux. That the system `libSceLibcInternal`, which Dusklight (PPSA99305) imports while bundling a zero-return `libc.prx`, has the same code is assumed - [getenv, setenv, unsetenv, putenv](../../../core/libs/prx/libc/src/Environment.cpp) (libc) keep one guest environment that starts empty, like the `environ` libkernel exports, and never reads or changes the host's. A title sees only the variables it sets itself; host variables such as `HOME` and `PATH` name host paths a guest cannot use (PS5MC, PPSA99420, exits with `XBMC_Run returned -1` when it gets the host's `HOME`). Whether the console starts a title with any variables set is not measured From 58697488a2760a9fddc2179ecbde8800d0413589 Mon Sep 17 00:00:00 2001 From: T3cm0 <srodriguez7604@gmail.com> Date: Sat, 17 Oct 2026 20:43:39 -0500 Subject: [PATCH 099/103] feat(libc): implement the uchar.h conversions, mblen and wctomb Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libc/src/Multibyte.cpp | 160 +++++++++++++++++++++++++++ core/libs/tests/GuestWideStrings.cpp | 56 ++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 217 insertions(+) diff --git a/core/libs/prx/libc/src/Multibyte.cpp b/core/libs/prx/libc/src/Multibyte.cpp index fee93d88d6..5865fe5933 100644 --- a/core/libs/prx/libc/src/Multibyte.cpp +++ b/core/libs/prx/libc/src/Multibyte.cpp @@ -5,6 +5,54 @@ #include <cstdint> #include <cstring> #include <climits> +#include <stdexcept> + +namespace { + +constexpr std::size_t PendingOffset = 0xa; +constexpr std::int8_t LowSurrogatePending = 0x10; + +void StorePending(unsigned char* state, std::int8_t pending) { + const auto stored = static_cast<std::int16_t>(pending); + std::memcpy(state + PendingOffset, &stored, sizeof(stored)); +} + +std::size_t EncodeUtf8(char* destination, std::uint64_t value) { + if (value < 0x80) { + destination[0] = static_cast<char>(value); + return 1; + } + int extra = 5; + unsigned lead = 0xfc; + int shift = 30; + if (value < 0x800) { extra = 1; lead = 0xc0; shift = 6; } + else if (value < 0x10000) { extra = 2; lead = 0xe0; shift = 12; } + else if (value < 0x200000) { extra = 3; lead = 0xf0; shift = 18; } + else if (value < 0x4000000) { extra = 4; lead = 0xf8; shift = 24; } + destination[0] = static_cast<char>((value >> shift) | lead); + for (int index = 1; index <= extra; ++index) + destination[index] = static_cast<char>(((value >> (shift - 6 * index)) & 0x3f) | 0x80); + return static_cast<std::size_t>(extra + 1); +} + +bool DecodeUtf8Byte(unsigned char byte, std::uint64_t& value, std::int8_t& pending) { + if (pending > 0) { + if ((byte & 0xc0) != 0x80) return false; + value = (value << 6) | (byte & 0x3f); + --pending; + return true; + } + if (byte < 0x80) value = byte; + else if ((byte & 0xe0) == 0xc0) { value = byte & 0x1f; pending = 1; } + else if ((byte & 0xf0) == 0xe0) { value = byte & 0x0f; pending = 2; } + else if ((byte & 0xf8) == 0xf0) { value = byte & 0x07; pending = 3; } + else if (byte >= 0xfc) { value = byte & 0x03; pending = 5; } + else if ((byte & 0xfc) == 0xf8) { value = byte & 0x03; pending = 4; } + else return false; + return true; +} + +} extern "C" { @@ -51,6 +99,118 @@ std::size_t APS5_VABI wcrtomb_nid_postfix(char* destination, std::uint16_t value return 1; } +int APS5_VABI mblen_nid_postfix(const char* source, std::size_t count) { + return mbtowc_nid_postfix(nullptr, source, count); +} + +int APS5_VABI wctomb_nid_postfix(char* destination, std::uint16_t value) { + if (!destination) return 0; + const auto result = wcrtomb_nid_postfix(destination, value, nullptr); + return result == static_cast<std::size_t>(-1) ? -1 : static_cast<int>(result); +} + +std::size_t APS5_VABI c32rtomb_nid_postfix(char* destination, std::uint32_t value, void*) { + char scratch[8]; + if (!destination) { + destination = scratch; + value = 0; + } + return EncodeUtf8(destination, value); +} + +std::size_t APS5_VABI c16rtomb_nid_postfix(char* destination, std::uint16_t unit, void* state) { + if (!state) throw std::invalid_argument("c16rtomb: null conversion state"); + char scratch[8]; + if (!destination) { + destination = scratch; + unit = 0; + } + auto* const bytes = static_cast<unsigned char*>(state); + std::uint64_t value = unit; + if (bytes[PendingOffset] != 0) { + if (static_cast<std::uint16_t>(unit + 0x2000) < 0xfc00) return static_cast<std::size_t>(-1); + std::uint64_t high = 0; + std::memcpy(&high, bytes, sizeof(high)); + value = high | (unit & 0x3ffu); + StorePending(bytes, 0); + } else if (static_cast<std::uint16_t>(unit + 0x2400) >= 0xfc00) { + const std::uint64_t high = ((static_cast<std::uint64_t>(unit) << 10) + 0x10000) & 0x1ffc00; + std::memcpy(bytes, &high, sizeof(high)); + StorePending(bytes, 1); + return 0; + } + return EncodeUtf8(destination, value); +} + +std::size_t APS5_VABI mbrtoc32_nid_postfix(std::uint32_t* destination, const char* source, std::size_t count, void* state) { + if (!state) throw std::invalid_argument("mbrtoc32: null conversion state"); + auto* const bytes = static_cast<unsigned char*>(state); + if (!source) { + destination = nullptr; + source = ""; + count = 1; + } + std::uint64_t value = 0; + std::memcpy(&value, bytes, sizeof(value)); + auto pending = static_cast<std::int8_t>(bytes[PendingOffset]); + for (std::size_t consumed = 0; consumed < count;) { + const auto byte = static_cast<unsigned char>(source[consumed++]); + if (!DecodeUtf8Byte(byte, value, pending)) { + errno = 86; + return static_cast<std::size_t>(-1); + } + if (pending == 0) { + if (destination) *destination = static_cast<std::uint32_t>(value); + StorePending(bytes, 0); + return value == 0 ? 0 : consumed; + } + } + std::memcpy(bytes, &value, sizeof(value)); + StorePending(bytes, pending); + return static_cast<std::size_t>(-2); +} + +std::size_t APS5_VABI mbrtoc16_nid_postfix(std::uint16_t* destination, const char* source, std::size_t count, void* state) { + if (!state) throw std::invalid_argument("mbrtoc16: null conversion state"); + auto* const bytes = static_cast<unsigned char*>(state); + if (!source) { + destination = nullptr; + source = ""; + count = 1; + } + std::uint64_t value = 0; + std::memcpy(&value, bytes, sizeof(value)); + auto pending = static_cast<std::int8_t>(bytes[PendingOffset]); + if (pending == LowSurrogatePending) { + if (destination) *destination = static_cast<std::uint16_t>((value & 0x3ff) | 0xdc00); + StorePending(bytes, 0); + return static_cast<std::size_t>(-3); + } + for (std::size_t consumed = 0; consumed < count;) { + const auto byte = static_cast<unsigned char>(source[consumed++]); + if (!DecodeUtf8Byte(byte, value, pending)) { + errno = 86; + return static_cast<std::size_t>(-1); + } + if (pending != 0) continue; + if (value >= 0x110000) { + errno = 86; + return static_cast<std::size_t>(-1); + } + if (value >= 0x10000) { + if (destination) *destination = static_cast<std::uint16_t>(((value >> 10) + 0x3fc0) | 0xd800); + StorePending(bytes, LowSurrogatePending); + return consumed; + } + if (destination) *destination = static_cast<std::uint16_t>(value); + StorePending(bytes, 0); + return value == 0 ? 0 : consumed; + } + std::memcpy(bytes, &value, sizeof(value)); + StorePending(bytes, pending); + return static_cast<std::size_t>(-2); +} + std::size_t APS5_VABI wcsrtombs_nid_postfix(char* destination, const std::uint16_t** source, std::size_t capacity, void* state) { const auto* wide = *source; std::size_t converted = 0; diff --git a/core/libs/tests/GuestWideStrings.cpp b/core/libs/tests/GuestWideStrings.cpp index 39b16ff31e..ceb09aaa92 100644 --- a/core/libs/tests/GuestWideStrings.cpp +++ b/core/libs/tests/GuestWideStrings.cpp @@ -8,8 +8,16 @@ #include <cmath> #include <cstdint> #include <cstring> +#include <string> extern "C" { +std::size_t APS5_VABI c32rtomb_nid_postfix(char* destination, std::uint32_t value, void* state); +std::size_t APS5_VABI c16rtomb_nid_postfix(char* destination, std::uint16_t unit, void* state); +std::size_t APS5_VABI mbrtoc32_nid_postfix(std::uint32_t* destination, const char* source, std::size_t count, void* state); +std::size_t APS5_VABI mbrtoc16_nid_postfix(std::uint16_t* destination, const char* source, std::size_t count, void* state); +int APS5_VABI mblen_nid_postfix(const char* source, std::size_t count); +int APS5_VABI wctomb_nid_postfix(char* destination, std::uint16_t value); +int* APS5_VABI __error_nid_postfix(); const char16_t* APS5_VABI wmemchr_nid_postfix(const char16_t* s, char16_t c, std::size_t n); int APS5_VABI wmemcmp_nid_postfix(const char16_t* s1, const char16_t* s2, std::size_t n); char16_t* APS5_VABI wmemcpy_nid_postfix(char16_t* dest, const char16_t* src, std::size_t n); @@ -55,6 +63,54 @@ bool same(const char16_t* left, const char16_t* right, std::size_t count) { } int main() { + { + const auto bytesOf = [](const char* text, std::size_t count) { return std::string(text, count); }; + char out[8] = {}; + require(c32rtomb_nid_postfix(out, 0x41, nullptr) == 1 && out[0] == 'A'); + require(c32rtomb_nid_postfix(out, 0xe9, nullptr) == 2 && bytesOf(out, 2) == "\xc3\xa9"); + require(c32rtomb_nid_postfix(out, 0x20ac, nullptr) == 3 && bytesOf(out, 3) == "\xe2\x82\xac"); + require(c32rtomb_nid_postfix(out, 0x1f600, nullptr) == 4 && bytesOf(out, 4) == "\xf0\x9f\x98\x80"); + require(c32rtomb_nid_postfix(out, 0x7fffffff, nullptr) == 6 && bytesOf(out, 6) == "\xfd\xbf\xbf\xbf\xbf\xbf"); + require(c32rtomb_nid_postfix(nullptr, 0x20ac, nullptr) == 1); + + alignas(8) unsigned char state[16] = {}; + require(c16rtomb_nid_postfix(out, 0xd83d, state) == 0 && state[10] == 1); + require(c16rtomb_nid_postfix(out, 0x41, state) == static_cast<std::size_t>(-1) && state[10] == 1); + require(c16rtomb_nid_postfix(nullptr, 0x41, state) == static_cast<std::size_t>(-1)); + require(c16rtomb_nid_postfix(out, 0xde00, state) == 4 && bytesOf(out, 4) == "\xf0\x9f\x98\x80" && state[10] == 0); + require(c16rtomb_nid_postfix(out, 0xdc00, state) == 3 && bytesOf(out, 3) == "\xed\xb0\x80"); + require(c16rtomb_nid_postfix(out, 0x7f, state) == 1 && out[0] == 0x7f); + + std::memset(state, 0, sizeof(state)); + std::uint32_t wide = 0; + require(mbrtoc32_nid_postfix(&wide, "\xf0\x9f\x98\x80z", 5, state) == 4 && wide == 0x1f600); + require(mbrtoc32_nid_postfix(&wide, "\xe2\x82", 2, state) == static_cast<std::size_t>(-2) && state[10] == 1); + require(mbrtoc32_nid_postfix(&wide, "\xac", 1, state) == 1 && wide == 0x20ac && state[10] == 0); + require(mbrtoc32_nid_postfix(&wide, "", 1, state) == 0 && wide == 0); + require(mbrtoc32_nid_postfix(&wide, "A", 0, state) == static_cast<std::size_t>(-2)); + wide = 7; + require(mbrtoc32_nid_postfix(&wide, nullptr, 9, state) == 0 && wide == 7); + *__error_nid_postfix() = 0; + require(mbrtoc32_nid_postfix(&wide, "\x80", 1, state) == static_cast<std::size_t>(-1) && *__error_nid_postfix() == 86); + require(mbrtoc32_nid_postfix(&wide, "\xfd\xbf\xbf\xbf\xbf\xbf", 6, state) == 6 && wide == 0x7fffffff); + + std::memset(state, 0, sizeof(state)); + std::uint16_t unit = 0; + require(mbrtoc16_nid_postfix(&unit, "\xc3\xa9", 2, state) == 2 && unit == 0xe9); + require(mbrtoc16_nid_postfix(&unit, "\xf0\x9f", 2, state) == static_cast<std::size_t>(-2)); + require(mbrtoc16_nid_postfix(&unit, "\x98\x80", 2, state) == 2 && unit == 0xd83d && state[10] == 0x10); + require(mbrtoc16_nid_postfix(&unit, "A", 1, state) == static_cast<std::size_t>(-3) && unit == 0xdc1f && state[10] == 0); + std::memset(state, 0, sizeof(state)); + require(mbrtoc16_nid_postfix(&unit, "\xf0\x9f\x98\x80", 4, state) == 4 && unit == 0xd83d); + require(mbrtoc16_nid_postfix(&unit, nullptr, 0, state) == static_cast<std::size_t>(-3) && unit == 0xd83d); + *__error_nid_postfix() = 0; + require(mbrtoc16_nid_postfix(&unit, "\xf4\x90\x80\x80", 4, state) == static_cast<std::size_t>(-1) && *__error_nid_postfix() == 86); + + require(mblen_nid_postfix(nullptr, 1) == 0 && mblen_nid_postfix("ab", 2) == 1 && mblen_nid_postfix("", 1) == 0); + require(wctomb_nid_postfix(nullptr, 0x41) == 0); + require(wctomb_nid_postfix(out, 0x42) == 1 && out[0] == 'B'); + require(wctomb_nid_postfix(out, 0x263a) == -1); + } const char16_t split[] = {u'a', u'b', 0, u'c', 0}; require(wcslen_nid_postfix(u"") == 0); require(wcslen_nid_postfix(u"héllo") == 5); diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 497130f640..c36b632a21 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -72,6 +72,7 @@ - [getifaddrs](../../../core/libs/prx/libc/src/Interfaces.cpp) (libc) lists the host's IPv4 and IPv6 addresses in FreeBSD's `ifaddrs` and sockaddr layouts. FreeBSD's per-interface AF_LINK entries (MAC, index, MTU) and their `if_data` are not produced, so interfaces without an address are omitted. Names are the host's (Linux `lo`, `eth0`; Windows NDIS names such as `ethernet_32769`, or `if<index>` when one does not fit 15 bytes), not the console's. A Linux point-to-point address without a peer loses `IFF_POINTOPOINT`, and an IPv4 broadcast address the host omits is computed from the netmask. On Windows the flags are only `IFF_UP`, `IFF_RUNNING`, `IFF_LOOPBACK`, `IFF_BROADCAST` and `IFF_MULTICAST`, derived from the adapter: PPP and tunnel adapters get neither `IFF_BROADCAST` nor `IFF_POINTOPOINT` because their peer address is not reported, and an address whose prefix length the host reports as unspecified gets a host mask. - [gethostname](../../../core/libs/prx/libc/src/Hostname.cpp) (libc) reports the fixed guest name `AnyPS5` without querying the host machine name. Buffer copying, null queries and `ENAMETOOLONG` follow [FreeBSD gethostname](https://github.com/freebsd/freebsd-src/blob/main/lib/libc/gen/gethostname.c); the console's configurable hostname and its integration with `sysctl` and name resolution are unverified and not implemented. - [wcsrtombs_s](../../../core/libs/prx/libc/src/Multibyte.cpp) (libc) follows the console's `libc.prx`, but a runtime-constraint violation only returns its error: the default handler there writes the message and a newline to `stderr` before returning, which is not done here. Conversion uses the C locale of `wcrtomb` (`MB_CUR_MAX` 1), so the path where a multibyte character does not fit the remaining space is not reached +- [c16rtomb, c32rtomb, mbrtoc16 and mbrtoc32](../../../core/libs/prx/libc/src/Multibyte.cpp) (libSceLibcInternal, implemented in libc) follow the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x7D480 to 0x7D920), which converts to and from UTF-8 whatever the locale, with the original sequences of up to six bytes and no check for surrogates or for values above 0x10FFFF when encoding. The conversion state keeps the value in its first eight bytes and the count of bytes still expected (or 0x10 for a pending low surrogate, 1 for a stored high surrogate) as 16 bits at offset 0xA; a null state, which the console dereferences, throws. As on the console, `mbrtoc16` does not store the complete code point when it returns a high surrogate, so the low surrogate of the next call is built from the value the state last held, which is the partial value of an earlier incomplete call or whatever the caller initialised it to, `c16rtomb` returns -1 without setting `errno` when a high surrogate is not followed by a low one, and `mbrtoc16` and `mbrtoc32` report `EILSEQ` as 86. Measured on a PS5 (firmware 13.60) from a homebrew payload, the system `libSceLibcInternal`'s default locale also converts UTF-8 in `wcrtomb`, `wctomb`, `mbrtowc` and `mblen` (`wctomb` of U+263A gives `E2 98 BA`, `mblen` of a lone 0xE9 gives -1); this libc keeps its single-byte C locale, so [wctomb and mblen](../../../core/libs/prx/libc/src/Multibyte.cpp), which call `wcrtomb` and `mbtowc` here as they call the same internal routines on the console, share that difference. Titles bundle their own `libc.prx`, whose locale was not measured - [The narrow printf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc) converts 16-bit guest wide arguments to UTF-8 on both platforms, independently of the host locale. It does not use the guest's single-byte `wcrtomb`; the console's treatment of invalid UTF-16 and locale-specific encoding remains unverified. Linux retains host formatting for formats without wide arguments or `%m`, so a narrow `%p` and a NaN there still follow the host (glibc prints a null pointer as `(nil)`, where FreeBSD prints `0x0`, and a NaN with its sign, `-nan`, and with the `+` and space flags, where FreeBSD prints `nan`); argument positions above 99 (FreeBSD's `NL_ARGMAX`) throw when a format requires the guest formatter - [collate<char> and collate<wchar_t> _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build the facet for the C locale only and throw for any other locale name, the wide entry point following the narrow one it sits next to. The wide facet carries its own vtable, whose `compare`, `transform` and `hash` take the guest's 16-bit characters, so the length of a range is a count of characters: `compare` orders by 16-bit value (the C locale, as Dinkumware's `char_traits<wchar_t>::compare` does), `transform` copies the characters into the result and records size and capacity in 16-bit elements (inline capacity 7, heap capacity `length | 7`, `(capacity + 1) * 2` bytes including the terminator), following Dinkumware's [VC12 `basic_string`](https://github.com/ganboing/crt.vc12/blob/f1d47126bf13d1c9dccaa6c53b72217fd0474968/src/xstring#L500-L531), and `hash` mixes one whole character per round, so wide text below 0x100 hashes like its narrow spelling and a character above 0xff hashes as one value instead of the two bytes it holds. The console's `libc.prx` exports both `_Getcat` (NIDs BSVJqITGCyI and 2wz4rthdiy8) with identical 0x19d-byte bodies, believed to build the same 0x20-byte facet; the console bodies themselves are unreadable (obfuscated dynsym, and the text of the relinked `libc.prx` is zeroed), so the wide entries follow the narrow algorithm and the C locale instead of the console's wide code, and no console run checks the two. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size. The narrow result capacity (`length | 15`) and the over-aligned storage of results of 4080 characters or more follow the console's `libc.prx` code and are not checked on hardware. No module of the PPSA59322, PPSA76677 or PPSA97358 dumps in this tree imports either `_Getcat`, so neither has been exercised by a title. - [The scanf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc), the `_s` forms included, on Linux throws for the 16-bit wide conversions (`%ls`, `%lc`, `%l[`, `%S`, `%C`): glibc would store a 32-bit `wchar_t` per character into the guest's 16-bit buffer. Windows hands `%ls`, `%lc` and `%l[` to the host, whose `wchar_t` is 16 bits From f68ab450e2a793c177f786ddbe9644bb12ab03e2 Mon Sep 17 00:00:00 2001 From: T3cm0 <srodriguez7604@gmail.com> Date: Sat, 17 Oct 2026 20:44:06 -0500 Subject: [PATCH 100/103] feat(libc): implement wcscspn, wcstoimax, wcstoumax, strtouq, rindex and flsl Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libc/src/Strings.cpp | 26 ++++++++++++++++++++++++++ core/libs/tests/GuestWideStrings.cpp | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/core/libs/prx/libc/src/Strings.cpp b/core/libs/prx/libc/src/Strings.cpp index 1a4215cf35..225d81c4a7 100644 --- a/core/libs/prx/libc/src/Strings.cpp +++ b/core/libs/prx/libc/src/Strings.cpp @@ -152,6 +152,14 @@ char* APS5_VABI strrchr_nid_postfix(const char* s, int c) { return std::strrchr(const_cast<char*>(s), c); } +char* APS5_VABI rindex_nid_postfix(const char* s, int c) { + return strrchr_nid_postfix(s, c); +} + +int APS5_VABI flsl_nid_postfix(std::int64_t mask) { + return mask == 0 ? 0 : 64 - std::countl_zero(static_cast<std::uint64_t>(mask)); +} + char* APS5_VABI strstr_nid_postfix(const char* haystack, const char* needle) { return std::strstr(const_cast<char*>(haystack), needle); } @@ -200,6 +208,10 @@ unsigned long long APS5_VABI strtoull_nid_postfix(const char* str, char** endptr return std::strtoull(str, endptr, base); } +unsigned long long APS5_VABI strtouq_nid_postfix(const char* str, char** endptr, int base) { + return strtoull_nid_postfix(str, endptr, base); +} + double APS5_VABI strtod_nid_postfix(const char* str, char** endptr) { return ParseFloat<double>(str, endptr, std::strtod); } @@ -478,6 +490,12 @@ size_t APS5_VABI wcsspn_nid_postfix(const char16_t* s, const char16_t* accept) { return count; } +size_t APS5_VABI wcscspn_nid_postfix(const char16_t* s, const char16_t* reject) { + size_t count = 0; + while (s[count] != 0 && !Contains(reject, s[count])) ++count; + return count; +} + char16_t* APS5_VABI wcstok_nid_postfix(char16_t* text, const char16_t* delimiters, char16_t** last) { if (text == nullptr) text = *last; if (text == nullptr) return nullptr; @@ -532,6 +550,14 @@ unsigned long long APS5_VABI wcstoull_nid_postfix(const char16_t* str, char16_t* return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return StopAtPrefixLetter_nid_no_patch(text, end, base) ? 0ULL : std::strtoull(text, end, base); }); } +long long APS5_VABI wcstoimax_nid_postfix(const char16_t* str, char16_t** endptr, int base) { + return wcstoll_nid_postfix(str, endptr, base); +} + +unsigned long long APS5_VABI wcstoumax_nid_postfix(const char16_t* str, char16_t** endptr, int base) { + return wcstoull_nid_postfix(str, endptr, base); +} + std::uint64_t APS5_VABI _WStoul_nid_postfix(const char16_t* str, char16_t** endptr, int base) { if (base < 0 || base == 1 || base > 36) throw std::invalid_argument("_WStoul: unsupported base " + std::to_string(base)); return wcstoul_nid_postfix(str, endptr, base); diff --git a/core/libs/tests/GuestWideStrings.cpp b/core/libs/tests/GuestWideStrings.cpp index 39b16ff31e..4502e1fe81 100644 --- a/core/libs/tests/GuestWideStrings.cpp +++ b/core/libs/tests/GuestWideStrings.cpp @@ -26,6 +26,7 @@ const char16_t* APS5_VABI wcsrchr_nid_postfix(const char16_t* s, char16_t c); const char16_t* APS5_VABI wcsstr_nid_postfix(const char16_t* haystack, const char16_t* needle); const char16_t* APS5_VABI wcspbrk_nid_postfix(const char16_t* s, const char16_t* accept); std::size_t APS5_VABI wcsspn_nid_postfix(const char16_t* s, const char16_t* accept); +std::size_t APS5_VABI wcscspn_nid_postfix(const char16_t* s, const char16_t* reject); char16_t* APS5_VABI wcstok_nid_postfix(char16_t* text, const char16_t* delimiters, char16_t** last); char16_t* APS5_VABI wmemset_nid_postfix(char16_t* s, char16_t c, std::size_t n); double APS5_VABI wcstod_nid_postfix(const char16_t* str, char16_t** endptr); @@ -33,6 +34,11 @@ float APS5_VABI wcstof_nid_postfix(const char16_t* str, char16_t** endptr); long double APS5_VABI wcstold_nid_postfix(const char16_t* str, char16_t** endptr); long long APS5_VABI wcstol_nid_postfix(const char16_t* str, char16_t** endptr, int base); long long APS5_VABI wcstoll_nid_postfix(const char16_t* str, char16_t** endptr, int base); +long long APS5_VABI wcstoimax_nid_postfix(const char16_t* str, char16_t** endptr, int base); +unsigned long long APS5_VABI wcstoumax_nid_postfix(const char16_t* str, char16_t** endptr, int base); +unsigned long long APS5_VABI strtouq_nid_postfix(const char* str, char** endptr, int base); +char* APS5_VABI rindex_nid_postfix(const char* s, int c); +int APS5_VABI flsl_nid_postfix(std::int64_t mask); unsigned long long APS5_VABI wcstoul_nid_postfix(const char16_t* str, char16_t** endptr, int base); unsigned long long APS5_VABI wcstoull_nid_postfix(const char16_t* str, char16_t** endptr, int base); int APS5_VABI wcscoll_nid_postfix(const char16_t* first, const char16_t* second); @@ -118,6 +124,7 @@ int main() { require(wcspbrk_nid_postfix(haystack, u"xyz") == nullptr); require(wcsspn_nid_postfix(u"aabbc", u"ab") == 4); require(wcsspn_nid_postfix(u"abc", u"") == 0); + require(wcscspn_nid_postfix(u"abcde", u"dx") == 3 && wcscspn_nid_postfix(u"abc", u"") == 3 && wcscspn_nid_postfix(u"", u"a") == 0 && wcscspn_nid_postfix(u"abc", u"a") == 0); char16_t tokens[] = u",,one,,two;three,"; char16_t* last = tokens; @@ -238,5 +245,21 @@ int main() { require(same(transformed, test.expected, test.written)); for (std::size_t index = test.written; index < 8; ++index) require(transformed[index] == 0xaaaa); } + { + const char16_t number[] = u" -0x7fffffffffffffff9"; + char16_t* wideEnd = nullptr; + require(wcstoimax_nid_postfix(number, &wideEnd, 0) == INT64_MIN && wideEnd == number + 22); + const char16_t positive[] = u"+42z"; + require(wcstoumax_nid_postfix(positive, &wideEnd, 10) == 42 && wideEnd == positive + 3); + require(wcstoumax_nid_postfix(u"-1", nullptr, 10) == UINT64_MAX); + char* narrowEnd = nullptr; + const char text[] = "0x1ffffffffffffffffg"; + require(strtouq_nid_postfix(text, &narrowEnd, 16) == UINT64_MAX && narrowEnd == text + 19); + const char path[] = "a/b/c"; + require(rindex_nid_postfix(path, '/') == path + 3 && rindex_nid_postfix(path, 'x') == nullptr); + require(rindex_nid_postfix(path, 0) == path + 5); + require(flsl_nid_postfix(0) == 0 && flsl_nid_postfix(1) == 1 && flsl_nid_postfix(0x80) == 8); + require(flsl_nid_postfix(0x100000000ll) == 33 && flsl_nid_postfix(INT64_MIN) == 64 && flsl_nid_postfix(-1) == 64); + } return 0; } From 26898755e5c62e0da6a283fb6fccb0b2f98c1814 Mon Sep 17 00:00:00 2001 From: T3cm0 <srodriguez7604@gmail.com> Date: Sat, 17 Oct 2026 20:44:32 -0500 Subject: [PATCH 101/103] feat(libc): implement seed48 and lcong48 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/libs/prx/libc/src/Random.cpp | 28 +++++++++++++++++++++++++--- core/libs/tests/GuestRandomState.cpp | 19 +++++++++++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libc/src/Random.cpp b/core/libs/prx/libc/src/Random.cpp index 49b48e16ca..787fe00b2b 100644 --- a/core/libs/prx/libc/src/Random.cpp +++ b/core/libs/prx/libc/src/Random.cpp @@ -96,14 +96,17 @@ void SaveTypeWordLocked() { } std::uint16_t g_rand48Seed[3]{0x330e, 0xabcd, 0x1234}; +constexpr std::uint64_t DefaultRand48Multiplier = 0x5deece66dull; +constexpr std::uint16_t DefaultRand48Addend = 0xb; +std::uint64_t g_rand48Multiplier = DefaultRand48Multiplier; +std::uint16_t g_rand48Addend = DefaultRand48Addend; +std::uint16_t g_seed48Result[3]{}; std::uint64_t NextRand48(std::uint16_t* seed, const char* function) { if (seed == nullptr) throw std::invalid_argument(std::string(function) + ": null seed"); - constexpr std::uint64_t Multiplier = 0x5deece66dull; - constexpr std::uint64_t Addend = 0xb; constexpr std::uint64_t Mask = (1ull << 48) - 1; std::uint64_t value = (static_cast<std::uint64_t>(seed[2]) << 32) | (static_cast<std::uint64_t>(seed[1]) << 16) | seed[0]; - value = (value * Multiplier + Addend) & Mask; + value = (value * g_rand48Multiplier + g_rand48Addend) & Mask; seed[0] = static_cast<std::uint16_t>(value); seed[1] = static_cast<std::uint16_t>(value >> 16); seed[2] = static_cast<std::uint16_t>(value >> 32); @@ -146,6 +149,25 @@ void APS5_VABI srand48_nid_postfix(std::int64_t seedval) { g_rand48Seed[0] = 0x330e; g_rand48Seed[1] = static_cast<std::uint16_t>(static_cast<std::uint32_t>(seedval)); g_rand48Seed[2] = static_cast<std::uint16_t>(static_cast<std::uint32_t>(seedval) >> 16); + g_rand48Multiplier = DefaultRand48Multiplier; + g_rand48Addend = DefaultRand48Addend; +} + +std::uint16_t* APS5_VABI seed48_nid_postfix(const std::uint16_t* seed) { + if (seed == nullptr) throw std::invalid_argument("seed48: null seed"); + std::lock_guard lock(g_randomLock); + for (int index = 0; index < 3; ++index) g_rand48Seed[index] = g_seed48Result[index] = seed[index]; + g_rand48Multiplier = DefaultRand48Multiplier; + g_rand48Addend = DefaultRand48Addend; + return g_seed48Result; +} + +void APS5_VABI lcong48_nid_postfix(const std::uint16_t* parameters) { + if (parameters == nullptr) throw std::invalid_argument("lcong48: null parameters"); + std::lock_guard lock(g_randomLock); + for (int index = 0; index < 3; ++index) g_rand48Seed[index] = parameters[index]; + g_rand48Multiplier = parameters[3] | (static_cast<std::uint64_t>(parameters[4]) << 16) | (static_cast<std::uint64_t>(parameters[5]) << 32); + g_rand48Addend = parameters[6]; } std::int64_t APS5_VABI random_nid_postfix() { diff --git a/core/libs/tests/GuestRandomState.cpp b/core/libs/tests/GuestRandomState.cpp index 59b336db38..6f2ab74c7d 100644 --- a/core/libs/tests/GuestRandomState.cpp +++ b/core/libs/tests/GuestRandomState.cpp @@ -23,6 +23,8 @@ double APS5_VABI drand48_nid_postfix(); std::int64_t APS5_VABI lrand48_nid_postfix(); std::int64_t APS5_VABI mrand48_nid_postfix(); void APS5_VABI srand48_nid_postfix(std::int64_t); +std::uint16_t* APS5_VABI seed48_nid_postfix(const std::uint16_t*); +void APS5_VABI lcong48_nid_postfix(const std::uint16_t*); } static int failures = 0; @@ -74,6 +76,23 @@ static void CheckRand48() { Require(mrand48_nid_postfix() == jrand48_nid_postfix(expectedSeed), "mrand48 matches jrand48 on initialized seed"); Require(drand48_nid_postfix() == erand48_nid_postfix(expectedSeed), "drand48 fourth value matches erand48"); + const std::uint16_t parameters[7] = {1, 0, 0, 3, 0, 0, 5}; + lcong48_nid_postfix(parameters); + Require(lrand48_nid_postfix() == 0 && mrand48_nid_postfix() == 0, "lcong48 sets seed, multiplier and addend"); + std::uint16_t high[3] = {0, 0, 1}; + Require(jrand48_nid_postfix(high) == 196608 && SeedIs(high, 5, 0, 3), "lcong48 multiplier scales the seed"); + std::uint16_t custom[3] = {2, 0, 0}; + Require(nrand48_nid_postfix(custom) == 0 && SeedIs(custom, 11, 0, 0), "lcong48 parameters apply to nrand48"); + const std::uint16_t newSeed[3] = {0x330e, 0xabcd, 0x1234}; + std::uint16_t* const echoed = seed48_nid_postfix(newSeed); + Require(SeedIs(echoed, 0x330e, 0xabcd, 0x1234), "seed48 returns a copy of the new seed"); + std::uint16_t again[3] = {0x330e, 0xabcd, 0x1234}; + Require(drand48_nid_postfix() == erand48_nid_postfix(again), "seed48 restores the default multiplier and addend"); + lcong48_nid_postfix(parameters); + srand48_nid_postfix(0x1234abcd); + std::uint16_t reset[3] = {0x330e, 0xabcd, 0x1234}; + Require(lrand48_nid_postfix() == nrand48_nid_postfix(reset), "srand48 restores the default multiplier and addend"); + #ifndef _WIN32 std::uint16_t guest[3] = {0x1234, 0x5678, 0x9abc}; unsigned short host[3] = {0x1234, 0x5678, 0x9abc}; diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 497130f640..4d0addbf70 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -54,6 +54,7 @@ - [realpath](../../../core/libs/prx/libc/src/RealPath.cpp) (libc) follows FreeBSD 11's [`realpath.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/libc/stdlib/realpath.c) except for symlinks. The console's guest filesystem has none, so a host symlink under the guest root is followed for existence and type but not expanded, and `..` drops the previous component lexically, as `Resolve` does for every guest path. Through a host link `linkdir -> sub/deep`, `linkdir/../x` gives `x` beside the link, not beside its target. - [strtod, strtof, strtold](../../../core/libs/prx/libc/src/Strings.cpp) (libc) - `NAN(n-char-sequence)` converts to the quiet NaN that `nan` and `nanf` return, negated after a minus sign, and ignores the sequence as they ignore their tag. This is inferred from C17 7.12.11.2p2, which makes `nan("n")` equivalent to `strtod("NAN(n)", NULL)`, and not observed on the console. No console `nanl` was observed, so `strtold` returns the x87 default quiet NaN (exponent `0x7fff`, significand `0xc000000000000000`) - [nan and nanf](../../../core/libs/prx/libc/src/Math.cpp) (libc) return fixed quiet NaN values (`0x7fc00000` and `0x7ff8000000000000`), ignoring the tag argument as observed in the console's `libc.prx` (both exports load a global constant via `mov rax,[rip+..]; vmovsd xmm0,[rax]; ret`); VLC (PPSA85300) stops on nanf at start-up after strsep. +- [seed48 and lcong48](../../../core/libs/prx/libc/src/Random.cpp) (libSceLibcInternal, implemented in libc) follow the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x6F460 and 0x6F420): the multiplier and addend are shared by the whole `rand48` family, including `erand48`, `nrand48` and `jrand48` on a caller's seed; `lcong48` sets them with the seed, and `srand48` and `seed48` restore 0x5DEECE66D and 0xB. Unlike FreeBSD, `seed48` returns a static copy of the new seed rather than the previous one. A null argument, which the console dereferences, throws - [lgamma and signgam](../../../core/libs/prx/libc/src/Math.cpp) (libc) - `lgamma` returns the host's `lgamma` and stores the sign of Γ(x) in the exported `signgam` (NID `5gOOC0kzW0c`, starting at 0 as in FreeBSD 11's [`s_signgam.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/msun/src/s_signgam.c)) by the rules of [`e_lgamma_r.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/msun/src/e_lgamma_r.c): -1 for -0 and for a negative non-integer whose floor is odd, 1 otherwise, including poles, infinities and NaN. Taken from the FreeBSD source, not measured on the console. The NID is in the aerolib symbol list; no title is known to read `signgam`. PPSA99004 imports `lgamma` - [sinhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `sinhf` of the `libc.prx` bundled with PPSA28319 (disassembly): FreeBSD msun's [`e_sinhf.c`](https://github.com/freebsd/freebsd-src/blob/5aa554c7e5b6fd9042986df07f12ac5687e41392/lib/msun/src/e_sinhf.c) from before `__ldexp_expf`, which squares `expf(|x| / 2)` between `logf(FLT_MAX)` and the overflow threshold, calling msun's [`expm1f`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/s_expm1f.c) and [`expf`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_expf.c), whose paths for the arguments it passes are inlined here. Past the threshold Sony's error helper sets `errno` to `ERANGE` before returning `x * 1e37`; msun itself sets no `errno`. That code, run on a Windows host, gives the same bits as this implementation for every finite argument up to the threshold, of both signs; it was not run on Linux. That the system `libSceLibcInternal`, which Ruffle (PPSA68091) and RetroArch (PPSA99169) import while bundling a zero-return `libc.prx`, has the same code is assumed - [asinl](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `asinl` of the `libc.prx` bundled with PPSA28319 (disassembly), FreeBSD msun's [`e_asinl.c`](https://github.com/freebsd/freebsd-src/blob/9f0432041b8fb69832f10c32da50a67f34feeb61/lib/msun/src/e_asinl.c) with the ld80 [`invtrig.c`](https://github.com/freebsd/freebsd-src/blob/9f0432041b8fb69832f10c32da50a67f34feeb61/lib/msun/ld80/invtrig.c) coefficients, which the binary holds as the same 80-bit constants. For |x| > 1, the infinities and NaN, Sony's error helper raises `FE_INVALID` and sets `errno` to `EDOM` before returning `(x - x) / (x - x)`, so a NaN argument also sets `EDOM`; msun itself sets no `errno`. 20 million random arguments in (-1, 1) give the same bits as that code run on a Windows host. The result needs the x87 precision control at 64 bits, as FreeBSD amd64 starts threads (control word `0x37f`), but guest threads do not set it: on Windows a thread started by `_beginthreadex` runs at 53 bits (`0x27f`), where `asinl(0.75)` returns mantissa `0xd91a98ae3406e000` instead of `0xd91a98ae3406e041`. The system `libSceLibcInternal` that RetroArch (PPSA99169) imports is assumed to have the same code From e273a65352f5f4caedde0f34e0f81394da606414 Mon Sep 17 00:00:00 2001 From: milescoviello <kitslayer29@gmail.com> Date: Sun, 18 Oct 2026 01:22:30 +0000 Subject: [PATCH 102/103] perf(shader): initialize the sRGB decode table once per module DecodeSrgbTexel looked up the 256 table constants and stored the whole table into its Private array at every decode site. The constants are interned, but once a module has more than the declaration cache's 512 entries, each of those lookups that misses walks every slot before it reaches the map. The table is now built on first use as the Private array's initializer, as the buffer format table already is, and later sites only index it. The sRGB image load test checks that the recompiled module holds one table, initializes one variable with it and never stores it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../tests/execution/ImageSrgbLoad.cpp | 24 +++++++++++++++++-- .../SpirvBackend/src/SpirvImageEmitter.cpp | 16 +++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp index ec759a3f7c..eafabe0d7c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp @@ -156,7 +156,26 @@ void Fill(std::uint32_t format) { } } -void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const void* texels, std::uint32_t format, std::uint32_t swizzle, std::uint32_t groups, bool useCache = false) { +void RequireInitializedTable(std::span<const std::uint32_t> words, const std::string& what) { + std::vector<std::uint32_t> tables; + std::size_t stores = 0; + std::size_t initialized = 0; + for (std::size_t at = 5; at < words.size();) { + const auto count = words[at] >> 16u; + const auto opcode = words[at] & 0xffffu; + Require(count != 0u && at + count <= words.size(), what + ": malformed SPIR-V"); + const auto operands = words.subspan(at + 1u, count - 1u); + const auto table = [&](std::uint32_t id) { return std::find(tables.begin(), tables.end(), id) != tables.end(); }; + if (opcode == 44u && operands.size() == 2u + 256u) tables.push_back(operands[1]); + if (opcode == 59u && operands.size() == 4u && table(operands[3])) ++initialized; + if (opcode == 62u && operands.size() >= 2u && table(operands[1])) ++stores; + at += count; + } + Require(tables.size() == 1u, what + ": the shader has " + std::to_string(tables.size()) + " sRGB tables, expected 1"); + Require(stores == 0u && initialized == 1u, what + ": the sRGB table is stored " + std::to_string(stores) + " times and initializes " + std::to_string(initialized) + " variables, expected no store and one variable"); +} + +std::vector<std::uint32_t> Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const void* texels, std::uint32_t format, std::uint32_t swizzle, std::uint32_t groups, bool useCache = false) { Output.fill(0xdeadbeefu); std::vector<std::uint32_t> userData(24, 0u); const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u)); @@ -176,6 +195,7 @@ void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, c const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, groups, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data())); device.WaitIdle(); + return result.spirv.Copy(); } void Verify(std::uint32_t dmask, std::uint32_t format, std::uint32_t swizzle, const std::string& what) { @@ -194,7 +214,7 @@ void Verify(std::uint32_t dmask, std::uint32_t format, std::uint32_t swizzle, co void CheckLoad(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t dmask, std::uint32_t format, std::uint32_t swizzle, const std::string& what) { Fill(format); - Run(device, code, Texels.data(), format, swizzle, Groups); + RequireInitializedTable(Run(device, code, Texels.data(), format, swizzle, Groups), what); Verify(dmask, format, swizzle, what); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 6b9f77a57e..c6d59e1b90 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -391,13 +391,19 @@ constexpr std::array<std::uint32_t, 256> SrgbToLinear{ 0x3f700000u, 0x3f730000u, 0x3f750000u, 0x3f770000u, 0x3f790000u, 0x3f7b0000u, 0x3f7e0000u, 0x3f800000u, }; -std::uint32_t DecodeSrgbTexel(SpirvEmitterState& state, std::uint32_t color) { - const auto pointerType = TypePointer(state, spv::StorageClassPrivate, TypeU32(state)); +std::uint32_t SrgbTable(SpirvEmitterState& state) { + if (state.srgbTableVariable != 0u) return state.srgbTableVariable; std::array<std::uint32_t, SrgbToLinear.size()> words{}; for (std::size_t index = 0; index < words.size(); ++index) words[index] = ConstantU32(state, SrgbToLinear[index]); const auto arrayType = state.module.Type(spv::OpTypeArray, TypeU32(state), ConstantU32(state, static_cast<std::uint32_t>(words.size()))); - if (state.srgbTableVariable == 0) state.srgbTableVariable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassPrivate, arrayType), spv::StorageClassPrivate); - state.module.AddFunction(spv::OpStore, state.srgbTableVariable, state.module.Constant(spv::OpConstantComposite, arrayType, std::span<const std::uint32_t>(words))); + const auto table = state.module.Constant(spv::OpConstantComposite, arrayType, std::span<const std::uint32_t>(words)); + state.srgbTableVariable = state.module.DefineInitializedGlobalVariable(TypePointer(state, spv::StorageClassPrivate, arrayType), spv::StorageClassPrivate, table); + return state.srgbTableVariable; +} + +std::uint32_t DecodeSrgbTexel(SpirvEmitterState& state, std::uint32_t color) { + const auto pointerType = TypePointer(state, spv::StorageClassPrivate, TypeU32(state)); + const auto table = SrgbTable(state); std::uint32_t components[4] = {}; for (std::uint32_t index = 0; index < 4u; index++) { const auto value = state.module.AllocateId(); @@ -406,7 +412,7 @@ std::uint32_t DecodeSrgbTexel(SpirvEmitterState& state, std::uint32_t color) { const auto code = state.module.AllocateId(); state.module.AddFunction(spv::OpExtInst, TypeU32(state), code, GlslStd450(state), GLSLstd450UMin, Unary(state, spv::OpConvertFToU, TypeU32(state), scaled), ConstantU32(state, 255u)); const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, state.srgbTableVariable, code); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, table, code); const auto bits = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, TypeU32(state), bits, pointer); components[index] = Unary(state, spv::OpBitcast, TypeF32(state), bits); From 7f9427a7636a23cad65a6bf3528420e911ad695d Mon Sep 17 00:00:00 2001 From: SandstromPL <paritoshlahre@gmail.com> Date: Fri, 16 Oct 2026 08:40:11 +0530 Subject: [PATCH 103/103] fix(test): sample the anisotropic min/max rows without a mip filter The two anisotropic rows of agc_driver_image_sample_reduction sampled a two-level texture at explicit LOD 0.25 and 0.75 through a point mip filter, and expected only the nearest level in the min or max. Vulkan leaves the anisotropic filtering scheme to the implementation, and only recommends that it consider mipmapMode, minLod and maxLod. On an RTX 4090 (NVIDIA 616.92) the min row returns 5, a level-1 texel, where level 0's bilinear minimum is 17, so the test fails on main there. The rows now use no mip filter, which the decoder turns into a host sampler with minLod and maxLod 0, so they read level 0 only, as the RDNA2 measurements without derivatives did (gfx1030 with no mip filter over a single-level texture, gfx1035 with image_sample_lz). They still throw if the min or max reduction with anisotropy is refused again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../libSceAgcDriver/tests/execution/ImageSampleReduction.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleReduction.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleReduction.cpp index a6536f8c9a..e2f8d2dc66 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleReduction.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleReduction.cpp @@ -304,8 +304,8 @@ int main() { Run(*device, {ReductionMax, FilterPoint, MipPoint}, "max, point, point mip"); Run(*device, {ReductionMin, FilterBilinear, MipNone}, "min, bilinear, no mip"); Run(*device, {ReductionMax, FilterPoint, MipNone}, "max, point, no mip"); - Run(*device, {ReductionMin, FilterAnisoBilinear, MipPoint, Aniso2x}, "min, 2x anisotropic bilinear, point mip"); - Run(*device, {ReductionMax, FilterAnisoBilinear, MipPoint, Aniso2x}, "max, 2x anisotropic bilinear, point mip"); + Run(*device, {ReductionMin, FilterAnisoBilinear, MipNone, Aniso2x}, "min, 2x anisotropic bilinear, no mip"); + Run(*device, {ReductionMax, FilterAnisoBilinear, MipNone, Aniso2x}, "max, 2x anisotropic bilinear, no mip"); Gather(*device, {ReductionMin, FilterBilinear, MipPoint}, "gather4_lz, min"); Gather(*device, {ReductionMax, FilterBilinear, MipPoint}, "gather4_lz, max"); RunWithColors(*device, loaded, LoadedCode, false, "min 32_FLOAT, image_load 8_8_8_8");