diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index aa0c2f1c43..61dd26a9a9 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2100,6 +2100,9 @@ if(BUILD_TESTING) set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES SKIP_RETURN_CODE 77 RUN_SERIAL TRUE WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 240) if(WIN32) set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + add_test(NAME agc_shader_disk_cache_long_paths COMMAND agc_shader_disk_cache_tests --long-host-paths) + set_tests_properties(agc_shader_disk_cache_long_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() if(WIN32) add_test(NAME agc_driver_untracked_snapshot_hint COMMAND agc_driver_partially_resident_snapshot_tests --untracked-hint) @@ -2142,12 +2145,17 @@ if(BUILD_TESTING) endif() set_tests_properties(agc_sparse_buffer_holes PROPERTIES TIMEOUT 30) add_test(NAME agc_pipeline_journal COMMAND agc_driver_graphics_tests --pipeline-journal) + add_test(NAME agc_pipeline_journal_long_paths COMMAND agc_driver_graphics_tests --pipeline-journal-long-paths) + set_tests_properties(agc_pipeline_journal_long_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_pipeline_library_clear_waits_for_build COMMAND agc_driver_graphics_tests --pipeline-library-clear-waits-for-build) add_test(NAME agc_pipeline_store_lazy_buffer_pool COMMAND agc_driver_graphics_tests --pipeline-store-lazy-buffer-pool) set_tests_properties(agc_pipeline_journal PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") set_tests_properties(agc_pipeline_library_clear_waits_for_build PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") set_tests_properties(agc_pipeline_store_lazy_buffer_pool PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) + add_test(NAME agc_pipeline_journal_extended_paths COMMAND agc_driver_graphics_tests --pipeline-journal-extended-paths) + set_tests_properties(agc_pipeline_journal_extended_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + set_tests_properties(agc_pipeline_journal_long_paths agc_pipeline_journal_extended_paths PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() set_tests_properties(agc_driver_sparse_buffer_holes PROPERTIES TIMEOUT 30) @@ -3157,6 +3165,6 @@ endif() if(BUILD_TESTING) get_directory_property(agcRegisteredTests TESTS) - list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal|agc_driver_pipeline_journal)$") + list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal)$") set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1") endif() diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp index 45481c3c5f..c7a0f679a5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp @@ -564,8 +564,15 @@ private: Journal() { const char* off = std::getenv("APS5_NO_PIPELINE_JOURNAL"); if ((off != nullptr && std::strcmp(off, "0") != 0) || !ShaderRecompiler::ShaderDiskCache::Enabled()) return; - path = ShaderRecompiler::ShaderDiskCache::EntryDirectory() / "pipelines.journal"; - lockPath = ShaderRecompiler::ShaderDiskCache::EntryDirectory() / "pipelines.journal.lock"; + auto directory = ShaderRecompiler::ShaderDiskCache::EntryDirectory(); +#ifdef _WIN32 + std::error_code error; + const auto native = ShaderRecompiler::NativeCachePath_nid_no_patch(directory, error); + if (error) throw std::filesystem::filesystem_error("cannot resolve the pipeline journal directory", directory, error); + directory = native; +#endif + path = directory / "pipelines.journal"; + lockPath = directory / "pipelines.journal.lock"; maxBytes = configured("APS5_PIPELINE_JOURNAL_MAX_MIB", 1024u) << 20u; std::thread([this] { run(); }).detach(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index 5192c29061..e5174b200b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -12,6 +12,7 @@ #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "ShaderDiskCache.hpp" +#include "ShaderCacheDirectory.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" @@ -9735,21 +9736,36 @@ void pipelineStoreLazyBufferPoolTests() { Require(mock.live == 0, "lazy buffer pool pipeline tests leaked Vulkan objects"); } -void pipelineJournalTests() { +void pipelineJournalTests(bool longPath = false, bool extended = false) { using namespace AgcDriver::Graphics; using Stage = ShaderRecompiler::ShaderStage; - const auto directory = std::filesystem::temp_directory_path() / ("aps5-pipeline-journal-mock-" + std::to_string(std::random_device{}())); + const auto root = std::filesystem::temp_directory_path() / ("aps5-pipeline-journal-mock-" + std::to_string(std::random_device{}())); + auto directory = root; + if (longPath) { + Require(root.native().size() < 180, "the temporary root is too long for the journal path fixture"); + directory /= std::filesystem::path(u8"cache-\u00e4\u6f22"); + directory /= std::string(229 - directory.native().size(), 'd'); + } std::filesystem::create_directories(directory); struct Cleanup { std::filesystem::path directory; + std::filesystem::path journal; ~Cleanup() { + ShaderRecompiler::ShaderDiskCache::Shutdown(); std::error_code error; + if (!journal.empty()) { + std::filesystem::remove(journal, error); + auto lock = journal; + lock += ".lock"; + std::filesystem::remove(lock, error); + } std::filesystem::remove_all(directory, error); } - } cleanup{directory}; + } cleanup{root, {}}; #ifdef _WIN32 + if (extended) directory = std::filesystem::path(L"\\\\?\\" + directory.make_preferred().native()); _putenv_s("ANYPS5_NO_SHADER_CACHE", "0"); - _putenv_s("ANYPS5_SHADER_CACHE_DIR", directory.string().c_str()); + _wputenv_s(L"ANYPS5_SHADER_CACHE_DIR", directory.c_str()); _putenv_s("APS5_PIPELINE_JOURNAL_THREADS", "2"); #else setenv("ANYPS5_NO_SHADER_CACHE", "0", 1); @@ -9757,6 +9773,14 @@ void pipelineJournalTests() { setenv("APS5_PIPELINE_JOURNAL_THREADS", "2", 1); #endif Require(PipelineJournalEnabled(), "the journal is disabled with a shader cache directory set"); + auto journalFile = PipelineJournalPath(); +#ifdef _WIN32 + std::error_code pathError; + journalFile = ShaderRecompiler::NativeCachePath_nid_no_patch(journalFile, pathError); + Require(!pathError, "cannot normalize the journal fixture path"); +#endif + cleanup.journal = journalFile; + if (longPath) Require(journalFile.native().size() > 260, "the journal fixture did not cross MAX_PATH"); const auto module = [](std::uint32_t salt) { ShaderRecompiler::RecompileResult result; result.spirv = std::vector{0x07230203u, 0x00010000u, 0u, 8u, 0u, 0x00020011u, 1u, 0x00020011u, 2u + salt}; @@ -9948,13 +9972,15 @@ void pipelineJournalTests() { FlushPipelineJournal(); const auto repeated = PipelineJournalTotals(); Require(repeated.recordedGraphics == recorded.recordedGraphics && repeated.recordedCompute == recorded.recordedCompute && repeated.recordedModules == recorded.recordedModules, "a repeated pipeline was recorded again"); - Require(std::filesystem::file_size(PipelineJournalPath()) > 0, "the journal file is empty"); + Require(std::filesystem::file_size(journalFile) > 0, "the journal file is empty"); const auto storeBefore = PipelineStoreTotals(); std::vector receivedWords; ShaderRecompiler::RecompileResult received; bool receivedPush = false; std::uint32_t computeCalls = 0; + std::mutex computeMutex; PipelineJournalHooks hooks{[&](const ShaderRecompiler::RecompileResult& shader, bool pushConstants) { + std::lock_guard lock(computeMutex); ++computeCalls; received = shader; receivedPush = pushConstants; @@ -9977,7 +10003,7 @@ void pipelineJournalTests() { const auto storeHit = PipelineStoreTotals(); Require(pipeline != nullptr && !built && mock.pipelineCreateCount == created && storeHit.hits == storeReplayed.hits + 1 && storeHit.misses == storeReplayed.misses, "the draw's configuration did not find the replayed pipeline"); { - const auto journalPath = PipelineJournalPath(); + const auto& journalPath = journalFile; const auto whole = std::filesystem::file_size(journalPath); Require(whole % 8 == 0 && whole > 3, "the journal file is not record aligned"); std::filesystem::resize_file(journalPath, whole - 3); @@ -9991,9 +10017,33 @@ void pipelineJournalTests() { torn->Wait(); const auto reloaded = PipelineJournalTotals(); Require(reloaded.droppedRecords == 0 && reloaded.loadedModules == 4 && reloaded.loadedRecipes == 2 && reloaded.replayFailed == 0, "the journal did not recover from a tail torn inside a record's padding"); + const auto writtenBefore = PipelineJournalTotals().writtenBytes; RecordComputePipeline(compute, true); FlushPipelineJournal(); Require(PipelineJournalTotals().recordedCompute == reloaded.recordedCompute + 1 && PipelineJournalTotals().recordedModules == reloaded.recordedModules, "the pipeline whose record was torn was not recorded again, or its module was"); + const auto recordBytes = PipelineJournalTotals().writtenBytes - writtenBefore; + std::vector bytes; + Require(ShaderRecompiler::ReadWholeFile(journalFile, bytes) && recordBytes > 0 && recordBytes <= bytes.size(), "cannot read the re-recorded journal entry"); + { + std::ofstream file(journalFile, std::ios::binary | std::ios::app); + file.write(reinterpret_cast(bytes.data() + bytes.size() - recordBytes), static_cast(recordBytes)); + const std::array tornTail{}; + file.write(tornTail.data(), tornTail.size()); + file.close(); + Require(file.good(), "cannot append an external complete record and torn tail"); + } + auto another = module(5); + another.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 1, std::vector(4))); + const auto beforeAppend = PipelineJournalTotals(); + RecordComputePipeline(another, false); + FlushPipelineJournal(); + const auto appended = PipelineJournalTotals(); + Require(appended.writeFailures == 0 && appended.recordedCompute == beforeAppend.recordedCompute + 1, "the append after an external torn tail failed"); + Require(std::filesystem::file_size(journalFile) == bytes.size() + recordBytes + appended.writtenBytes - beforeAppend.writtenBytes, "tail recovery discarded an external complete record or retained torn bytes"); + auto recovered = StartPipelineJournalReplay(context, fingerprint, hooks); + Require(recovered != nullptr, "the replay after the external append did not start"); + recovered->Wait(); + Require(PipelineJournalTotals().loadedModules == 5 && PipelineJournalTotals().loadedRecipes == 4 && PipelineJournalTotals().droppedRecords == 0, "the external append and torn-tail recovery did not replay cleanly"); } { const auto before = PipelineJournalTotals(); @@ -10006,7 +10056,7 @@ void pipelineJournalTests() { } { const std::uint32_t meshWithoutOutputSize = 2u; - std::fstream file(PipelineJournalPath(), std::ios::in | std::ios::out | std::ios::binary); + std::fstream file(journalFile, std::ios::in | std::ios::out | std::ios::binary); file.seekp(4); file.write(reinterpret_cast(&meshWithoutOutputSize), sizeof(meshWithoutOutputSize)); file.close(); @@ -10049,8 +10099,8 @@ int main(int argc, char** argv) { commandBatchFenceFailureTests(); return 0; } - if (argc == 2 && std::string_view(argv[1]) == "--pipeline-journal") { - pipelineJournalTests(); + if (argc == 2 && (std::string_view(argv[1]) == "--pipeline-journal" || std::string_view(argv[1]) == "--pipeline-journal-long-paths" || std::string_view(argv[1]) == "--pipeline-journal-extended-paths")) { + pipelineJournalTests(std::string_view(argv[1]) != "--pipeline-journal", std::string_view(argv[1]) == "--pipeline-journal-extended-paths"); std::cout << "pipeline journal recipe, record and replay tests passed\n"; return 0; } diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 1796f83acb..c86a452b79 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -973,6 +973,62 @@ void verifyPersist() { requireSameArtifact(variant.artifact, persisted->artifact, "a persisted entry whose file was replaced and removed"); } +#ifdef _WIN32 +void verifyLongHostPaths() { + using namespace ShaderRecompiler::ShaderDiskCache; + auto directory = std::filesystem::temp_directory_path() / ("aps5-cache-long-" + std::to_string(std::random_device{}())); + require(directory.native().size() < 224, "temporary path is too long for the cache control"); + directory /= std::wstring(224 - directory.native().size(), L'd'); + const auto extended = [](const auto& path) { return std::filesystem::path(L"\\\\?\\" + path.native()); }; + std::filesystem::create_directories(extended(directory)); + setEnvironment("ANYPS5_NO_SHADER_CACHE", "0"); + setEnvironment("ANYPS5_SHADER_CACHE_DIR", directory.string()); + std::vector files; + std::filesystem::path entries; + struct Cleanup { + const std::filesystem::path& directory; + const std::vector& files; + const std::filesystem::path& entries; + ~Cleanup() { + ShaderRecompiler::ShaderDiskCache::Shutdown(); + std::error_code error; + for (const auto& file : files) std::filesystem::remove(std::filesystem::path(L"\\\\?\\" + file.native()), error); + if (!entries.empty()) std::filesystem::remove(std::filesystem::path(L"\\\\?\\" + entries.native()), error); + std::filesystem::remove(std::filesystem::path(L"\\\\?\\" + directory.native()), error); + } + } cleanup{directory, files, entries}; + SampleRequest sample; + const auto key = sample.Key(); + const auto variant = sampleVariant(); + entries = ShaderDiskCache::EntryDirectory(); + files.push_back(entries / EntryName(key)); + files.push_back(entries / "last-used"); + require(files.front().native().size() >= MAX_PATH, "the cache persistence control did not cross MAX_PATH"); + const auto persisted = ShaderDiskCache::Persist(key, variant); + std::vector published; + require(ReadWholeFile(files.front(), published) && published == EncodeEntry(key, variant), + "the long-path cache entry was not correctly published before mapping"); + require(persisted != nullptr, "a long-path cache entry was not persisted and mapped"); + MEMORY_BASIC_INFORMATION region{}; + require(VirtualQuery(persisted->artifact.spirv.data(), ®ion, sizeof(region)) == sizeof(region) && region.Type == MEM_MAPPED, + "long-path persisted SPIR-V is not mapped"); + requireSameVariant(variant, *persisted, "long-path persist round trip"); + CompiledVariant loaded; + require(ShaderDiskCache::Load(key, loaded), "a long-path persisted cache entry did not load"); + requireSameVariant(variant, loaded, "long-path load round trip"); + const auto packPath = directory / (std::wstring(75, L'p') + L".pack"); + files.push_back(packPath); + require(WriteFileAtomically(packPath, packBytes({{key, EncodeEntry(key, variant)}})), "cannot publish the long-path cache pack"); + CachePack pack(packPath); + require(pack.Valid(), "a long-path cache pack was rejected: " + pack.Error()); + PackLoadTimings timings; + CompiledVariant decoded; + require(pack.Load(key, decoded, timings) == LoadStatus::Loaded, "a long-path cache pack entry did not load"); + requireSameVariant(variant, decoded, "long-path cache pack round trip"); + std::cout << "long-path shader cache persistence and pack tests passed\n"; +} +#endif + struct ComputeRequest { std::vector code{0xe0700000u, 0x80000000u, 0xbf810000u}; std::array userData{0x10000000u, 0x00000000u, 0x40u, 0x00027facu}; @@ -2321,6 +2377,12 @@ int main(int argc, char** argv) { std::cout << "artifact codec tests passed\n"; return 0; } +#ifdef _WIN32 + if (argc == 2 && std::string_view(argv[1]) == "--long-host-paths") { + verifyLongHostPaths(); + return 0; + } +#endif if (argc == 2 && std::string_view(argv[1]) == "--large-file") return runLargeFileWrite(); if (argc == 2 && std::string_view(argv[1]) == "--load") return runLoadingProcess(); if (argc == 2 && std::string_view(argv[1]) == "--load-prebuilt") return runPrebuiltLoadingProcess(); diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 7d3750a196..01ffcb201c 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -238,6 +238,7 @@ function(add_shader_recompiler target runtimeTarget) HashBytes LayoutBarycentricEmulation MeshBarycentricLayoutOf + NativeCachePath_nid_no_patch ImageTableAbi::PoisonCode ImageTableAbi::PoisonReasonName ImageTableAbi::TableHeader diff --git a/core/shader/recompiler/ShaderCacheDirectory.hpp b/core/shader/recompiler/ShaderCacheDirectory.hpp index 84fbd1ae36..9b3893ce2a 100644 --- a/core/shader/recompiler/ShaderCacheDirectory.hpp +++ b/core/shader/recompiler/ShaderCacheDirectory.hpp @@ -33,18 +33,42 @@ namespace ShaderRecompiler { std::filesystem::path ShaderCacheDirectory(); +#ifdef _WIN32 +inline std::filesystem::path NativeCachePath_nid_no_patch(const std::filesystem::path& path, std::error_code& error) { + auto native = path; + native.make_preferred(); + if (native.native().starts_with(L"\\\\?\\") || native.native().starts_with(L"\\\\.\\")) return native; + const auto capacity = GetFullPathNameW(native.c_str(), 0, nullptr, nullptr); + if (capacity == 0) { error = std::error_code(GetLastError(), std::system_category()); return {}; } + std::wstring absolute(capacity, L'\0'); + const auto size = GetFullPathNameW(native.c_str(), capacity, absolute.data(), nullptr); + if (size == 0 || size >= capacity) { + error = std::error_code(size == 0 ? GetLastError() : ERROR_INSUFFICIENT_BUFFER, std::system_category()); + return {}; + } + absolute.resize(size); + if (absolute.starts_with(L"\\\\")) return std::filesystem::path(L"\\\\?\\UNC\\" + absolute.substr(2)); + return std::filesystem::path(L"\\\\?\\" + absolute); +} +#endif + inline bool WriteFileAtomically(const std::filesystem::path& path, std::span bytes) { std::error_code error; - std::filesystem::create_directories(path.parent_path(), error); +#ifdef _WIN32 + const auto destination = NativeCachePath_nid_no_patch(path, error); + if (error) return false; +#else + const auto& destination = path; +#endif + std::filesystem::create_directories(destination.parent_path(), error); static std::atomic serial{0}; const auto unique = std::hash{}(std::this_thread::get_id()) ^ static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); - auto temporary = path; + auto temporary = destination; temporary += ".tmp." + std::to_string(unique) + "." + std::to_string(serial.fetch_add(1, std::memory_order_relaxed)); #ifdef _WIN32 struct RenameInfo { DWORD flags; HANDLE root; DWORD nameBytes; WCHAR name[1]; }; static_assert(sizeof(RenameInfo) == sizeof(FILE_RENAME_INFO) && offsetof(RenameInfo, name) == offsetof(FILE_RENAME_INFO, FileName)); - const auto destination = std::filesystem::absolute(path, error); - if (error || destination.native().size() > (MAXDWORD - sizeof(RenameInfo)) / sizeof(wchar_t)) return false; + if (destination.native().size() > (MAXDWORD - sizeof(RenameInfo)) / sizeof(wchar_t)) return false; const auto nameBytes = destination.native().size() * sizeof(wchar_t); std::vector renameBuffer(sizeof(RenameInfo) + nameBytes); auto* rename = reinterpret_cast(renameBuffer.data()); @@ -89,7 +113,7 @@ inline bool WriteFileAtomically(const std::filesystem::path& path, std::span PrebuiltCachePaths(const std::filesystem::pat CachePack::CachePack(const std::filesystem::path& path) { static_assert(std::endian::native == std::endian::little); #ifdef _WIN32 - const auto handle = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + std::error_code pathError; + const auto filename = NativeCachePath_nid_no_patch(path, pathError); + if (pathError) { error = "cannot resolve"; return; } + const auto handle = CreateFileW(filename.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (handle == INVALID_HANDLE_VALUE) { error = "cannot open"; return; } LARGE_INTEGER size{}; if (!GetFileSizeEx(handle, &size) || size.QuadPart < static_cast(sizeof(PackHeader)) || static_cast(size.QuadPart) > std::numeric_limits::max()) { diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 31815e3fb6..3a3d947b8c 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -195,7 +195,10 @@ private: std::shared_ptr MapFile(const std::filesystem::path& path) { #ifdef _WIN32 - const HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + std::error_code error; + const auto filename = NativeCachePath_nid_no_patch(path, error); + if (error) return nullptr; + const HANDLE file = CreateFileW(filename.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (file == INVALID_HANDLE_VALUE) return nullptr; LARGE_INTEGER length{}; if (!GetFileSizeEx(file, &length) || length.QuadPart <= 0 || static_cast(length.QuadPart) > std::numeric_limits::max()) { diff --git a/core/shader/recompiler/tests/CacheFileReadTests.cpp b/core/shader/recompiler/tests/CacheFileReadTests.cpp index 29f5e61b3f..f0e936b798 100644 --- a/core/shader/recompiler/tests/CacheFileReadTests.cpp +++ b/core/shader/recompiler/tests/CacheFileReadTests.cpp @@ -16,11 +16,88 @@ void* operator new[](std::size_t size) { return ::operator new(size); } void operator delete[](void* memory) noexcept { ::operator delete(memory); } void operator delete[](void* memory, std::size_t) noexcept { ::operator delete(memory); } -static void Require(bool ok) { if (!ok) throw std::runtime_error("Cache file read failed"); } +static void Require(bool ok, const char* message = "Cache file read failed") { if (!ok) throw std::runtime_error(message); } + +#ifdef _WIN32 +static void RemoveCachePaths(const std::filesystem::path& path) { + WIN32_FIND_DATAW data{}; + const auto search = FindFirstFileW((path / L"*").c_str(), &data); + if (search != INVALID_HANDLE_VALUE) { + do { + const std::wstring name = data.cFileName; + if (name == L"." || name == L"..") continue; + const auto child = path / name; + if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) RemoveCachePaths(child); + else DeleteFileW(child.c_str()); + } while (FindNextFileW(search, &data)); + FindClose(search); + } + RemoveDirectoryW(path.c_str()); +} + +static void VerifyCachePaths(const std::filesystem::path& root) { + const auto extended = [](const std::filesystem::path& path) { + auto native = path; + native.make_preferred(); + return std::filesystem::path(L"\\\\?\\" + native.native()); + }; + const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}}; + const std::vector replacement{std::byte{5}, std::byte{6}}; + const auto verify = [&](const std::filesystem::path& path) { + for (const auto& alias : {path, extended(path)}) { + Require(ShaderRecompiler::WriteFileAtomically(alias, original), "cannot write cache path"); + std::vector bytes; + Require(ShaderRecompiler::ReadWholeFile(path, bytes) && bytes == original, "cannot read ordinary cache path"); + Require(ShaderRecompiler::ReadWholeFile(extended(path), bytes) && bytes == original, "cannot read extended cache path"); + const auto old = CreateFileW(extended(path).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + Require(old != INVALID_HANDLE_VALUE, "cannot retain the original cache reader"); + struct Close { HANDLE file; ~Close() { CloseHandle(file); } } close{old}; + Require(ShaderRecompiler::WriteFileAtomically(alias, replacement), "cannot replace cache path with retained reader"); + Require(ShaderRecompiler::ReadWholeFile(alias, bytes) && bytes == replacement, "replacement cache bytes do not match"); + std::byte retained[4]{}; + DWORD count = 0; + Require(ReadFile(old, retained, sizeof(retained), &count, nullptr) && count == original.size(), "cannot read the retained cache handle"); + Require(std::equal(original.begin(), original.end(), retained), "retained cache bytes changed after replacement"); + } + WIN32_FIND_DATAW data{}; + const auto temporary = FindFirstFileW((extended(path).parent_path() / L"*.tmp.*").c_str(), &data); + if (temporary != INVALID_HANDLE_VALUE) FindClose(temporary); + Require(temporary == INVALID_HANDLE_VALUE && GetLastError() == ERROR_FILE_NOT_FOUND, "cache temporary file was left behind"); + }; + auto directory = root / "paths"; + while (directory.native().size() < 150) directory /= std::wstring(std::min(40, 150 - directory.native().size()), L'd'); + for (const auto length : {180u, 235u, 269u, 320u}) { + const auto size = std::max(length, directory.native().size() + 12); + verify(directory / (std::wstring(size - directory.native().size() - 5, L'x') + L".bin")); + } + auto deep = root; + while (deep.native().size() < 300) deep /= std::wstring(40, L'd'); + verify(deep / L"cache \u00e4\u6f22.bin"); + std::filesystem::create_directories(root / "relative"); + struct RestoreDirectory { + std::filesystem::path previous = std::filesystem::current_path(); + ~RestoreDirectory() { + std::error_code error; + std::filesystem::current_path(previous, error); + } + } restoreDirectory; + std::filesystem::current_path(root); + auto relative = (root / "relative" / ".." / "relative.bin").lexically_relative(std::filesystem::current_path()); + Require(!relative.empty(), "cannot construct the cache relative path"); + Require(ShaderRecompiler::WriteFileAtomically(relative, original), "cannot write the cache relative path"); + std::vector bytes; + Require(ShaderRecompiler::ReadWholeFile(relative, bytes) && bytes == original, "cannot read the cache relative path"); + const auto missing = deep / "missing.bin"; + Require(!ShaderRecompiler::ReadWholeFile(missing, bytes) && bytes.empty(), "missing cache path did not clear the read buffer"); +} +#endif int main() { const auto path = std::filesystem::temp_directory_path() / ("anyps5-cache-read-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + auto directory = path; + directory += "-paths"; int status = 0; try { std::vector bytes{std::byte{1}}; @@ -37,8 +114,15 @@ int main() { FailLargeAllocation = false; Require(!read && failed.empty()); std::cout << "5 cache file read cases passed\n"; +#ifdef _WIN32 + VerifyCachePaths(directory); + std::cout << "Windows cache path and retained-reader replacement cases passed\n"; +#endif } catch (const std::exception& error) { FailLargeAllocation = false; std::cerr << error.what() << '\n'; status = 1; } std::error_code error; std::filesystem::remove(path, error); +#ifdef _WIN32 + RemoveCachePaths(std::filesystem::path(L"\\\\?\\" + directory.native())); +#endif return status; } diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 5ed4ff62f9..f756ed2ade 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -213,3 +213,5 @@ - [2D reads of 2D arrays](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a sampled image whose shader shape is 2D and whose T# is a 2D array is built from its base slice alone: the T# is narrowed to that slice (`FirstLayerDescriptor`) before its decode, cache key and image, in stage A records, stage B bindings, revalidation and image tables. A whole-array read of the same T# keeps its own cache entry, so both images exist when a title uses both. Storage images read or written through a 2D shape are not narrowed and still build every slice the T# names. Ghost of Yotei (PPSA26344) reads a 3328x1872 R_X T# of 1537 slices this way: the whole array was a 12.4 GiB image that did not fit the device-local budget of an RTX 4090. - [Sampled and storage views of 1 GiB or more](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) that stay whole are only logged (sampled views by `cachedTexture`, storage views by `cachedStorageTexture`), with the registered guest range holding their base and how many of their slices fit in it; nothing is capped to that range. Ghost of Yotei (PPSA26344) samples a 1467 MiB cube array window of which only 838 of 2934 slices lie inside its 3380 MiB range: the rest is read from whatever follows the range. Whether such slices are ever sampled is not measured. - [Color export type against the target format](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - the export-format check refuses a `UINT16_ABGR` export (`SPI_SHADER_COL_FORMAT` 7) into a target whose color-buffer number format is not unsigned integer, but not the reverse. A 32-bit export (`32_R`, `32_GR`, `32_AR` or `32_ABGR`) into an unsigned-integer target leaves `uintExport` clear, so the recompiler declares the fragment output `float4`; `DecodeColorBuffer` maps a number format of 4 to `R32_UINT` from the color buffer alone, independent of the export format, so the float output reaches an integer attachment. Desktop drivers keep the raw bits, but the Khronos validation layer reports `Undefined-Value-ShaderFragmentOutputMismatch` and MoltenVK's Metal compiler refuses the pipeline. Which export format a title pairs with a 32-bit integer target is not measured, so the output type is left following the export format rather than the attachment. + +- [Long Windows pipeline-cache paths](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineCache.cpp): `readHeader` and the failed-read existence check still use ordinary paths. A filename beyond MAX_PATH can prevent header reuse or be reported missing; the shared content reader and journal normalization do not cover these calls.