Files
AnyPS5/core/shader/recompiler/ShaderCacheDirectory.hpp
thibautandClaude Opus 5.5 a03ccc6be1 fix(shader): write cache files of 2 GiB and more in chunks
On Windows (MinGW libstdc++) one std::ofstream::write of 2 GiB or more
never returns: the whole temporary file is written but WriteFileAtomically
hangs, the entry is never replaced and the temporary file stays behind.
A Vulkan pipeline cache of 2.1 GB hung every save and left a 2 GB .tmp
per run. Write in chunks of at most 1 GiB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XBXM4pgkMHYEtqN1EQni2P
2026-10-17 14:48:47 +02:00

180 lines
6.9 KiB
C++

#ifndef CORE_SHADER_RECOMPILER_SHADERCACHEDIRECTORY_HPP
#define CORE_SHADER_RECOMPILER_SHADERCACHEDIRECTORY_HPP
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <functional>
#include <limits>
#include <new>
#include <stdexcept>
#include <span>
#include <string>
#include <system_error>
#include <thread>
#include <vector>
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#endif
namespace ShaderRecompiler {
std::filesystem::path ShaderCacheDirectory();
inline bool WriteFileAtomically(const std::filesystem::path& path, std::span<const std::byte> bytes) {
std::error_code error;
std::filesystem::create_directories(path.parent_path(), error);
static std::atomic<std::uint64_t> serial{0};
const auto unique = std::hash<std::thread::id>{}(std::this_thread::get_id()) ^ static_cast<std::size_t>(std::chrono::steady_clock::now().time_since_epoch().count());
auto temporary = path;
temporary += ".tmp." + std::to_string(unique) + "." + std::to_string(serial.fetch_add(1, std::memory_order_relaxed));
#ifdef _WIN32
struct RenameInfo { DWORD flags; HANDLE root; DWORD nameBytes; WCHAR name[1]; };
static_assert(sizeof(RenameInfo) == sizeof(FILE_RENAME_INFO) && offsetof(RenameInfo, name) == offsetof(FILE_RENAME_INFO, FileName));
const auto destination = std::filesystem::absolute(path, error);
if (error || destination.native().size() > (MAXDWORD - sizeof(RenameInfo)) / sizeof(wchar_t)) return false;
const auto nameBytes = destination.native().size() * sizeof(wchar_t);
std::vector<std::byte> renameBuffer(sizeof(RenameInfo) + nameBytes);
auto* rename = reinterpret_cast<RenameInfo*>(renameBuffer.data());
constexpr DWORD replaceExisting = 0x00000001, posixSemantics = 0x00000002;
rename->flags = replaceExisting | posixSemantics;
rename->nameBytes = static_cast<DWORD>(nameBytes);
std::memcpy(rename->name, destination.c_str(), nameBytes);
#endif
bool written = false;
{
std::ofstream file(temporary, std::ios::binary | std::ios::trunc);
if (!file) return false;
constexpr std::size_t largestWrite = std::size_t{1} << 30u;
for (std::size_t offset = 0; offset < bytes.size() && file;) {
const auto chunk = std::min(bytes.size() - offset, largestWrite);
file.write(reinterpret_cast<const char*>(bytes.data() + offset), static_cast<std::streamsize>(chunk));
offset += chunk;
}
file.close();
written = !file.fail();
}
if (!written) {
std::filesystem::remove(temporary, error);
return false;
}
#ifdef _WIN32
DWORD renameError = ERROR_SUCCESS;
{
const auto handle = CreateFileW(temporary.c_str(), DELETE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
if (handle == INVALID_HANDLE_VALUE) {
std::filesystem::remove(temporary, error);
return false;
}
struct Close { HANDLE handle; ~Close() { CloseHandle(handle); } } close{handle};
constexpr auto renameInfoEx = static_cast<FILE_INFO_BY_HANDLE_CLASS>(22);
if (!SetFileInformationByHandle(handle, renameInfoEx, rename, static_cast<DWORD>(renameBuffer.size()))) renameError = GetLastError();
}
if (renameError == ERROR_SUCCESS) return true;
if (renameError != ERROR_INVALID_PARAMETER && renameError != ERROR_INVALID_FUNCTION && renameError != ERROR_NOT_SUPPORTED) {
std::filesystem::remove(temporary, error);
return false;
}
#endif
std::filesystem::rename(temporary, path, error);
if (error) {
std::filesystem::remove(temporary, error);
return false;
}
return true;
}
inline bool ReadWholeFile(const std::filesystem::path& path, std::vector<std::byte>& bytes) {
bytes.clear();
try {
#ifdef _WIN32
const auto file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
if (file == INVALID_HANDLE_VALUE) return false;
struct Close { HANDLE file; ~Close() { CloseHandle(file); } } close{file};
LARGE_INTEGER fileSize{};
if (!GetFileSizeEx(file, &fileSize)) return false;
const auto size = fileSize.QuadPart;
#else
std::ifstream file(path, std::ios::binary | std::ios::ate);
if (!file) return false;
const auto size = static_cast<std::streamoff>(file.tellg());
#endif
if (size < 0 || static_cast<std::uintmax_t>(size) > bytes.max_size() ||
static_cast<std::uintmax_t>(size) > static_cast<std::uintmax_t>(std::numeric_limits<std::streamsize>::max()))
return false;
#ifdef _WIN32
bytes.resize(static_cast<std::size_t>(size));
for (std::size_t offset = 0; offset < bytes.size();) {
const auto left = bytes.size() - offset;
const auto chunk = static_cast<DWORD>(left > MAXDWORD ? MAXDWORD : left);
DWORD count = 0;
if (!ReadFile(file, bytes.data() + offset, chunk, &count, nullptr) || count == 0) {
bytes.clear();
return false;
}
offset += count;
}
#else
if (!file.seekg(0, std::ios::beg)) return false;
bytes.resize(static_cast<std::size_t>(size));
if (!file.read(reinterpret_cast<char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()))) {
bytes.clear();
return false;
}
#endif
return true;
} catch (const std::bad_alloc&) {
bytes.clear();
return false;
} catch (const std::length_error&) {
bytes.clear();
return false;
}
}
inline std::uint64_t HashBytes(std::span<const std::byte> bytes, std::uint64_t seed = 0) {
constexpr std::uint64_t multiplier = 0x9e3779b97f4a7c15ull;
std::uint64_t hash = seed ^ (static_cast<std::uint64_t>(bytes.size()) * multiplier);
const auto mix = [&](std::uint64_t chunk) {
chunk *= 0xff51afd7ed558ccdull;
chunk ^= chunk >> 32u;
hash = (hash ^ chunk) * multiplier;
hash = (hash << 27u) | (hash >> 37u);
};
std::size_t index = 0;
for (; index + 8 <= bytes.size(); index += 8) {
std::uint64_t chunk;
std::memcpy(&chunk, bytes.data() + index, sizeof(chunk));
mix(chunk);
}
if (index < bytes.size()) {
std::uint64_t chunk = 0;
std::memcpy(&chunk, bytes.data() + index, bytes.size() - index);
mix(chunk ^ 0x8000000000000000ull);
}
hash ^= hash >> 33u;
hash *= 0xff51afd7ed558ccdull;
hash ^= hash >> 33u;
hash *= 0xc4ceb9fe1a85ec53ull;
hash ^= hash >> 33u;
return hash;
}
}
#endif