fix(libc): give collate<wchar_t> its own 16-bit vtable

core/libs/prx/libc/src/LocaleSupport.cpp

@@ -157,6 +157,68 @@ std::int64_t APS5_VABI CollateHash(const GuestLocale::CollateFacet*, const char*
 
 const GuestLocale::CollateVtable g_collateVtable{{DestroyCollate, DeleteCollate, RetainFacet, ReleaseFacet}, CollateCompare, CollateTransform, CollateHash};
 
+// The guest's wchar_t is 16 bits, so the wide facet needs its own entries: collate<wchar_t> reaches the
+// same slots with ranges of 16-bit characters, whose length is a count of characters and not of bytes.
+// The C locale orders them by value, as Dinkumware's char_traits<wchar_t>::compare does, mixes one
+// whole character per hash round, and copies them into the result, whose size and capacity are bytes.
+constexpr std::size_t GuestWideCharacterSize = 2;
+constexpr std::size_t GuestWideInlineCapacity = sizeof(GuestLocale::String::buffer) - GuestWideCharacterSize;
+
+void ValidateWideCharacterRange(const std::uint16_t* first, const std::uint16_t* last, const char* function) {
+    if ((first == nullptr) != (last == nullptr) || last < first) throw std::invalid_argument(std::string(function) + ": invalid character range");
+}
+
+int APS5_VABI WideCollateCompare(const GuestLocale::CollateFacet*, const char* first1, const char* last1, const char* first2, const char* last2) {
+    const auto* characters1 = reinterpret_cast<const std::uint16_t*>(first1);
+    const auto* end1 = reinterpret_cast<const std::uint16_t*>(last1);
+    const auto* characters2 = reinterpret_cast<const std::uint16_t*>(first2);
+    const auto* end2 = reinterpret_cast<const std::uint16_t*>(last2);
+    ValidateWideCharacterRange(characters1, end1, "collate::do_compare");
+    ValidateWideCharacterRange(characters2, end2, "collate::do_compare");
+    const auto length1 = static_cast<std::size_t>(end1 - characters1);
+    const auto length2 = static_cast<std::size_t>(end2 - characters2);
+    const auto common = std::min(length1, length2);
+    for (std::size_t index = 0; index < common; ++index) {
+        if (characters1[index] != characters2[index]) return characters1[index] < characters2[index] ? -1 : 1;
+    }
+    if (length1 == length2) return 0;
+    return length1 < length2 ? -1 : 1;
+}
+
+GuestLocale::String* APS5_VABI WideCollateTransform(GuestLocale::String* result, const GuestLocale::CollateFacet*, const char* first, const char* last) {
+    if (result == nullptr) throw std::invalid_argument("collate::do_transform: null result");
+    const auto* characters = reinterpret_cast<const std::uint16_t*>(first);
+    const auto* end = reinterpret_cast<const std::uint16_t*>(last);
+    ValidateWideCharacterRange(characters, end, "collate::do_transform");
+    const auto length = static_cast<std::size_t>(end - characters);
+    if (length != 0 && std::find(characters, end, static_cast<std::uint16_t>(0)) != end) throw std::runtime_error("collate::do_transform: embedded null characters are not supported");
+    const auto bytes = length * GuestWideCharacterSize;
+    char* data = result->buffer;
+    std::size_t capacity = GuestWideInlineCapacity;
+    if (bytes > GuestWideInlineCapacity) {
+        capacity = bytes | GuestStringCapacityMask;
+        data = static_cast<char*>(AllocateStringStorage(capacity + GuestWideCharacterSize));
+    }
+    if (length != 0) std::memcpy(data, characters, bytes);
+    data[bytes] = 0;
+    data[bytes + 1] = 0;
+    if (data != result->buffer) result->pointer = data;
+    result->size = bytes;
+    result->capacity = capacity;
+    return result;
+}
+
+std::int64_t APS5_VABI WideCollateHash(const GuestLocale::CollateFacet*, const char* first, const char* last) {
+    const auto* characters = reinterpret_cast<const std::uint16_t*>(first);
+    const auto* end = reinterpret_cast<const std::uint16_t*>(last);
+    ValidateWideCharacterRange(characters, end, "collate::do_hash");
+    std::uint64_t hash = 0xcbf29ce484222325;
+    for (; characters != end; ++characters) hash = (hash ^ static_cast<std::uint64_t>(*characters)) * 0x100000001b3;
+    return static_cast<std::int64_t>(hash);
+}
+
+const GuestLocale::CollateVtable g_wideCollateVtable{{DestroyCollate, DeleteCollate, RetainFacet, ReleaseFacet}, WideCollateCompare, WideCollateTransform, WideCollateHash};
+
 constexpr std::array<short, 257> MakeClassificationTable() {
     std::array<short, 257> table{};
     for (unsigned int value = 0; value < 256; ++value) {
@@ -275,7 +337,7 @@ std::size_t APS5_VABI _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_pos
     if (facet != nullptr && *facet == nullptr) {
         if (locale == nullptr || *locale == nullptr || (*locale)->name == nullptr || std::strcmp((*locale)->name, "C") != 0) throw std::invalid_argument("collate::_Getcat: only the C locale is supported");
         auto* collate = static_cast<GuestLocale::CollateFacet*>(_Znwm_nid_postfix(sizeof(GuestLocale::CollateFacet)));
-        *collate = {{&g_collateVtable.facet, 0, 0}, nullptr, nullptr};
+        *collate = {{&g_wideCollateVtable.facet, 0, 0}, nullptr, nullptr};
         *facet = &collate->base;
     }
     return 1;

core/libs/tests/GuestCollate.cpp

@@ -64,9 +64,27 @@ std::uint64_t Hash(const GuestLocale::CollateFacet* facet, const std::string& te
     return static_cast<std::uint64_t>(Vtable(&facet->base).hash(facet, text.data(), text.data() + text.size()));
 }
 
-// The wide facet mirrors the narrow one: the same facet allocation, the same C-locale
-// validation and the same vtable, because GuestLocale models one CollateVtable for both.
-void CheckWideGetcat(const GuestLocale::Implementation* const* classic, const void* narrowVtable) {
+// collate<wchar_t> hands the vtable pointers to its 16-bit characters, so the tests below pass the
+// characters of a wide string where the vtable declares a range of char and count it in characters.
+const char* Characters(const char16_t* text) {
+    return reinterpret_cast<const char*>(text);
+}
+
+int CompareWide(const GuestLocale::CollateFacet* facet, const std::u16string& left, const std::u16string& right) {
+    return Vtable(&facet->base).compare(facet, Characters(left.data()), Characters(left.data() + left.size()), Characters(right.data()), Characters(right.data() + right.size()));
+}
+
+std::uint64_t HashWide(const GuestLocale::CollateFacet* facet, const std::u16string& text) {
+    return static_cast<std::uint64_t>(Vtable(&facet->base).hash(facet, Characters(text.data()), Characters(text.data() + text.size())));
+}
+
+GuestLocale::String* TransformWide(const GuestLocale::CollateFacet* facet, GuestLocale::String* result, const std::u16string& text) {
+    return Vtable(&facet->base).transform(result, facet, Characters(text.data()), Characters(text.data() + text.size()));
+}
+
+// The wide facet mirrors the narrow one in the facet it allocates and the C-locale validation, but it
+// carries its own vtable, whose entries work on the guest's 16-bit characters.
+void CheckWideGetcat(const GuestLocale::Implementation* const* classic, const GuestLocale::CollateFacet* narrow) {
     require(_ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(nullptr, classic) == 1);
 
     const std::size_t before = allocations;
@@ -95,17 +113,67 @@ void CheckWideGetcat(const GuestLocale::Implementation* const* classic, const vo
     require(_ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(&facet, classic) == 1);
     require(facet != nullptr && allocations == before + 1 && lastSize == sizeof(GuestLocale::CollateFacet) && lastAllocation == facet);
     const auto* wide = reinterpret_cast<const GuestLocale::CollateFacet*>(facet);
-    require(facet->references == 0 && facet->vtable == narrowVtable);
+    require(facet->references == 0 && facet->vtable != narrow->base.vtable);
     require(wide->collation == nullptr && wide->wideCollation == nullptr);
 
-    require(Compare(wide, "abc", "abd") == -1);
-    require(Compare(wide, "abd", "abc") == 1);
-    require(Compare(wide, "abc", "abc") == 0);
-    require(Compare(wide, "ab", "abc") == -1);
-    require(Compare(wide, "", "a") == -1);
-    require(Hash(wide, "foobar") == 0x85944171f73967e8ull);
-    const char narrow[] = "abc";
-    reject([&] { Vtable(facet).compare(wide, narrow + 2, narrow, narrow, narrow + 1); });
+    const std::u16string text = u"abc";
+    require(CompareWide(wide, text, u"abd") == -1);
+    require(CompareWide(wide, u"abd", text) == 1);
+    require(CompareWide(wide, text, text) == 0);
+    require(CompareWide(wide, u"ab", text) == -1);
+    require(CompareWide(wide, u"", text) == -1);
+    require(CompareWide(wide, u"", u"") == 0);
+    // Characters above 0xff order by their whole value: 0x0100 and 0x0001 compare the other way round
+    // when the same characters are read as the little-endian bytes 0x00 0x01 and 0x01 0x00.
+    require(CompareWide(wide, u"\u0100", u"\u0001") == 1);
+    require(CompareWide(wide, u"\u0001", u"\u0100") == -1);
+    require(CompareWide(wide, u"\u00ff", u"\u0100") == -1);
+    require(CompareWide(wide, u"\u00e9", u"z") == 1);
+    require(CompareWide(wide, u"a\u0101b", u"a\u0100b") == 1);
+    require(CompareWide(wide, u"a\u0101b", u"a\u0102b") == -1);
+    const std::u16string embeddedWide(u"a\0b", 3);
+    require(CompareWide(wide, embeddedWide, u"a") == 1);
+    require(CompareWide(wide, text, u"abd") == Compare(narrow, "abc", "abd"));
+
+    require(HashWide(wide, u"") == 0xcbf29ce484222325ull);
+    require(HashWide(wide, text) == 0xe71fa2190541574bull);
+    require(HashWide(wide, u"a") == 0xaf63dc4c8601ec8cull);
+    // One round per character, so a wide string of characters below 0x100 hashes like its narrow
+    // spelling and a character above 0xff hashes like one value instead of the two bytes it holds.
+    require(HashWide(wide, u"foobar") == 0x85944171f73967e8ull);
+    require(HashWide(wide, u"foobar") == Hash(narrow, "foobar"));
+    require(HashWide(wide, u"\u0100") == 0x0af62bd4c860004dfull);
+
+    GuestLocale::String result;
+    std::memset(&result, 0xcd, sizeof(result));
+    require(TransformWide(wide, &result, text) == &result);
+    require(result.reserved == 0xcdcdcdcdcdcdcdcdull && result.size == 6 && result.capacity == 14 && allocations == before + 1);
+    const auto* inlineUnits = reinterpret_cast<const std::uint16_t*>(result.buffer);
+    require(inlineUnits[0] == u'a' && inlineUnits[1] == u'b' && inlineUnits[2] == u'c' && inlineUnits[3] == 0);
+
+    std::memset(&result, 0xcd, sizeof(result));
+    require(Vtable(facet).transform(&result, wide, nullptr, nullptr) == &result);
+    require(result.size == 0 && result.capacity == 14 && reinterpret_cast<const std::uint16_t*>(result.buffer)[0] == 0);
+
+    const std::u16string seven(7, u'q');
+    require(TransformWide(wide, &result, seven) == &result);
+    require(result.size == 14 && result.capacity == 14 && allocations == before + 1);
+    const auto* inlineSeven = reinterpret_cast<const std::uint16_t*>(result.buffer);
+    require(inlineSeven[6] == u'q' && inlineSeven[7] == 0);
+
+    const std::u16string eight(8, u'y');
+    require(TransformWide(wide, &result, eight) == &result);
+    require(allocations == before + 2 && lastSize == 33 && result.size == 16 && result.capacity == 31 && result.pointer == lastAllocation);
+    require(reinterpret_cast<const std::uint16_t*>(result.pointer)[0] == u'y' && reinterpret_cast<const std::uint16_t*>(result.pointer)[8] == 0);
+    std::free(result.pointer);
+
+    const std::u16string embeddedText(u"a\0b", 3);
+    reject([&] { TransformWide(wide, &result, embeddedText); });
+    reject([&] { Vtable(facet).transform(nullptr, wide, Characters(seven.data()), Characters(seven.data() + seven.size())); });
+    require(allocations == before + 2);
+    reject([&] { Vtable(facet).compare(wide, Characters(text.data() + 2), Characters(text.data()), Characters(text.data()), Characters(text.data() + 1)); });
+    reject([&] { Vtable(facet).compare(wide, nullptr, Characters(text.data() + 1), Characters(text.data()), Characters(text.data() + 1)); });
+    reject([&] { Vtable(facet).hash(wide, Characters(text.data() + 1), Characters(text.data())); });
 
     Vtable(facet).facet.retain(facet);
     require(facet->references == 1);
@@ -115,7 +183,7 @@ void CheckWideGetcat(const GuestLocale::Implementation* const* classic, const vo
     require(frees == freed);
     Vtable(facet).facet.deleteObject(facet);
     require(frees == freed + 1 && lastFree == wide);
-    require(allocations == before + 1);
+    require(allocations == before + 2);
 }
 
 }
@@ -142,7 +210,6 @@ int main() {
     require(facet != nullptr && allocations == 1 && lastSize == sizeof(GuestLocale::CollateFacet) && lastAllocation == facet);
     const auto* collate = reinterpret_cast<const GuestLocale::CollateFacet*>(facet);
     require(facet->references == 0 && collate->collation == nullptr && collate->wideCollation == nullptr);
-    const void* narrowVtable = facet->vtable;
 
     require(Compare(collate, "abc", "abd") == -1);
     require(Compare(collate, "abd", "abc") == 1);
@@ -211,12 +278,12 @@ int main() {
     require(facet->references == 2);
     require(Vtable(facet).facet.release(facet) == nullptr && facet->references == 1);
     require(Vtable(facet).facet.release(facet) == facet && facet->references == 0);
-    Vtable(facet).facet.destroy(facet);
-    require(frees == 0);
-    Vtable(facet).facet.deleteObject(facet);
-    require(frees == 1 && lastFree == collate);
+    CheckWideGetcat(classic, collate);
 
-    CheckWideGetcat(classic, narrowVtable);
-    require(frees == 2 && allocations == 5);
+    Vtable(facet).facet.destroy(facet);
+    require(frees == 1);
+    Vtable(facet).facet.deleteObject(facet);
+    require(frees == 2 && lastFree == collate);
+    require(allocations == 6);
     return 0;
 }

docs/dev/debt/libc.md

@@ -58,7 +58,7 @@
 - [gethostname](../../../core/libs/prx/libc/src/Hostname.cpp) (libc) reports the fixed guest name `AnyPS5` without querying the host machine name. Buffer copying, null queries and `ENAMETOOLONG` follow [FreeBSD gethostname](https://github.com/freebsd/freebsd-src/blob/main/lib/libc/gen/gethostname.c); the console's configurable hostname and its integration with `sysctl` and name resolution are unverified and not implemented.
 - [wcsrtombs_s](../../../core/libs/prx/libc/src/Multibyte.cpp) (libc) follows the console's `libc.prx`, but a runtime-constraint violation only returns its error: the default handler there writes the message and a newline to `stderr` before returning, which is not done here. Conversion uses the C locale of `wcrtomb` (`MB_CUR_MAX` 1), so the path where a multibyte character does not fit the remaining space is not reached
 - [The narrow printf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc) converts 16-bit guest wide arguments to UTF-8 on both platforms, independently of the host locale. It does not use the guest's single-byte `wcrtomb`; the console's treatment of invalid UTF-16 and locale-specific encoding remains unverified. Linux retains host formatting for formats without wide arguments or `%m`, so a narrow `%p` and a NaN there still follow the host (glibc prints a null pointer as `(nil)`, where FreeBSD prints `0x0`, and a NaN with its sign, `-nan`, and with the `+` and space flags, where FreeBSD prints `nan`); argument positions above 99 (FreeBSD's `NL_ARGMAX`) throw when a format requires the guest formatter
-- [collate<char> and collate<wchar_t> _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build the facet for the C locale only and throw for any other locale name, the wide entry point following the narrow one it sits next to. The wide facet carries the narrow vtable because [GuestLocale](../../../core/libs/prx/libc/include/GuestLocale.hpp) models one facet layout and one vtable for both element types, so a `collate<wchar_t>` comparison, transform or hash runs the narrow `do_compare`, `do_transform` and `do_hash` over the guest's 16-bit characters read as bytes; the console's `libc.prx` exports both `_Getcat` (NIDs BSVJqITGCyI and 2wz4rthdiy8) with identical 0x19d-byte bodies, believed to build the same 0x20-byte facet; the console bodies themselves are unreadable (obfuscated dynsym), the wide vtable is not reproduced here and no console run checks the two. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size. Its result capacity (`length | 15`) and the over-aligned storage of results of 4080 characters or more follow the console's `libc.prx` code and are not checked on hardware. No module of the PPSA59322, PPSA76677 or PPSA97358 dumps in this tree imports either `_Getcat`, so neither has been exercised by a title.
+- [collate<char> and collate<wchar_t> _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build the facet for the C locale only and throw for any other locale name, the wide entry point following the narrow one it sits next to. The wide facet carries its own vtable, whose `compare`, `transform` and `hash` take the guest's 16-bit characters, so the length of a range is a count of characters: `compare` orders by 16-bit value (the C locale, as Dinkumware's `char_traits<wchar_t>::compare` does), `transform` copies the characters into the result as the guest's bytes and sizes the result in bytes (two per character, a 16-bit terminator, the 7 characters that still fit next to it and `length * 2 | 15` plus two bytes of storage beyond that), and `hash` mixes one whole character per round, so wide text below 0x100 hashes like its narrow spelling and a character above 0xff hashes as one value instead of the two bytes it holds. The console's `libc.prx` exports both `_Getcat` (NIDs BSVJqITGCyI and 2wz4rthdiy8) with identical 0x19d-byte bodies, believed to build the same 0x20-byte facet; the console bodies themselves are unreadable (obfuscated dynsym, and the text of the relinked `libc.prx` is zeroed), so the wide entries follow the narrow algorithm and the C locale instead of the console's wide code, and no console run checks the two. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size. The narrow result capacity (`length | 15`) and the over-aligned storage of results of 4080 characters or more follow the console's `libc.prx` code and are not checked on hardware. No module of the PPSA59322, PPSA76677 or PPSA97358 dumps in this tree imports either `_Getcat`, so neither has been exercised by a title.
 - [The scanf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc), the `_s` forms included, on Linux throws for the 16-bit wide conversions (`%ls`, `%lc`, `%l[`, `%S`, `%C`): glibc would store a 32-bit `wchar_t` per character into the guest's 16-bit buffer. Windows hands `%ls`, `%lc` and `%l[` to the host, whose `wchar_t` is 16 bits
 - [The scanf family](../../../core/libs/prx/libc/include/WindowsScanning.hpp) (libc) on Windows returns 0, as C17 and FreeBSD do, when input fails after only suppressed conversions have completed (`sscanf("12", "%*d%d", &value)`). Linux hands the format to glibc, which returns `EOF` there. On Windows the host stops a `%x`, `%X`, `%i` or `%p` after the `x` of a `0x` that no hex digit follows (or that the width cuts off), and a float conversion (`%e`, `%f`, `%g`, `%a`) after the `e` of an exponent with no digits (`1e`), where C17 and FreeBSD stop before the `x` or the `e`
 - [Guest path resolution](../../../core/libs/prx/libc/src/General.cpp) falls back to ASCII case-insensitive component lookup, following [shadPS4's host filesystem](https://github.com/shadps4-emu/shadPS4/blob/945dbc3cc3eee80ac3e053b438502ed936fa6bb2/src/core/file_sys/backends/host_fs.cpp). PPSA03747 requests `Skyrim.INI`, `Data/Video` and lowercase archive/plugin names that differ from the dump's filenames. Exact host names take precedence; multiple fallback matches throw instead of selecting an arbitrary file. Existing parent names are resolved when creating a new file, and guest mount aliases use the same ASCII comparison. Results are not cached, so renames and newly created files remain visible. Unicode case folding and PS5 behaviour for case-colliding host entries are unverified; host roots are used as supplied. Explicit native Windows drive and UNC paths bypass guest parent-component validation.. A relinked module's `<name>.guest.prx` is matched the same way when the original module is not beside it: Borderlands 4 (PPSA01494) loads `/app0/prx/libRenoirCore.PS5.prx`, which it ships as `librenoircore.ps5.prx`