fix(libSceFiber): run fibers on host stacks with room for host calls

Host code that a guest fiber calls runs on the fiber's stack. Demon's
Souls (PPSA01341) gives its job fibers 16 KiB contexts packed next to each
other, and a fiber that reaches shader preparation through
Driver::ResolveGraphicsStagesAbi goes about 21 KiB deep when SPIRV-Tools
validates the module. That overwrites the saved frame of the fiber below,
and the next switch to that fiber pops garbage into rbp, rbx, r12-r15 and
rip (a jump to 0x1a in the observed crash).

Fibers now run on an mmap'd host stack of the context size plus 1 MiB,
with a guard page at the bottom, the same margin guest threads get since
#2647. The guest context is no longer written. Context size checks fill
and scan the top context-size bytes of the host stack, so
size_context_margin keeps its meaning. If the mapping fails the fiber
runs in its context as before. Windows is unchanged: it pins and
publishes the guest context as the stack, and that path is not covered
here.

The fiber test now runs a 256 KiB host call from a 16 KiB fiber placed at
the top of a guard-filled area and checks nothing below the context was
written. It fails on main.

core/libs/prx/libSceFiber/Export.cpp

@@ -15,6 +15,9 @@
 #ifdef _WIN32
 #include <windows.h>
 #include "prx/libc/include/GuestArena.hpp"
+#else
+#include <sys/mman.h>
+#include <unistd.h>
 #endif
 
 static constexpr int32_t SCE_OK = 0;
@@ -29,6 +32,7 @@ static constexpr std::size_t FIBER_OBJECT_SIZE = 0x100;
 static constexpr std::size_t FIBER_OPT_PARAM_SIZE = 0x80;
 static constexpr std::size_t FIBER_MIN_CONTEXT_SIZE = 512;
 static constexpr std::uint64_t FIBER_CONTEXT_FILL = 0xdeadbeefdeadbeefull;
+static constexpr std::size_t FIBER_HOST_STACK_MARGIN = 1u << 20;
 
 using GuestFiberEntry = void (APS5_VABI*)(std::uint64_t argOnInitialize, std::uint64_t argOnRun);
 
@@ -53,6 +57,8 @@ struct Fiber {
     void* savedStack;
     char name[FIBER_MAX_NAME_LENGTH + 1];
     bool contextSizeCheck;
+    std::uint8_t* hostStack;
+    std::uint64_t hostStackBytes;
 };
 static_assert(sizeof(Fiber) <= FIBER_OBJECT_SIZE, "guest reserves 0x100 bytes for SceFiber");
 
@@ -259,6 +265,44 @@ static void UnpinStack(const void*, std::uint64_t) {}
 
 #endif
 
+#ifdef _WIN32
+
+static void AllocateHostStack(Fiber*) {}
+
+static void ReleaseHostStack(Fiber*) {}
+
+#else
+
+static void AllocateHostStack(Fiber* fiber) {
+    const auto page = static_cast<std::uint64_t>(sysconf(_SC_PAGESIZE));
+    const auto bytes = (fiber->contextSize + FIBER_HOST_STACK_MARGIN + page - 1) / page * page + page;
+    void* mapping = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_STACK, -1, 0);
+    if (mapping == MAP_FAILED) {
+        std::fprintf(stderr, "[fiber] no host stack for '%s' (0x%llx bytes); it runs in its 0x%llx byte context\n", fiber->name,
+                     static_cast<unsigned long long>(bytes), static_cast<unsigned long long>(fiber->contextSize));
+        return;
+    }
+    mprotect(mapping, page, PROT_NONE);
+    fiber->hostStack = static_cast<std::uint8_t*>(mapping);
+    fiber->hostStackBytes = bytes;
+}
+
+static void ReleaseHostStack(Fiber* fiber) {
+    if (fiber->hostStack) munmap(fiber->hostStack, fiber->hostStackBytes);
+    fiber->hostStack = nullptr;
+    fiber->hostStackBytes = 0;
+}
+
+#endif
+
+static std::uint8_t* StackTop(const Fiber* fiber) {
+    return fiber->hostStack ? fiber->hostStack + fiber->hostStackBytes : fiber->context + fiber->contextSize;
+}
+
+static std::uint64_t* SizedWindow(const Fiber* fiber) {
+    return reinterpret_cast<std::uint64_t*>(StackTop(fiber) - fiber->contextSize);
+}
+
 static StackBounds FiberBounds(const Fiber* fiber) {
     return {fiber->context + fiber->contextSize, fiber->context, fiber->context};
 }
@@ -275,7 +319,7 @@ extern "C" [[noreturn]] void Aps5FiberMain_nid_no_patch(Fiber* fiber) {
 }
 
 static void PrepareInitialStack(Fiber* fiber) {
-    const auto top = reinterpret_cast<std::uintptr_t>(fiber->context + fiber->contextSize) & ~static_cast<std::uintptr_t>(15);
+    const auto top = reinterpret_cast<std::uintptr_t>(StackTop(fiber)) & ~static_cast<std::uintptr_t>(15);
     auto* frame = reinterpret_cast<InitialFrame*>(top - 256);
     std::memset(frame, 0, sizeof(*frame));
     frame->mxcsr = _mm_getcsr();
@@ -330,7 +374,10 @@ int32_t APS5_VABI _sceFiberInitializeImpl_nid_postfix(FiberObject* object, const
     if (addr_context == nullptr) return SCE_FIBER_ERROR_INVALID;
     if (size_context < FIBER_MIN_CONTEXT_SIZE) return SCE_FIBER_ERROR_RANGE;
     auto* fiber = reinterpret_cast<Fiber*>(object);
-    if (fiber->magic == FIBER_MAGIC) UnpinStack(fiber->context, fiber->contextSize);
+    if (fiber->magic == FIBER_MAGIC) {
+        UnpinStack(fiber->context, fiber->contextSize);
+        ReleaseHostStack(fiber);
+    }
     std::memset(object, 0, FIBER_OBJECT_SIZE);
     fiber->magic = FIBER_MAGIC;
     fiber->state.store(FiberState::Idle, std::memory_order_relaxed);
@@ -339,9 +386,10 @@ int32_t APS5_VABI _sceFiberInitializeImpl_nid_postfix(FiberObject* object, const
     fiber->context = static_cast<std::uint8_t*>(addr_context);
     fiber->contextSize = size_context;
     std::strncpy(fiber->name, name, FIBER_MAX_NAME_LENGTH);
+    AllocateHostStack(fiber);
     fiber->contextSizeCheck = g_contextSizeCheck.load(std::memory_order_relaxed);
     if (fiber->contextSizeCheck) {
-        auto* words = static_cast<std::uint64_t*>(addr_context);
+        auto* words = SizedWindow(fiber);
         std::fill(words, words + size_context / sizeof(std::uint64_t), FIBER_CONTEXT_FILL);
     }
     PinStack(addr_context, size_context);
@@ -356,6 +404,7 @@ int32_t APS5_VABI sceFiberFinalize(FiberObject* object) {
     if (state == FiberState::Running || state == FiberState::Suspending) return SCE_FIBER_ERROR_STATE;
     fiber->magic = 0;
     UnpinStack(fiber->context, fiber->contextSize);
+    ReleaseHostStack(fiber);
     return SCE_OK;
 }
 
@@ -430,7 +479,7 @@ int32_t APS5_VABI sceFiberGetInfo(FiberObject* object, FiberInfo* fiber_info) {
     std::memcpy(fiber_info->name, fiber->name, sizeof(fiber_info->name));
     fiber_info->size_context_margin = static_cast<uint64_t>(-1);
     if (fiber->contextSizeCheck) {
-        const auto* words = reinterpret_cast<const std::uint64_t*>(fiber->context);
+        const auto* words = SizedWindow(fiber);
         const auto* end = words + fiber->contextSize / sizeof(std::uint64_t);
         fiber_info->size_context_margin = static_cast<uint64_t>(std::find_if(words, end, [](std::uint64_t word) { return word != FIBER_CONTEXT_FILL; }) - words) * sizeof(std::uint64_t);
     }

core/libs/tests/Fiber.cpp

@@ -109,6 +109,43 @@ void RunSmallContextFirst() {
     }
 }
 
+constexpr std::size_t HostCallContextSize = 16 * 1024;
+constexpr std::size_t HostCallStack = 256 * 1024;
+constexpr std::size_t HostCallFrameSize = 4096;
+constexpr unsigned char HostCallGuardFill = 0x5c;
+alignas(16) std::array<unsigned char, HostCallStack + 64 * 1024> g_hostCallArea;
+FiberStorage g_hostCallStorage;
+
+std::size_t DescendHostStack(std::size_t remaining);
+std::size_t (*volatile g_descendHostStack)(std::size_t) = DescendHostStack;
+
+std::size_t DescendHostStack(std::size_t remaining) {
+    volatile unsigned char frame[HostCallFrameSize];
+    frame[0] = 0x11;
+    frame[HostCallFrameSize - 1] = 0x22;
+    const std::size_t frames = remaining > HostCallFrameSize ? g_descendHostStack(remaining - HostCallFrameSize) : 0;
+    Require(frame[0] == 0x11 && frame[HostCallFrameSize - 1] == 0x22, "host call frame intact");
+    return frames + 1;
+}
+
+void APS5_VABI HostCallEntry(std::uint64_t, std::uint64_t) {
+    Require(g_descendHostStack(HostCallStack) == HostCallStack / HostCallFrameSize, "host call depth");
+    std::uint64_t received = 0;
+    for (;;) sceFiberReturnToThread(0, &received);
+}
+
+void RunHostCallOnSmallContext() {
+    auto* fiber = reinterpret_cast<FiberObject*>(&g_hostCallStorage);
+    auto* context = g_hostCallArea.data() + g_hostCallArea.size() - HostCallContextSize;
+    g_hostCallArea.fill(HostCallGuardFill);
+    Require(_sceFiberInitializeImpl_nid_postfix(fiber, "host call", HostCallEntry, 0, context, HostCallContextSize, nullptr, 0) == 0, "initialize host call fiber");
+    Require(sceFiberRun_nid_postfix(fiber, 0, nullptr) == 0, "run a 256 KiB host call from a 16 KiB fiber");
+    Require(sceFiberFinalize(fiber) == 0, "finalize host call fiber");
+    for (auto* byte = g_hostCallArea.data(); byte != context; ++byte) {
+        Require(*byte == HostCallGuardFill, "a host call from a 16 KiB fiber writes nothing below its context");
+    }
+}
+
 [[gnu::noinline]] std::int32_t RunFirst(std::uint64_t* returned) {
     g_threadFramePointer = reinterpret_cast<std::uint64_t>(__builtin_frame_address(0));
     const std::int32_t result = sceFiberRun_nid_postfix(g_first, 100, returned);
@@ -154,4 +191,6 @@ int main() {
     Require(info.size_context_margin > 0 && info.size_context_margin < g_checkedContext.size() - 4096, "margin below the used stack");
     Require(info.size_context_margin % 8 == 0, "margin in whole words");
     Require(sceFiberStopContextSizeCheck() == 0, "stop size check");
+
+    RunHostCallOnSmallContext();
 }