fix(libkernel): account for fixed flexible memory replacements

core/libs/prx/libkernel/DirectMemory/Export.cpp

@@ -123,7 +123,20 @@ int _mapFlexible(void** addr, size_t len, int prot, int flags) {
     const size_t configured = _configuredFlexibleSize();
     const size_t images = GuestImageSegmentBytes_nid_no_patch();
     std::lock_guard lock(g_flexibleLock);
-    const size_t used = images + _flexibleUsedLocked();
+    size_t replaced = 0;
+    constexpr int MapFixed = 0x10;
+    constexpr int MapNoOverwrite = 0x80;
+    if (addr != nullptr && *addr != nullptr && (flags & (MapFixed | MapNoOverwrite)) == MapFixed) {
+        const auto start = reinterpret_cast<uintptr_t>(*addr);
+        if (len <= std::numeric_limits<uintptr_t>::max() - start) {
+            const auto end = start + len;
+            for (const auto& [rangeStart, rangeLength] : g_flexibleRanges) {
+                const auto rangeEnd = rangeStart + rangeLength;
+                if (rangeStart < end && rangeEnd > start) replaced += std::min(end, rangeEnd) - std::max(start, rangeStart);
+            }
+        }
+    }
+    const size_t used = images + _flexibleUsedLocked() - replaced;
     if (len > configured - std::min(configured, used)) {
         std::fprintf(stderr, "[memory] flexible memory exhausted: request 0x%zx with 0x%zx of 0x%zx in use\n", len, used, configured);
         return SCE_KERNEL_ERROR_ENOMEM;

core/libs/tests/GuestFlexibleMemoryBudget.cpp

@@ -72,11 +72,71 @@ int main(int argc, char** argv) {
     Require(sceKernelMapFlexibleMemory(&oversized, before + PS5_PAGE_SIZE, 3, 0) == SCE_KERNEL_ERROR_ENOMEM && oversized == nullptr);
     Require(sceKernelAvailableFlexibleMemorySize(&available) == 0 && available == before);
     if (mode == "limited") {
+        const auto checkBudget = [&](std::size_t expected) {
+            std::size_t current = 0;
+            Require(sceKernelAvailableFlexibleMemorySize(&current) == 0);
+            std::printf("Fixed flexible mapping: available=%zu expected=%zu\n", current, expected);
+            std::fflush(stdout);
+            Require(current == expected);
+        };
+        constexpr auto page = PS5_PAGE_SIZE;
+        void* original = nullptr;
+        Require(sceKernelMapFlexibleMemory(&original, page * 4, 3, 0) == 0);
+        auto* bytes = static_cast<volatile unsigned char*>(original);
+        for (std::size_t i = 0; i < 4; ++i) bytes[i * page] = static_cast<unsigned char>(17 + i);
+        auto* middle = static_cast<unsigned char*>(original) + page;
+        void* replacement = middle;
+        Require(sceKernelMapFlexibleMemory(&replacement, page, 3, 0x10) == 0 && replacement == middle);
+        Require(bytes[0] == 17 && bytes[page] == 0 && bytes[page * 2] == 19 && bytes[page * 3] == 20);
+        checkBudget(before - page * 4);
+        bytes[page] = 23;
+        Require(sceKernelMapFlexibleMemory(&replacement, page, 3, 0x90) == SCE_KERNEL_ERROR_ENOMEM);
+        Require(bytes[page] == 23);
+        checkBudget(before - page * 4);
+        replacement = original;
+        Require(sceKernelMapFlexibleMemory(&replacement, page, 3, 0x10) == 0 && replacement == original);
+        Require(bytes[0] == 0 && bytes[page] == 23 && bytes[page * 3] == 20);
+        checkBudget(before - page * 4);
+        Require(sceKernelMapFlexibleMemory(&replacement, page * 3, 3, 0x10) == 0 && replacement == original);
+        Require(bytes[0] == 0 && bytes[page] == 0 && bytes[page * 2] == 0 && bytes[page * 3] == 20);
+        checkBudget(before - page * 4);
+        Require(sceKernelReleaseFlexibleMemory(middle, page * 2) == 0);
+        checkBudget(before - page * 2);
+        replacement = middle;
+        Require(sceKernelMapFlexibleMemory(&replacement, page * 3, 3, 0x10) == 0 && replacement == middle);
+        Require(bytes[page] == 0 && bytes[page * 3] == 0);
+        checkBudget(before - page * 4);
+        Require(sceKernelReleaseFlexibleMemory(original, page * 4) == 0);
+        checkBudget(before);
         const auto length = before & ~(PS5_PAGE_SIZE - 1);
         void* full = nullptr;
         Require(sceKernelMapFlexibleMemory(&full, length, 3, 0) == 0);
         void* extra = nullptr;
         Require(sceKernelMapFlexibleMemory(&extra, PS5_PAGE_SIZE, 3, 0) == SCE_KERNEL_ERROR_ENOMEM && extra == nullptr);
+        static_cast<volatile unsigned char*>(full)[0] = 31;
+        replacement = full;
+        Require(sceKernelMapFlexibleMemory(&replacement, length, 3, 0x90) == SCE_KERNEL_ERROR_ENOMEM);
+        Require(static_cast<volatile unsigned char*>(full)[0] == 31);
+        checkBudget(before - length);
+        auto* fullBytes = static_cast<volatile unsigned char*>(full);
+        fullBytes[page] = 37;
+        fullBytes[length - page] = 41;
+        auto* fullMiddle = static_cast<unsigned char*>(full) + page;
+        replacement = fullMiddle;
+        Require(sceKernelMapFlexibleMemory(&replacement, page, 3, 0x10) == 0 && replacement == fullMiddle);
+        Require(fullBytes[0] == 31 && fullBytes[page] == 0 && fullBytes[length - page] == 41);
+        checkBudget(before - length);
+        replacement = static_cast<unsigned char*>(full) + length - page;
+        Require(sceKernelMapFlexibleMemory(&replacement, page * 2, 3, 0x10) == SCE_KERNEL_ERROR_ENOMEM);
+        Require(fullBytes[0] == 31 && fullBytes[length - page] == 41);
+        checkBudget(before - length);
+        replacement = full;
+        Require(sceKernelMapFlexibleMemory(&replacement, length + page, 3, 0x10) == SCE_KERNEL_ERROR_ENOMEM);
+        Require(static_cast<volatile unsigned char*>(full)[0] == 31);
+        checkBudget(before - length);
+        Require(sceKernelMapFlexibleMemory(&replacement, length, 3, 0x10) == 0 && replacement == full);
+        Require(static_cast<volatile unsigned char*>(full)[0] == 0);
+        checkBudget(before - length);
         Require(sceKernelReleaseFlexibleMemory(full, length) == 0);
         Require(sceKernelAvailableFlexibleMemorySize(&available) == 0 && available == before);
     }