fix(libkernel): fail open, stat, lstat, access and unlink through a regular file with ENOTDIR on Windows

A path whose directory part is a regular file, such as file/child, failed
with ENOENT in open, stat, lstat, access and unlink on Windows, where
FreeBSD 11.4 lookup fails it with ENOTDIR (vfs_cache_lookup, sys/kern/vfs_cache.c).
mkdir and rmdir already did. A not-found error of these calls now asks whether the
nearest existing ancestor is not a directory and reports ENOTDIR then.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

core/libs/prx/libc/include/FilesystemError.hpp

@@ -23,6 +23,20 @@ inline int FilesystemError(const std::error_code& error) {
     return 5;
 }
 
+inline bool HasNonDirectoryPrefix_nid_no_patch(const std::filesystem::path& native) {
+    std::error_code error;
+    const auto trimmed = native.has_filename() ? native : native.parent_path();
+    for (auto prefix = trimmed.parent_path(); prefix.has_relative_path(); prefix = prefix.parent_path()) {
+        const auto status = std::filesystem::status(prefix, error);
+        if (std::filesystem::exists(status)) return !std::filesystem::is_directory(status);
+    }
+    return false;
+}
+
+inline int NotFoundError_nid_no_patch(const std::filesystem::path& native) {
+    return HasNonDirectoryPrefix_nid_no_patch(native) ? 20 : 2;
+}
+
 std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, bool replacesExisting);
 
 inline int ResolvePathError_nid_no_patch(const char* path, std::filesystem::path& resolved, bool module = false, bool archived = false) {

core/libs/prx/libc/src/Filesystem.cpp

@@ -21,6 +21,7 @@ extern "C" int APS5_VABI access_nid_postfix(const char* path, int mode) {
         const DWORD attributes = GetFileAttributesW(resolved.c_str());
         if (attributes == INVALID_FILE_ATTRIBUTES) {
             errno = FilesystemError(std::error_code(static_cast<int>(GetLastError()), std::system_category()));
+            if (errno == 2) errno = NotFoundError_nid_no_patch(resolved);
             return -1;
         }
         if (mode == 0) return 0;

core/libs/prx/libkernel/File/src/Open.cpp

@@ -228,7 +228,7 @@ static int OpenNative(const std::filesystem::path& native, int flags, std::uint1
     }
 #endif
     if (fd < 0) {
-        return SceErrorFromErrno(errno);
+        return SceErrorFromErrno(errno == ENOENT ? NotFoundError_nid_no_patch(native) : errno);
     }
     if ((flags & SCE_KERNEL_O_ACCMODE) != SCE_KERNEL_O_RDONLY || (flags & (SCE_KERNEL_O_CREAT | SCE_KERNEL_O_TRUNC)))
         RecordWrittenPath_nid_no_patch(native);
@@ -370,7 +370,7 @@ int APS5_VABI sceKernelStat(const char* path, FileStat* sb) {
     if (error == std::errc::not_a_directory) return SceErrorFromErrno(GuestEnotdir);
     if (error == std::errc::filename_too_long || error == std::errc::too_many_symbolic_link_levels) return SceKernelError(FilesystemError(error));
     if (!std::filesystem::exists(status)) {
-        return SceErrorFromErrno(2);
+        return SceErrorFromErrno(NotFoundError_nid_no_patch(native));
     }
     const std::string_view guestPath(path);
     if (!guestPath.empty() && guestPath.back() == '/' && !std::filesystem::is_directory(status)) return SceErrorFromErrno(GuestEnotdir);
@@ -388,7 +388,7 @@ int APS5_VABI sceKernelUnlink(const char* path) {
     std::error_code error;
     if (std::filesystem::is_directory(std::filesystem::symlink_status(native, error))) return SceErrorFromErrno(GuestEperm);
     if (NativeUnlink(native) != 0) {
-        return SceErrorFromErrno(errno);
+        return SceErrorFromErrno(errno == ENOENT ? NotFoundError_nid_no_patch(native) : errno);
     }
     RecordWrittenPath_nid_no_patch(native);
     return 0;

core/libs/prx/libkernel/File/src/Stdio.cpp

@@ -964,7 +964,7 @@ int APS5_VABI lstat_nid_postfix(const char* path, FileStat* sb) {
     if (const int error = PathError(path)) return PosixFailure(error);
     std::filesystem::path native;
     if (const int error = ResolvePathError_nid_no_patch(path, native)) return PosixFailure(error);
-    if (!File::FillLinkStat(native, sb)) return PosixResult(SceErrorFromErrno(errno));
+    if (!File::FillLinkStat(native, sb)) return PosixResult(SceErrorFromErrno(errno == ENOENT ? NotFoundError_nid_no_patch(native) : errno));
     return 0;
 }
 

core/libs/tests/GuestFilesystem.cpp

@@ -1145,6 +1145,17 @@ int main() {
     Require(rmdir_nid_postfix((missingName + "/").c_str()) == -1 && *__error_nid_postfix() == 2);
     Require(sceKernelRmdir((presentName + "/").c_str()) == static_cast<int>(0x80020014u));
     Require(sceKernelRmdir((missingName + "/").c_str()) == static_cast<int>(0x80020002u));
+    const auto missingChild = missingName + "/child";
+    for (const auto* blocked : {&underFile, &deepUnderFile, &missingChild}) {
+        const int expected = blocked == &missingChild ? 2 : 20;
+        FileStat blockedStatus{};
+        Require(open_nid_postfix(blocked->c_str(), 0, 0) == -1 && *__error_nid_postfix() == expected);
+        Require(open_nid_postfix(blocked->c_str(), 0x202, 0644) == -1 && *__error_nid_postfix() == expected);
+        Require(stat_nid_postfix(blocked->c_str(), &blockedStatus) == -1 && *__error_nid_postfix() == expected);
+        Require(lstat_nid_postfix(blocked->c_str(), &blockedStatus) == -1 && *__error_nid_postfix() == expected);
+        Require(access_nid_postfix(blocked->c_str(), 0) == -1 && *__error_nid_postfix() == expected);
+        Require(unlink_nid_postfix(blocked->c_str()) == -1 && *__error_nid_postfix() == expected);
+    }
 #ifndef _WIN32
     const auto linkedDirectory = root / "linked-directory";
     const auto directoryLink = root / "directory-link";