diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 328669a882..08279beaa2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,13 +22,18 @@ on: required: false default: true type: boolean + select: + description: Build and run only the tests whose build inputs the pull requests change (push builds of main always run every test) + required: false + default: false + type: boolean permissions: contents: read concurrency: group: build-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.pr || github.ref }} - cancel-in-progress: ${{ github.event_name != 'push' }} + cancel-in-progress: true env: CONFIGURE: >- @@ -300,8 +305,6 @@ jobs: CCACHE_COMPILERCHECK: content MESA_SHADER_CACHE_DIR: /var/cache/anyps5/mesa MESA_SHADER_CACHE_MAX_SIZE: 4G - CMAKE_BUILD_PARALLEL_LEVEL: 1 - CTEST_PARALLEL_LEVEL: 1 steps: - uses: actions/checkout@v7 with: @@ -335,16 +338,87 @@ jobs: ccache --show-config dpkg-query -W mesa-vulkan-drivers grep -m1 '^model name' /proc/cpuinfo - - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON + - name: Install the wild linker + shell: bash + env: + WILD_VERSION: 0.10.0 + WILD_SHA256: 641265506a7c06cfb03181b8916ab663ec8407855db6d4db7f8450667d105283 + run: | + tools=$RUNNER_TOOL_CACHE/wild + mkdir -p "$tools" 2>/dev/null && test -w "$tools" || { tools=$RUNNER_TEMP/wild; mkdir -p "$tools"; } + dir=$tools/$WILD_VERSION + tmp= + probe=$(mktemp -d "$RUNNER_TEMP/wild-probe.XXXXXX") + trap 'rm -rf "$tmp" "$probe"' EXIT + if [ ! -x "$dir/bin/ld" ]; then + tmp=$(mktemp -d "$tools/.wild.XXXXXX") + name=wild-linker-$WILD_VERSION-x86_64-unknown-linux-gnu + curl -fsSL --retry 3 -o "$tmp/$name.tar.gz" "https://github.com/davidlattimore/wild/releases/download/$WILD_VERSION/$name.tar.gz" + echo "$WILD_SHA256 $tmp/$name.tar.gz" | sha256sum -c - + tar -xzf "$tmp/$name.tar.gz" -C "$tmp" + mkdir -p "$tmp/out/bin" + mv "$tmp/$name/wild" "$tmp/out/bin/wild" + ln -s wild "$tmp/out/bin/ld" + mv -T "$tmp/out" "$dir" 2>/dev/null || test -x "$dir/bin/ld" + fi + echo 'int main(void) { return 0; }' > "$probe/main.c" + gcc "-B$dir/bin" "$probe/main.c" -o "$probe/main" + readelf -p .comment "$probe/main" | grep -F "Linker: Wild $WILD_VERSION" + echo "LINKER_FLAGS=-B$dir/bin" >> "$GITHUB_ENV" + - name: Size parallelism to what the shared host has free + run: | + cpus=$(nproc) + busy=$(awk '{ printf "%d", $1 + 0.5 }' /proc/loadavg) + mem_mb=$(awk '/^MemTotal:/ { print int($2 / 1024) }' /proc/meminfo) + slots=$(( (mem_mb - 2048) / 1024 )) + slots=$(( slots < 1 ? 1 : slots )) + tests=$(( cpus - busy )) + tests=$(( tests < 1 ? 1 : tests )) + slot_dir=$CCACHE_DIR/compile-slots + mkdir -p -m 1777 "$slot_dir" + test -w "$slot_dir" + echo "cpus=$cpus load=$busy mem_total_mb=$mem_mb -> build jobs $cpus, cold compiles $slots per host, tests $tests" + echo "CMAKE_BUILD_PARALLEL_LEVEL=$cpus" >> "$GITHUB_ENV" + echo "CTEST_PARALLEL_LEVEL=$tests" >> "$GITHUB_ENV" + echo "CCACHE_PREFIX=$GITHUB_WORKSPACE/tools/compile_slot.sh" >> "$GITHUB_ENV" + echo "ANYPS5_COMPILE_SLOT_DIR=$slot_dir" >> "$GITHUB_ENV" + echo "ANYPS5_COMPILE_SLOTS=$slots" >> "$GITHUB_ENV" + - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON "-DCMAKE_EXE_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_SHARED_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_MODULE_LINKER_FLAGS=$LINKER_FLAGS" - name: Check that every test executable is registered with ctest run: python3 tools/check_test_registration.py build - - run: cmake --build build - - run: cmake --build build --target libs - - run: ctest --test-dir build --output-on-failure --timeout 120 --output-log ctest.log --output-junit ctest.xml + - if: ${{ inputs.select }} + uses: actions/cache/restore@v6 + with: + path: test-graph.json + key: test-graph-${{ runner.os }}-${{ needs.eligibility.outputs.sha }} + restore-keys: test-graph-${{ runner.os }}- + - name: Select the tests the change reaches + if: ${{ inputs.select }} + env: + BASE: ${{ needs.eligibility.outputs.sha }} + run: | + git diff --name-only "$BASE" HEAD > changed.txt + python3 tools/select_tests.py select build test-graph.json changed.txt + - id: build + run: | + if [ -f build/select.ninja ]; then ninja -C build -f select.ninja -j "$CMAKE_BUILD_PARALLEL_LEVEL" -l "$(nproc)" selected; else cmake --build build -- -l "$(nproc)"; fi + - id: libs + run: cmake --build build --target libs -- -l "$(nproc)" + - run: python3 tools/select_tests.py ctest build --test-load "$(nproc)" --output-on-failure --timeout 120 --output-log ctest.log --output-junit ctest.xml env: ANYPS5_REQUIRE_VULKAN: 1 GALLIUM_OVERRIDE_CPU_CAPS: avx - run: python3 tools/check_skips.py build/ctest.xml + - name: Write the test graph of main + if: ${{ !cancelled() && github.event_name == 'push' && steps.build.outcome == 'success' && steps.libs.outcome == 'success' }} + continue-on-error: true + run: python3 tools/select_tests.py dump build test-graph.json + - if: ${{ !cancelled() && github.event_name == 'push' && hashFiles('test-graph.json') != '' }} + continue-on-error: true + uses: actions/cache/save@v6 + with: + path: test-graph.json + key: test-graph-${{ runner.os }}-${{ github.sha }} - name: Report compiler cache statistics if: ${{ !cancelled() }} run: ccache --show-stats @@ -372,7 +446,6 @@ jobs: with: ref: ${{ needs.eligibility.outputs.sha }} persist-credentials: false - submodules: recursive - name: Merge the pull request into main if: ${{ needs.eligibility.outputs.head != '' }} shell: bash @@ -389,7 +462,28 @@ jobs: if [ "$(git rev-parse "refs/remotes/pull/${numbers[i]}")" != "${heads[i]}" ]; then echo "::error::Pull request #${numbers[i]} has newer commits; start a new build"; exit 1; fi if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit "refs/remotes/pull/${numbers[i]}"; then echo "::error::Pull request #${numbers[i]} does not merge cleanly into main and the pull requests before it; rebase it first"; exit 1; fi done - git submodule update --init --recursive + - id: submodules + name: Key the submodules on their commits + run: | + echo "key=$(git ls-files -s | awk '$1 == "160000" { print $2, $4 }' | cat - .gitmodules | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" + echo "ffmpeg=$(git rev-parse HEAD:3rdparty/ffmpeg-core)" >> "$GITHUB_OUTPUT" + - id: submodule-cache + uses: actions/cache/restore@v6 + with: + path: ${{ runner.temp }}/submodules.tar + key: submodules-${{ runner.os }}-${{ steps.submodules.outputs.key }} + - name: Check out the submodules + run: | + if [ -f "$RUNNER_TEMP/submodules.tar" ]; then tar --force-local -xf "$RUNNER_TEMP/submodules.tar"; fi + git submodule update --init --recursive --depth 1 --jobs 8 + - if: steps.submodule-cache.outputs.cache-hit != 'true' + run: tar --force-local -cf "$RUNNER_TEMP/submodules.tar" .git/modules 3rdparty + - if: steps.submodule-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + continue-on-error: true + with: + path: ${{ runner.temp }}/submodules.tar + key: submodules-${{ runner.os }}-${{ steps.submodules.outputs.key }} - name: Point the WinLibs CMake at the CA bundle of Git for Windows run: | for prefix in ucrt64 mingw64; do @@ -412,18 +506,70 @@ jobs: key: windows save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} evict-old-files: job + - id: ffmpeg-cache + uses: actions/cache/restore@v6 + with: + path: build/externals + key: ffmpeg-${{ runner.os }}-${{ steps.submodules.outputs.ffmpeg }} + - id: seed-cache + uses: actions/cache/restore@v6 + with: + path: configure-seed.cmake + key: configure-seed-${{ env.WINLIBS_ZIP }}-${{ steps.submodules.outputs.key }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'core/libs/CMakeLists.txt', '.github/workflows/build.yml') }} - shell: cmd - run: cmake -S . -B build %CONFIGURE% "-DANYPS5_AGC_EXECUTION_TESTS=F16ToF32Unpack;F64FmaSubnormals;RegistrationPrepareShutdown" + run: if exist configure-seed.cmake (cmake -C configure-seed.cmake -S . -B build %CONFIGURE% "-DANYPS5_AGC_EXECUTION_TESTS=F16ToF32Unpack;F64FmaSubnormals;RegistrationPrepareShutdown") else (cmake -S . -B build %CONFIGURE% "-DANYPS5_AGC_EXECUTION_TESTS=F16ToF32Unpack;F64FmaSubnormals;RegistrationPrepareShutdown") + - if: steps.seed-cache.outputs.cache-hit != 'true' + shell: cmd + run: python tools/configure_seed.py build/CMakeCache.txt configure-seed.cmake + - if: steps.seed-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + continue-on-error: true + with: + path: configure-seed.cmake + key: ${{ steps.seed-cache.outputs.cache-primary-key }} + - if: steps.ffmpeg-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + continue-on-error: true + with: + path: build/externals + key: ${{ steps.ffmpeg-cache.outputs.cache-primary-key }} - name: Check that every test executable is registered with ctest shell: cmd run: python tools/check_test_registration.py build - - shell: cmd - run: cmake --build build - - shell: cmd + - if: ${{ inputs.select }} + uses: actions/cache/restore@v6 + with: + path: test-graph.json + key: test-graph-${{ runner.os }}-${{ needs.eligibility.outputs.sha }} + restore-keys: test-graph-${{ runner.os }}- + - name: Select the tests the change reaches + if: ${{ inputs.select }} + env: + BASE: ${{ needs.eligibility.outputs.sha }} + shell: cmd + run: | + git diff --name-only %BASE% HEAD > changed.txt + python tools/select_tests.py select build test-graph.json changed.txt + - id: build + shell: cmd + run: if exist build\select.ninja (ninja -C build -f select.ninja selected) else (cmake --build build) + - id: libs + shell: cmd run: cmake --build build --target libs - shell: cmd - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml + run: python tools/select_tests.py ctest build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml - run: python tools/check_skips.py build/ctest.xml --job windows + - name: Write the test graph of main + if: ${{ !cancelled() && github.event_name == 'push' && steps.build.outcome == 'success' && steps.libs.outcome == 'success' }} + continue-on-error: true + shell: cmd + run: python tools/select_tests.py dump build test-graph.json + - if: ${{ !cancelled() && github.event_name == 'push' && hashFiles('test-graph.json') != '' }} + continue-on-error: true + uses: actions/cache/save@v6 + with: + path: test-graph.json + key: test-graph-${{ runner.os }}-${{ github.sha }} - name: Report tests slower than 30 s if: ${{ !cancelled() }} shell: bash diff --git a/.gitmodules b/.gitmodules index efcc5aa70b..f68f95f507 100644 --- a/.gitmodules +++ b/.gitmodules @@ -40,3 +40,9 @@ [submodule "3rdparty/mbedtls"] path = 3rdparty/mbedtls url = https://github.com/Mbed-TLS/mbedtls.git +[submodule "3rdparty/zstd"] + path = 3rdparty/zstd + url = https://github.com/facebook/zstd.git +[submodule "3rdparty/ZArchive"] + path = 3rdparty/ZArchive + url = https://github.com/Exzap/ZArchive.git diff --git a/3rdparty/ZArchive b/3rdparty/ZArchive new file mode 160000 index 0000000000..965b66c8d6 --- /dev/null +++ b/3rdparty/ZArchive @@ -0,0 +1 @@ +Subproject commit 965b66c8d67b6b7e30fd63b3b75aa91a99ff303b diff --git a/3rdparty/zstd b/3rdparty/zstd new file mode 160000 index 0000000000..f8745da6ff --- /dev/null +++ b/3rdparty/zstd @@ -0,0 +1 @@ +Subproject commit f8745da6ff1ad1e7bab384bd1f9d742439278e99 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..a1e14a99f6 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,14 @@ +# AGENTS.md + +AnyPS5 relinks PS5 executables into native Linux or Windows programs that run against reimplemented system libraries; it is not an emulator. The rules are [CONTRIBUTING.md](CONTRIBUTING.md) and [CONVENTIONS](docs/dev/CONVENTIONS.md); these are the ones agents break. + +- No comments in code, except `#endif` and namespace ends. +- An export you cannot implement calls `NotImplemented_nid_no_patch(__func__)`; never return a made-up success. +- Implement the function for every input. Never special-case a value, ID or path seen in one title's trace; if the general behaviour is unknown, throw and record it in `docs/dev/debt/.md`. +- New third-party code is a submodule at `3rdparty/`: add its `.gitmodules` entry (`git submodule add 3rdparty/`) and build it from source with CMake. Never `find_package` or copy sources in. +- Known gaps go in `docs/dev/debt/.md`; notes, reports and logs go in the pull request. Add no other files. +- Commit subjects and PR titles are Conventional Commits. +- PR body: copy [the template](.github/pull_request_template.md), keep every line, tick every box, and fill `What`, `Tested`, `Depends on` (#N or none), `Unblocks` (title ID and discussion, or none), `GPUs tested` (AGC, shader recompiler, VideoOut; otherwise n/a) and `AI-assisted` (yes or no). +- Before pushing run `python3 tools/check_conventions.py --base origin/main`; before opening run `python3 tools/pr_overlap.py --branch `. +- Build and test: [BUILD](docs/dev/BUILD.md); a relinker-only change builds with `-DANYPS5_RELINKER_ONLY=ON`. +- A speed claim shows `tools/trace_digest.py` summaries of `main` and of the change from the same machine and scene. diff --git a/CMakeLists.txt b/CMakeLists.txt index 23b603d59b..85d556c961 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,8 +135,12 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_build_eligibility.py) add_test(NAME check_skips COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_skips.py) + add_test(NAME select_tests COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_select_tests.py) add_test(NAME check_main_sync COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_main_sync.py) + add_test(NAME configure_seed COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_configure_seed.py) add_test(NAME stubgen COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_stubgen.py) add_test(NAME debt_split COMMAND ${Python3_EXECUTABLE} @@ -267,6 +271,8 @@ set(LIBUSB_ENABLE_UDEV OFF CACHE BOOL "" FORCE) add_subdirectory(3rdparty/libusb-cmake EXCLUDE_FROM_ALL) set_target_properties(usb-1.0 PROPERTIES POSITION_INDEPENDENT_CODE ON) +include(cmake/ZArchive.cmake) + add_subdirectory(core/shader/recompiler) add_subdirectory(core/Decoder/Jpeg) add_subdirectory(core/Decoder/Png) diff --git a/cmake/ZArchive.cmake b/cmake/ZArchive.cmake new file mode 100644 index 0000000000..c4c7eb2024 --- /dev/null +++ b/cmake/ZArchive.cmake @@ -0,0 +1,25 @@ +enable_language(C) + +set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_CONTRIB OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_SHARED OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_STATIC ON CACHE BOOL "" FORCE) +set(ZSTD_LEGACY_SUPPORT OFF CACHE BOOL "" FORCE) +set(ZSTD_MULTITHREAD_SUPPORT OFF CACHE BOOL "" FORCE) +add_subdirectory(${CMAKE_SOURCE_DIR}/3rdparty/zstd/build/cmake ${CMAKE_BINARY_DIR}/3rdparty/zstd EXCLUDE_FROM_ALL) +set_target_properties(libzstd_static PROPERTIES POSITION_INDEPENDENT_CODE ON) + +set(ZARCHIVE_SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/ZArchive) +add_library(zarchive STATIC EXCLUDE_FROM_ALL + ${ZARCHIVE_SOURCE_DIR}/src/zarchivereader.cpp + ${ZARCHIVE_SOURCE_DIR}/src/zarchivewriter.cpp + ${ZARCHIVE_SOURCE_DIR}/src/sha_256.c +) +target_include_directories(zarchive SYSTEM PUBLIC ${ZARCHIVE_SOURCE_DIR}/include) +target_compile_features(zarchive PUBLIC cxx_std_20) +target_link_libraries(zarchive PRIVATE libzstd_static) +set_target_properties(zarchive PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(zarchive INTERFACE LINKER:--exclude-libs,libzarchive.a:libzstd.a) +endif() diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index 8385138261..4e218b96f6 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -1,5 +1,7 @@ if(NOT WIN32) - string(REPLACE "-static " "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ") + foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static) + string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") + endforeach() string(STRIP "${CMAKE_EXE_LINKER_FLAGS}" CMAKE_EXE_LINKER_FLAGS) endif() @@ -221,6 +223,8 @@ if(BUILD_TESTING) if(Python3_Interpreter_FOUND) add_test(NAME nid_gnu_hash COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_gnu_hash.py $) + add_test(NAME nid_elf_duplicates COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_elf_duplicates.py $) add_test(NAME nid_patcher_macho COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_macho_nids.py $) add_test(NAME nid_patcher_native_paths COMMAND ${Python3_EXECUTABLE} diff --git a/core/libs/nid/src/ElfNidPatcher.cpp b/core/libs/nid/src/ElfNidPatcher.cpp index b5470147d9..7a6b8c3d1e 100644 --- a/core/libs/nid/src/ElfNidPatcher.cpp +++ b/core/libs/nid/src/ElfNidPatcher.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -408,6 +409,7 @@ void ElfNidPatcher::PatchNids(std::vector& elf, const std::string& }; std::vector uses; + std::unordered_map> patchedExports; for (std::size_t i = 1u; i < symCount; ++i) { const std::size_t symOffset = dynSymOffset + i * sizeof(Elf64_Sym); const auto sym = Read(elf, symOffset); @@ -424,6 +426,9 @@ void ElfNidPatcher::PatchNids(std::vector& elf, const std::string& const auto it = nidMap.find(symName); if (it != nidMap.end()) newValue = it->second; + const auto [existing, inserted] = patchedExports.try_emplace(newValue, sym.st_value, sym.st_shndx); + if (!inserted && existing->second != std::pair{sym.st_value, sym.st_shndx}) + throw std::runtime_error("duplicate patched export name: " + newValue); } else if (binding != kStbLocal && sym.st_shndx == kShnUndef) { const bool isUnversioned = versymOffset == 0u || (Read(elf, versymOffset + i * sizeof(std::uint16_t)) <= 1u); diff --git a/core/libs/nid/tests/test_elf_duplicates.py b/core/libs/nid/tests/test_elf_duplicates.py new file mode 100644 index 0000000000..504cf33bdc --- /dev/null +++ b/core/libs/nid/tests/test_elf_duplicates.py @@ -0,0 +1,69 @@ +import base64 +import hashlib +import sys +import unittest +from pathlib import Path + +from test_symbol_versions import build_image, patch, read_dynsym, IMPORTS, NID_NAME, SHN_UNDEF, STT_FUNC + + +def nid(name): + digest = hashlib.sha1(name.encode() + bytes.fromhex("518d64a635ded8c1e6b039b1c3e55230")).digest() + return base64.b64encode(digest[:8][::-1]).decode().rstrip("=").replace("/", "-") + + +def image(names, addresses=None): + if addresses is None: + addresses = [0x1200 + index * 0x20 for index in range(len(names))] + entries = [(None, 1, SHN_UNDEF, 0, 0)] + entries += [(name, 1, SHN_UNDEF, 0, 0) for name in IMPORTS] + entries += [(name, 1, 9, STT_FUNC, address) for name, address in zip(names, addresses)] + return bytes(build_image(entries, False)) + + +class ElfDuplicateTests(unittest.TestCase): + def test_exports_patched_to_one_name_are_rejected(self): + cases = ( + ["sceAlphaComputeLength", "sceAlphaComputeLength_nid_disambig1", "sceBetaCopyString"], + ["sceAlphaComputeLength_nid_postfix", "sceAlphaComputeLength_nid_disambig12", "sceBetaCopyString"], + ["sceBetaCopyString", "sceAlphaComputeLength_nid_no_patch", "sceAlphaComputeLength_nid_no_patch_nid_no_patch_cut"], + ) + for names in cases: + with self.subTest(names=names): + before = image(names) + result, after = patch(PATCHER, before, "duplicates.so") + self.assertEqual(result.returncode, 2, (result.returncode, result.stderr)) + self.assertIn(b"duplicate patched export name", result.stderr) + self.assertEqual(after, before) + + def test_aliases_of_one_address_are_patched(self): + names = ["sceAlphaComputeLength", "sceAlphaComputeLength_nid_postfix", "sceAlphaComputeLength_nid_no_patch_cut", "sceBetaCopyString"] + result, after = patch(PATCHER, image(names, [0x1200, 0x1200, 0x1200, 0x1220]), "aliases.so") + self.assertEqual(result.returncode, 0, result.stderr) + patched = [(entry[0], entry[2]) for entry in read_dynsym(after) if entry[1] == 9] + self.assertEqual(sorted(name for name, address in patched if address == 0x1200), + sorted(["sceAlphaComputeLength", "sceAlphaComputeLength", nid("sceAlphaComputeLength")])) + self.assertEqual([name for name, address in patched if address == 0x1220], [nid("sceBetaCopyString")]) + self.assertTrue(NID_NAME.match(nid("sceAlphaComputeLength"))) + + def test_aliases_of_different_addresses_are_rejected(self): + names = ["sceAlphaComputeLength", "sceAlphaComputeLength_nid_postfix", "sceAlphaComputeLength_nid_no_patch_cut"] + before = image(names, [0x1200, 0x1200, 0x1220]) + result, after = patch(PATCHER, before, "aliases.so") + self.assertEqual(result.returncode, 2, (result.returncode, result.stderr)) + self.assertIn(b"duplicate patched export name: sceAlphaComputeLength", result.stderr) + self.assertEqual(after, before) + + def test_distinct_exports_are_still_patched(self): + names = ["sceAlphaComputeLength_nid_disambig1", "sceBetaCopyString_nid_postfix", "sceGammaKeep_nid_no_patch_cut"] + result, after = patch(PATCHER, image(names), "distinct.so") + self.assertEqual(result.returncode, 0, result.stderr) + patched = [entry[0] for entry in read_dynsym(after) if entry[1] == 9] + self.assertEqual(len(patched), 3) + self.assertEqual(len(set(patched)), 3) + self.assertIn("sceGammaKeep", patched) + + +if __name__ == "__main__": + PATCHER = Path(sys.argv.pop(1)).resolve() + unittest.main() diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 34198c721c..eb1944fb08 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -154,6 +154,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPipeline.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPrecheck.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawRegisterKey.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawRelocate.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawTiming.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/IndirectDraw.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/EntryPoints.cpp @@ -173,6 +174,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Presentation.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Queues/QueueWorker.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Queues/Submission.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Queues/SubmissionRing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Queues/WorkerAffinity.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Shaders/ShaderDiagnostics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -204,6 +206,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Pipeline.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineLibrary.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineCache.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineJournal.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PlaceholderImages.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp @@ -348,6 +351,18 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() + add_test_executable(agc_async_log_tests tests/AsyncLog.cpp) + target_include_directories(agc_async_log_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(agc_async_log_tests PRIVATE libc) + configure_windows_unwind(agc_async_log_tests) + add_test(NAME agc_async_log COMMAND agc_async_log_tests) + add_test(NAME agc_async_log_fatal COMMAND agc_async_log_tests fatal) + set_tests_properties(agc_async_log agc_async_log_fatal PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(agc_async_log agc_async_log_fatal PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(agc_stack_walk_tests tests/StackWalk.cpp) target_include_directories(agc_stack_walk_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_stack_walk_tests PRIVATE libc) @@ -1455,6 +1470,8 @@ target_include_directories(agc_driver_provoking_vertex_tests PRIVATE ${agcTestSh target_include_directories(agc_driver_prepared_exports_tests PRIVATE $) +target_include_directories(agc_driver_pipeline_journal_tests PRIVATE $) + target_include_directories(agc_driver_buffer_descriptor_runtime_tests PRIVATE $) target_include_directories(agc_driver_prepared_publication_tests PRIVATE $) @@ -1553,6 +1570,11 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_native_depth_clears agc_driver_native_depth_clear_refusals PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() set_tests_properties(agc_sparse_buffer_holes PROPERTIES TIMEOUT 30) + add_test(NAME agc_pipeline_journal COMMAND agc_driver_graphics_tests --pipeline-journal) + set_tests_properties(agc_pipeline_journal PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_pipeline_journal PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() set_tests_properties(agc_driver_sparse_buffer_holes PROPERTIES TIMEOUT 30) if(WIN32) set_tests_properties(agc_sparse_buffer_holes PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") @@ -1674,6 +1696,23 @@ if(BUILD_TESTING) endif() add_test(NAME agc_driver_pm4_failure COMMAND agc_driver_pm4_tests failure) set_tests_properties(agc_driver_pm4_failure PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30) + set(pm4RingCases "ring|APS5_PM4_RING=4" "ring-empty|APS5_PM4_RING=16" "ring-disabled|APS5_PM4_RING=0" "ring-invalid|APS5_PM4_RING=4x" "ring-report|APS5_PM4_RING=2" "stall-watchdog|APS5_STALL_WATCHDOG_SECS=1" + "stall-watchdog-invalid|APS5_STALL_WATCHDOG_SECS=1x" "stall-watchdog-rewind|APS5_STALL_WATCHDOG_SECS=1" "stall-watchdog-order|APS5_STALL_WATCHDOG_SECS=1") + if(WIN32) + list(APPEND pm4RingCases "ring-report-heap|APS5_PM4_RING=2") + endif() + foreach(pm4RingCase IN LISTS pm4RingCases) + string(REPLACE "|" ";" pm4RingCase "${pm4RingCase}") + list(GET pm4RingCase 0 pm4RingMode) + list(GET pm4RingCase 1 pm4RingEnvironment) + string(REPLACE "-" "_" pm4RingTest "agc_driver_pm4_${pm4RingMode}") + add_test(NAME ${pm4RingTest} COMMAND agc_driver_pm4_tests ${pm4RingMode}) + set_tests_properties(${pm4RingTest} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;${pm4RingEnvironment}" SKIP_RETURN_CODE 77 TIMEOUT 30) + if(WIN32) + set_tests_properties(${pm4RingTest} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() add_test(NAME agc_descriptor_pool_reset_failure COMMAND agc_driver_graphics_tests --descriptor-pool-reset-failure) add_test(NAME agc_driver_address_space_teardown COMMAND agc_driver_graphics_tests --address-space-teardown) set_tests_properties(agc_driver_address_space_teardown PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") @@ -1716,6 +1755,12 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_stencil_replay_no_gpl PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_driver_pipelined_dispatch_item COMMAND agc_driver_pipelined_dispatch_tests) + set_tests_properties(agc_driver_pipelined_dispatch_item PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINE_DISPATCH=1" SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_driver_pipelined_dispatch_item PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() foreach(agcTest IN ITEMS agc_spirv_narrow_constant_store_tests agc_spirv13_image_tests agc_spirv_optimizer_tests agc_driver_constant_store_tests) if(NOT TARGET ${agcTest}) @@ -1747,7 +1792,7 @@ if(BUILD_TESTING) endif() endforeach() endforeach() - foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync) + foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync lazy lazy_sync) set(preparedTest "agc_driver_prepared_shaders_${preparedMode}") if(preparedMode STREQUAL "indirect") add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --indirect) @@ -1755,6 +1800,8 @@ if(BUILD_TESTING) add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred) elseif(preparedMode STREQUAL "deferred_undecodable") add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred-undecodable) + elseif(preparedMode STREQUAL "lazy" OR preparedMode STREQUAL "lazy_sync") + add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --lazy) elseif(preparedMode STREQUAL "registered_workgroup" OR preparedMode STREQUAL "registered_workgroup_sync") add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --registered-workgroup) elseif(preparedMode STREQUAL "debug_failure") @@ -1768,6 +1815,11 @@ if(BUILD_TESTING) set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_NO_DISPATCH_CACHE=1") elseif(preparedMode STREQUAL "registered_workgroup_sync") set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_SYNC_SHADER_PREPARE=1") + elseif(preparedMode STREQUAL "lazy" OR preparedMode STREQUAL "lazy_sync") + set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_LAZY_SHADER_PREPARE=1" "APS5_SUSPEND_DRAIN=1") + if(preparedMode STREQUAL "lazy_sync") + set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_SYNC_SHADER_PREPARE=1") + endif() elseif(preparedMode STREQUAL "debug" OR preparedMode STREQUAL "debug_failure") set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_PROFILE_DRAW=1" "APS5_NO_UNLOCKED_DEVICE=1") endif() @@ -2362,3 +2414,9 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + get_directory_property(agcRegisteredTests TESTS) + list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal|agc_driver_pipeline_journal)$") + set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1") +endif() diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp new file mode 100644 index 0000000000..8af0c00d39 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp @@ -0,0 +1,401 @@ +#pragma once + +#include "prx/libc/include/Shutdown.hpp" +#include "prx/libc/include/general/LogMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver { + +struct LogLiteral { + const char* text; +}; + +namespace AsyncLogDetail { + +template +inline constexpr bool Deferrable = std::is_arithmetic_v || std::is_enum_v || std::is_same_v; + +inline const char* Unwrap(LogLiteral value) { + return value.text; +} + +template +T Unwrap(T value) { + return value; +} + +class Formatter { +public: + virtual ~Formatter() = default; + virtual void Format(std::string& out) const = 0; +}; + +template +class Bound final : public Formatter { +public: + Bound(const char* format, TArgs... args) : format(format), values(args...) {} + + void Format(std::string& out) const override { + std::apply([&](const auto&... value) { Append(out, value...); }, values); + } + +private: + void Append(std::string& out, const auto&... value) const { + char small[512]; + const int size = std::snprintf(small, sizeof(small), format, Unwrap(value)...); + if (size < 0) return; + if (static_cast(size) < sizeof(small)) { + out.append(small, static_cast(size)); + return; + } + std::string large(static_cast(size) + 1, '\0'); + std::snprintf(large.data(), large.size(), format, Unwrap(value)...); + out.append(large.data(), static_cast(size)); + } + + const char* format; + std::tuple values; +}; + +struct Entry { + std::FILE* stream = nullptr; + long long micros = -1; + std::string text; + std::unique_ptr deferred; + std::size_t cost = 0; +}; + +} + +class AsyncLog { +public: + using Clock = std::chrono::steady_clock; + + static constexpr std::size_t Capacity = 16u * 1024u * 1024u; + + static AsyncLog& Get() { + static AsyncLog* const instance = new AsyncLog(); + return *instance; + } + + int Write(std::FILE* stream, const char* text, std::size_t length) { + const auto start = Clock::now(); + if (!asynchronous) return WriteNow(stream, text, length); + AsyncLogDetail::Entry entry; + entry.stream = stream; + entry.text.assign(text, length); + entry.cost = sizeof(entry) + length; + Enqueue(std::move(entry)); + callerNanoseconds.fetch_add(Elapsed(start), std::memory_order_relaxed); + return 0; + } + + template + void Defer(std::FILE* stream, const char* format, TArgs... args) { + static_assert((AsyncLogDetail::Deferrable && ...), "deferred log arguments are numbers, enums or LogLiteral"); + const auto start = Clock::now(); + const auto micros = LogTimestampMicros(); + auto bound = std::make_unique>(format, args...); + if (!asynchronous) { + std::string text; + if (micros >= 0) AppendTimestamp(text, micros); + bound->Format(text); + WriteNow(stream, text.data(), text.size()); + return; + } + AsyncLogDetail::Entry entry; + entry.stream = stream; + entry.micros = micros; + entry.deferred = std::move(bound); + entry.cost = sizeof(entry) + sizeof(AsyncLogDetail::Bound); + Enqueue(std::move(entry)); + callerNanoseconds.fetch_add(Elapsed(start), std::memory_order_relaxed); + } + + bool Admit(const std::string& line) { + const auto now = Clock::now(); + std::string summary; + { + std::lock_guard lock(coalesceMutex); + auto found = windows.find(line); + if (found == windows.end()) { + if (windows.size() >= CoalesceKeys) return true; + windows.emplace(line, Window{now, 1, 0}); + return true; + } + auto& window = found->second; + if (now - window.first < CoalesceWindow) { + if (++window.seen <= PrintedRepeats) return true; + ++window.suppressed; + coalescedTotal.fetch_add(1, std::memory_order_relaxed); + return false; + } + summary = Summary(found->first, window.suppressed); + window = Window{now, 1, 0}; + } + if (!summary.empty()) WriteNow(stderr, summary.data(), summary.size()); + return true; + } + + void Flush() { + if (!asynchronous) return; + std::unique_lock lock(mutex); + const auto target = queued.load(std::memory_order_relaxed); + idle.wait(lock, [&] { return finished.load(std::memory_order_relaxed) >= target; }); + } + + bool FlushFor(std::chrono::milliseconds limit) { + if (!asynchronous) return true; + const auto target = queued.load(std::memory_order_acquire); + const auto deadline = Clock::now() + limit; + while (finished.load(std::memory_order_acquire) < target) { + if (Clock::now() >= deadline) return false; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return true; + } + +private: + struct Window { + Clock::time_point first; + std::uint64_t seen; + std::uint64_t suppressed; + }; + + static constexpr std::size_t CoalesceKeys = 128; + static constexpr std::uint64_t PrintedRepeats = 3; + static constexpr std::chrono::seconds CoalesceWindow{10}; + static constexpr std::chrono::seconds StatsWindow{10}; + static constexpr std::chrono::milliseconds Tick{500}; + static constexpr std::chrono::milliseconds EndFlushLimit{2000}; + + AsyncLog() { + asynchronous = std::getenv("APS5_LOG_SYNC") == nullptr; + const char* stats = std::getenv("APS5_LOG_STATS"); + statsEnabled = stats == nullptr || std::string(stats) != "0"; + lastStats = Clock::now(); + if (!asynchronous) return; + LogSink().store(&AsyncLog::Sink, std::memory_order_release); + std::atexit(&AsyncLog::FlushAtEnd); + std::thread([this] { Run(); }).detach(); + LibcSetEndFlush_nid_no_patch(&AsyncLog::FlushAtEnd); + } + + static void FlushAtEnd() { + auto& log = Get(); + if (log.abandoned.load(std::memory_order_acquire)) return; + if (!log.FlushFor(EndFlushLimit)) log.abandoned.store(true, std::memory_order_release); + } + + static int Sink(std::FILE* stream, const char* text, std::size_t length) { + return Get().Write(stream, text, length); + } + + static std::uint64_t Elapsed(Clock::time_point start) { + return static_cast(std::chrono::duration_cast(Clock::now() - start).count()); + } + + static void AppendTimestamp(std::string& out, long long micros) { + char prefix[64]; + const int size = FormatLogTimestamp(prefix, sizeof(prefix), micros); + if (size > 0) out.append(prefix, static_cast(size)); + } + + static std::string Summary(const std::string& line, std::uint64_t suppressed) { + if (suppressed == 0) return {}; + std::string text = line; + while (!text.empty() && (text.back() == '\n' || text.back() == '\r')) text.pop_back(); + text += " (repeated " + std::to_string(suppressed) + " more times in the last 10 s)\n"; + return text; + } + + static int WriteNow(std::FILE* stream, const char* text, std::size_t length) { + std::fwrite(text, 1, length, stream); + return std::fflush(stream); + } + + void Enqueue(AsyncLogDetail::Entry&& entry) { + bool notify = false; + { + std::unique_lock lock(mutex); + space.wait(lock, [&] { return bytes == 0 || bytes + entry.cost <= Capacity; }); + bytes += entry.cost; + peak = std::max(peak, bytes); + pending.push_back(std::move(entry)); + queued.fetch_add(1, std::memory_order_release); + notify = sleeping; + sleeping = false; + } + if (notify) wake.notify_one(); + } + + void Run() noexcept { + std::deque batch; + std::string buffer; + for (;;) { + { + std::unique_lock lock(mutex); + sleeping = true; + wake.wait_for(lock, Tick, [&] { return !pending.empty(); }); + sleeping = false; + batch.swap(pending); + bytes = 0; + } + space.notify_all(); + if (!batch.empty()) { + const auto start = Clock::now(); + const auto count = batch.size(); + try { + Drain(batch, buffer); + } catch (...) { + buffer.clear(); + } + batch.clear(); + loggerNanoseconds.fetch_add(Elapsed(start), std::memory_order_relaxed); + { + std::lock_guard lock(mutex); + finished.fetch_add(count, std::memory_order_release); + } + idle.notify_all(); + } + try { + Housekeeping(); + } catch (...) { + } + } + } + + void Drain(std::deque& batch, std::string& buffer) { + std::FILE* current = nullptr; + buffer.clear(); + std::uint64_t outLines = 0; + std::uint64_t errLines = 0; + std::uint64_t written = 0; + const auto emit = [&] { + if (current != nullptr && !buffer.empty()) { + written += buffer.size(); + WriteNow(current, buffer.data(), buffer.size()); + } + buffer.clear(); + }; + for (auto& entry : batch) { + if (entry.stream != current) { + emit(); + current = entry.stream; + } + if (entry.deferred) { + if (entry.micros >= 0) AppendTimestamp(buffer, entry.micros); + entry.deferred->Format(buffer); + } else { + buffer += entry.text; + } + (entry.stream == stderr ? errLines : outLines) += 1; + if (buffer.size() >= 65536) emit(); + } + emit(); + stdoutLines.fetch_add(outLines, std::memory_order_relaxed); + stderrLines.fetch_add(errLines, std::memory_order_relaxed); + loggedBytes.fetch_add(written, std::memory_order_relaxed); + } + + void Housekeeping() { + const auto now = Clock::now(); + std::vector summaries; + { + std::lock_guard lock(coalesceMutex); + for (auto it = windows.begin(); it != windows.end();) { + if (now - it->second.first >= CoalesceWindow) { + const auto summary = Summary(it->first, it->second.suppressed); + if (!summary.empty()) summaries.push_back(summary); + it = windows.erase(it); + } else { + ++it; + } + } + } + for (const auto& summary : summaries) WriteNow(stderr, summary.data(), summary.size()); + if (now - lastStats < StatsWindow) return; + const double seconds = std::chrono::duration(now - lastStats).count(); + lastStats = now; + const auto outLines = stdoutLines.exchange(0, std::memory_order_relaxed); + const auto errLines = stderrLines.exchange(0, std::memory_order_relaxed); + const auto written = loggedBytes.exchange(0, std::memory_order_relaxed); + const auto caller = callerNanoseconds.exchange(0, std::memory_order_relaxed); + const auto logger = loggerNanoseconds.exchange(0, std::memory_order_relaxed); + const auto coalesced = coalescedTotal.exchange(0, std::memory_order_relaxed); + std::size_t high = 0; + { + std::lock_guard lock(mutex); + high = peak; + peak = bytes; + } + if (!statsEnabled || outLines + errLines + coalesced == 0) return; + char line[512]; + const int size = std::snprintf(line, sizeof(line), "[log] %.1f s: %llu lines (%llu stdout, %llu stderr), %.1f KiB written, %llu repeats coalesced; callers spent %.2f ms queueing and formatting (%.3f%% of one thread), logger thread %.2f ms, queue peak %.1f KiB\n", seconds, static_cast(outLines + errLines), static_cast(outLines), static_cast(errLines), static_cast(written) / 1024.0, static_cast(coalesced), static_cast(caller) / 1e6, static_cast(caller) / 1e9 / seconds * 100.0, static_cast(logger) / 1e6, static_cast(high) / 1024.0); + if (size > 0) WriteNow(stderr, line, std::min(static_cast(size), sizeof(line) - 1)); + } + + bool asynchronous = true; + bool statsEnabled = true; + std::mutex mutex; + std::condition_variable wake; + std::condition_variable space; + std::condition_variable idle; + std::deque pending; + std::size_t bytes = 0; + std::size_t peak = 0; + bool sleeping = false; + std::atomic queued{0}; + std::atomic finished{0}; + std::atomic abandoned{false}; + std::mutex coalesceMutex; + std::map windows; + Clock::time_point lastStats; + std::atomic stdoutLines{0}; + std::atomic stderrLines{0}; + std::atomic loggedBytes{0}; + std::atomic callerNanoseconds{0}; + std::atomic loggerNanoseconds{0}; + std::atomic coalescedTotal{0}; +}; + +inline void InstallAsyncLog_nid_no_patch() { + static_cast(AsyncLog::Get()); +} + +inline void FlushAsyncLog_nid_no_patch() { + AsyncLog::Get().Flush(); +} + +template +void LogCoalescedStderr_nid_no_patch(const char* format, TArgs... args) { + char small[1024]; + const int size = std::snprintf(small, sizeof(small), format, args...); + if (size < 0) return; + std::string line = static_cast(size) < sizeof(small) ? std::string(small, static_cast(size)) : std::string(static_cast(size), '\0'); + if (static_cast(size) >= sizeof(small)) std::snprintf(line.data(), line.size() + 1, format, args...); + if (!AsyncLog::Get().Admit(line)) return; + std::fwrite(line.data(), 1, line.size(), stderr); + std::fflush(stderr); +} + +} + +#define AGC_LOG_DEFERRED(stream, fmt, ...) \ +::AgcDriver::AsyncLog::Get().Defer(stream, "[%s:%d %s] " fmt "\n", ::AgcDriver::LogLiteral{_APS5_FILE_}, __LINE__, ::AgcDriver::LogLiteral{TrimNidPostfix(__func__)}, __VA_ARGS__) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp index 41d3bce67a..b96a4cdf0a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp @@ -41,6 +41,9 @@ struct DispatchVariant { std::uint32_t flatBinding = NoFlatBinding; std::uint32_t pushOffset = 0; + std::vector runBases; + std::uint64_t relocatableWords = 0; + std::vector> shaderDataPatches; std::shared_ptr vertexInfo; std::shared_ptr memory; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp index ee5ce7844f..23b6231072 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAWCACHE_HPP #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAWCACHE_HPP +#include "UserDataUse.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include @@ -30,6 +31,7 @@ struct DrawProgram { bool merged = false; bool mergedPointerRequired = false; std::uint32_t mergedPointer = 0; + std::uint32_t hiddenUserWords = 0; }; struct DrawDecode { @@ -118,6 +120,36 @@ struct StageCapture { std::vector regions; std::uint64_t forgetSerial = 0; std::uint32_t pushOffset = 0; + ShaderRecompiler::SrtUserDataUse userDataUse; +}; + +struct RelocatableUserWord { + std::uint32_t reg; + std::vector> reads; +}; + +struct DrawRelocation { + std::vector words; + std::vector banned; +}; + +struct DrawUserPatch { + std::size_t program; + std::uint32_t index, value; +}; + +struct DrawRelocationPending { + std::uint64_t key = 0, alias = 0, identity = 0; +}; + +struct DrawRelocationTrial { + static constexpr std::uint32_t Window = 1024, MinimumHitShare = 16, IdleDraws = 16384, ProbationAdmissions = 2; + std::uint32_t lookups = 0, hits = 0, idle = 0, admissions = 0; + bool productive = false; +}; + +struct DrawRelocationCounters { + std::uint64_t lookups = 0, donors = 0, refused = 0, hits = 0, evicted = 0; }; bool DecodeProgramsCurrent(const DrawDecode& decode, const ShaderRegistry& registry); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp index d89415137c..dc8e155378 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp @@ -26,12 +26,14 @@ public: static DrawPipeline& Queue0(); static std::size_t Depth(); + static std::size_t QueuedItems(); static bool& Active(); static std::atomic& EpochToken(); static void FollowEpoch(std::uint64_t token); void Enqueue(Commit commit, std::vector writes, std::uint64_t labelAddress = 0, std::vector labelBytes = {}); void Drain(DrainReason reason, std::uint32_t opcode = 0x100); + static std::uint64_t DrainRequestsByOpcode(std::uint32_t opcode); bool Busy() const { return outstanding.load(std::memory_order_acquire) != 0; } std::size_t Outstanding() const { return outstanding.load(std::memory_order_acquire); } bool Overlaps(std::uint64_t address, std::size_t bytes); @@ -67,6 +69,7 @@ private: std::array(DrainReason::Count)> drains{}; std::array(DrainReason::Count)> drainWaitNs{}; std::array drainOpcodes{}; + std::array, 257> drainRequests{}; std::uint64_t depthSum = 0; std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 434f0d992b..4bceb20d24 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -75,6 +75,7 @@ public: }; DispatchCacheFootprint DispatchCacheSize(); static std::uint64_t DispatchCacheBytes(); + std::shared_ptr RegisteredShaders(); private: friend class SampledReadScope; @@ -111,6 +112,7 @@ private: static bool stampValidate(); static bool dataHits(); static bool verifyDataHits(); + static bool pipelineDispatch(); static std::size_t dispatchVariants(); static bool insertCompare(); static std::size_t dispatchCacheEntries(); @@ -142,8 +144,16 @@ private: void maybeReportDrawCache(bool profile); void reportDrawCache(DrawEntryCounters& counters, bool profile); std::string profileDrawCacheLines(const DrawEntryCounters& counters); - static std::uint64_t drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial, std::uint64_t* shape = nullptr, std::array* components = nullptr); + static std::uint64_t drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial, std::uint64_t* shape = nullptr, std::array* components = nullptr, const DrawRelocation* relocation = nullptr, std::uint64_t* alias = nullptr); void traceDrawKeyLookup(const std::array& components, bool cacheMiss); + static std::uint64_t drawIdentityKey(const QueueState& queue, const ShaderRegistry& registry); + static bool drawRelocationEnabled(); + static std::uint64_t relocatedContentKey(const RelocatableUserWord& word, std::uint32_t value, bool& readable); + static void tagVariantRuns(DispatchVariant& variant, std::span regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords); + static void applyShaderDataPatches(DispatchVariant& variant); + void learnPendingRelocation(); + void relocateDrawEntry(const QueueState& queue, std::uint64_t alias, std::shared_ptr& entry, std::shared_ptr& decode, std::vector& patches); + void reportDrawRelocation(); static bool sameVertexInfo(const ShaderRecompiler::ShaderVertexStageInfo& a, const ShaderRecompiler::ShaderVertexStageInfo& b); static bool sameDecode(const DrawDecode& a, const DrawDecode& b); std::shared_ptr decodeDraw(const QueueState& queue, const Submission& submission); @@ -348,6 +358,11 @@ private: std::unordered_map drawShapes; std::unordered_set traceShapes, traceKeysSeen, traceKeysEvicted; std::list drawOrder; + std::unordered_map> drawRelocations; + std::unordered_map> drawAliases; + DrawRelocationPending drawRelocationPending; + std::unordered_map drawRelocationTrials; + DrawRelocationCounters drawRelocationCounters; std::mutex drawCacheMutex; std::uint64_t drawCacheHits = 0, drawCacheEvictions = 0, drawCacheVariants = 0, drawCacheVariantBytes = 0; std::optional drawKeySightings; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp index b5e1b03310..85cc30100c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp @@ -11,6 +11,7 @@ namespace AgcDriver::DriverDetail { std::vector> DrawWriteRanges(const Graphics::State& graphics, std::span stages, bool exact); +std::vector> DispatchWriteRanges(const ShaderRecompiler::RecompileResult& compiled, bool& bounded); bool ExactDrawWrites(); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp new file mode 100644 index 0000000000..a29def6cff --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp @@ -0,0 +1,40 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_QUEUES_SUBMISSIONRING_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_QUEUES_SUBMISSIONRING_HPP + +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +inline constexpr std::uint32_t SubmissionQueueEnd = 0x58; + +std::size_t SubmissionRingCapacity(); + +std::chrono::seconds StallWatchdogPeriod(); + +bool ReadSubmissionRingSettings(); + +void PrepareSubmissionRing(std::uint32_t queue); + +void RecordSubmission(std::uint32_t queue, std::uint64_t serial, std::span commands, const void* guest, bool rewindTail); + +void DumpSubmissionRings(std::string_view reason); + +class StallWatch { +public: + explicit StallWatch(std::uint32_t queue) : queue(queue) {} + void Progressed() { since = {}; } + void Check(const char* waiter, std::uint64_t value); + +private: + std::uint32_t queue; + std::chrono::steady_clock::time_point since{}; + bool dumped = false; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp index 58fde0469b..f902632e1a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp @@ -78,6 +78,7 @@ std::uint32_t ReadDrawCount(const DrawParameters::IndirectDraw& indirect); inline bool IndirectDrawOpcode(std::uint32_t opcode) { return opcode == 0x24 || opcode == 0x25 || opcode == 0x2c || opcode == 0x38; } inline bool DrawOpcode(std::uint32_t opcode) { return opcode == 0x27 || opcode == 0x2d || opcode == 0x35 || IndirectDrawOpcode(opcode); } +std::string_view KnownName(std::uint32_t header); std::string Name(std::uint32_t header); // A PM4 type-2 packet is a one-dword filler (command-buffer padding); type 3 and type 0 carry a // dword count in bits 29:16. Type 1 is undefined. diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 87800734a3..4aa2fa07f2 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -30,6 +30,8 @@ struct ComputePipelineObjects; struct RecipeHit; // The record tail shared by an ordinary dispatch and a recipe hit (VulkanDevice::recordDispatch). struct RecordedDispatch; +// The module, layout and pipeline a compute dispatch binds (VulkanDevice::createComputePipeline). +struct ComputePipelineObjects; class VulkanDevice { public: @@ -37,6 +39,14 @@ public: struct CompileOnly {}; explicit VulkanDevice(CompileOnly); ComputePipelineReplayResult ReplayComputePipeline(const ComputePipelineCapture& capture, std::span initialCache = {}); + // Builds the pipeline of a journaled compute result ahead of its first dispatch (the journal + // replay's hook, on its threads): false when the device already holds a pipeline for the module. + bool ReplayComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants); + // Starts the journal replay if the device has not (a headless device starts it at its first + // draw or dispatch) and joins it (tests; the teardown joins it anyway). + void WaitForPipelineJournalReplay(); + // Compute pipelines this device built, by dispatches and by the journal replay (tests). + std::uint64_t ComputePipelinesCreated() const; ~VulkanDevice(); VulkanDevice(const VulkanDevice&) = delete; VulkanDevice& operator=(const VulkanDevice&) = delete; @@ -48,6 +58,7 @@ public: const std::uint32_t* WorkGroupCountLimit() const; std::uint32_t VendorId() const; ShaderRecompiler::SpirvTarget ComputeTarget(std::uint32_t waveSize) const; + bool ComputeFullSubgroups() const; // Distinguishes this device from every earlier one in the process (a memo keyed by the device // cannot rely on the pointer, which a replacement may reuse). std::uint64_t Serial() const { return serial; } @@ -58,6 +69,7 @@ public: // three-step form below so the GPU wait happens without the mutex. void WaitIdle(); void PrepareForReplacement(); + void startJournalReplay(); bool ImportGuestMemory(const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice); // Sends recorded work to the GPU without waiting for it. With `reapFirst` it first retires batches // that already finished, so the in-flight list stays short (APS5_NO_OPPORTUNISTIC_REAP=1 skips @@ -316,6 +328,10 @@ private: // The device's Graphics::Context: a copy of the one built at setup (its instance functions // resolved then, its function table filled then), or with APS5_NO_CONTEXT_CACHE=1 built anew. Graphics::Context graphicsContext() const; + VkPipelineCreateFlags computePipelineFlags() const; + std::uint32_t computeSubgroupSize(const ShaderRecompiler::RecompileResult& shader) const; + VkPipelineShaderStageCreateFlags computeStageFlags(const ShaderRecompiler::RecompileResult& shader) const; + std::shared_ptr createComputePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress); ShaderRecompiler::SpirvTarget buildTarget() const; Graphics::Context buildContext() const; // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups. diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ComputePipelineCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ComputePipelineCapture.cpp index 6137a82dcc..4bdf335e98 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ComputePipelineCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ComputePipelineCapture.cpp @@ -94,7 +94,7 @@ void ComputePipelineCapture::Validate() const { RequireCapture(mainEntry, "compute main entry point missing"); RequireCapture(deviceName.size() < VK_MAX_PHYSICAL_DEVICE_NAME_SIZE && deviceName.find('\0') == std::string::npos, "invalid device name"); RequireCapture((flags & ~(VK_PIPELINE_CREATE_DISPATCH_BASE_BIT | VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR)) == 0, "unsupported pipeline flags"); - RequireCapture(stageFlags == 0, "unsupported shader stage flags"); + RequireCapture(stageFlags == 0 || (stageFlags == VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT && requiredSubgroupSize != 0), "unsupported shader stage flags"); RequireCapture(requiredSubgroupSize == 0 || (requiredSubgroupSize <= 128 && (requiredSubgroupSize & (requiredSubgroupSize - 1u)) == 0), "invalid required subgroup size"); RequireCapture(sets.size() <= 8, "too many descriptor sets"); std::size_t totalBindings = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 045daf3d24..b506cff5c1 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -2,9 +2,11 @@ #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" @@ -15,6 +17,47 @@ namespace AgcDriver::DriverDetail { +namespace { + +bool DispatchItemsEnabled() { + static const bool enabled = [] { + const char* text = std::getenv("APS5_PIPELINE_DISPATCH"); + return text != nullptr && text[0] != '\0' && text[0] != '0'; + }(); + return enabled; +} + +enum class ItemFallback : std::size_t { Miss, Reads, NoRecipe, Writes, Verify, Count }; +constexpr std::array(ItemFallback::Count)> ItemFallbackNames{"miss", "reads", "no recipe", "writes", "verify"}; +std::array, static_cast(ItemFallback::Count)> itemFallbacks{}; +std::atomic itemCommits{0}; +std::atomic itemRebuilds{0}; + +void NoteItemFallback(ItemFallback reason) { + itemFallbacks[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); +} + +void ReportDispatchItems(bool profile) { + static auto lastReport = std::chrono::steady_clock::now(); + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::string drained; + for (std::size_t i = 0; i < ItemFallbackNames.size(); ++i) { + char item[48]; + std::snprintf(item, sizeof(item), " %s %llu", ItemFallbackNames[i], static_cast(itemFallbacks[i].load(std::memory_order_relaxed))); + drained += item; + } + std::fprintf(stderr, "[dispatch-item] %llu commit items (%llu recipes rebuilt on the committer); drained instead, cumulative:%s\n", static_cast(itemCommits.load(std::memory_order_relaxed)), static_cast(itemRebuilds.load(std::memory_order_relaxed)), drained.c_str()); +} + +} + +bool Driver::pipelineDispatch() { + return DispatchItemsEnabled(); +} + void Driver::dispatch(QueueState& queue, std::span packet, const Submission& submission, std::uint64_t indirectArguments) { PerformanceTimer timing("Driver.Dispatch"); const auto address = (static_cast(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast(readRegister(queue.shader, 0x20d) & 0xffu) << 40u); @@ -72,6 +115,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, if (indirectArguments == 0 && (packet[4] & 0x2u) != 0) { partialGroups = Pm4::PartialGroupThreads({packet[1], packet[2], packet[3]}, {readRegister(queue.shader, 0x207), readRegister(queue.shader, 0x208), readRegister(queue.shader, 0x209)}); } + const bool pipelineCandidate = !resolving && !fromAhead && DispatchItemsEnabled() && DrawPipeline::Active() && indirectArguments == 0 && deferredLabels().labels.empty() && !matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) && !MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute); const bool partialLastGroup = partialGroups != std::array{}; if (resolving) { if (!partialLastGroup && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return; @@ -84,7 +128,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, pendingDispatchPhases().outcome = DispatchOutcome::FillHle; return; } - if (DrawPipeline::Active()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); + if (DrawPipeline::Active() && !pipelineCandidate) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) { const auto threads = (packet[4] & 0x20u) != 0 ? static_cast(packet[1]) : static_cast(packet[1]) * compute.numThreads[0]; const auto nativeBytes = Graphics::HtileDepthClearBytes_nid_no_patch(std::span(snapshot.code).subspan(codeOffset), userData, compute, packet, indirectArguments); @@ -223,6 +267,27 @@ void Driver::dispatch(QueueState& queue, std::span packet, } phaseTiming.Phase(PhaseKey); timing.Mark("decode_key_device_setup"); + bool readsClear = false; + if (pipelineCandidate) { + std::vector reads; + bool entryFound = false; + { + std::lock_guard cacheLock(dispatchCacheMutex); + if (const auto found = dispatchCache.find(key); found != dispatchCache.end()) { + entryFound = true; + for (const auto& variant : found->second->variants) appendEntryRegions(*variant, reads); + } + } + if (!entryFound || noDispatchCache || stampValidate()) { + NoteItemFallback(ItemFallback::Miss); + } else if (verifyDataHits()) { + NoteItemFallback(ItemFallback::Verify); + } else { + readsClear = std::none_of(reads.begin(), reads.end(), [](const auto& region) { return DrawPipeline::Queue0().Overlaps(region.guestAddress, region.bytes.size()); }); + if (!readsClear) NoteItemFallback(ItemFallback::Reads); + } + if (!readsClear && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); + } if (fromAhead) { compiledResult = std::move(ahead.compiledResult); keepVariant = std::move(ahead.keepVariant); @@ -244,6 +309,19 @@ void Driver::dispatch(QueueState& queue, std::span packet, lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering); } timing.Mark("lookup_cache"); + std::vector itemWrites; + bool pipelined = false; + if (readsClear) { + if (cached && keepVariant != nullptr && keepVariant->recipe.load(std::memory_order_acquire) != nullptr) { + bool bounded = false; + itemWrites = DispatchWriteRanges(*compiledResult, bounded); + pipelined = bounded; + if (!bounded) NoteItemFallback(ItemFallback::Writes); + } else { + NoteItemFallback(ItemFallback::NoRecipe); + } + if (!pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); + } if (cached) { require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader"); captureMs += phaseTiming.Elapsed(); @@ -361,6 +439,64 @@ void Driver::dispatch(QueueState& queue, std::span packet, }; phaseTiming.Phase(PhaseSnapshots); + if (pipelined) { + const auto recipe = keepVariant->recipe.load(std::memory_order_acquire); + const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed); + const auto queueId = submission.queue; + itemCommits.fetch_add(1, std::memory_order_relaxed); + DrawPipeline::Queue0().Enqueue([this, localDevice = std::move(localDevice), result = compiledResult, snapshots = std::move(snapshots), groups, address, queueId, recipe, keepVariant, dataHit, epoch] { + DrawPipeline::FollowEpoch(epoch); + GuestMemory::SetCurrentPacket(0x15, queueId); + const auto& compiledCommit = *result; + const bool noteWrites = writeEvidenceEnabled() || traceCapSync(); + try { + std::shared_ptr recipeHit = localDevice->PrepareRecipe(recipe, false); + std::shared_ptr builtRecipe; + bool writersNoted = false; + for (;;) { + std::shared_ptr prepared; + if (recipeHit == nullptr || VulkanDevice::VerifyRecipes()) prepared = localDevice->PrepareDispatch(compiledCommit, snapshots); + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Dispatch); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + recordLabelsForPacket(localDevice.get(), queueId); + if (noteWrites && writerKeyedEvidence() && !writersNoted) { + noteWrittenBuffers(address, queueId, compiledCommit); + writersNoted = true; + } + if (recipeHit != nullptr) { + VulkanDevice::IndirectOutcome outcome{0, 0}; + const auto recipeResult = localDevice->DispatchRecipe(compiledCommit, groups[0], groups[1], groups[2], 0, address, recipeHit, outcome, VulkanDevice::VerifyRecipes() ? prepared : nullptr, dataHit); + if (recipeResult == RecipeOutcome::Rebuild) { + recipeHit = nullptr; + VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Restart, false); + itemRebuilds.fetch_add(1, std::memory_order_relaxed); + continue; + } + } else { + localDevice->Dispatch(compiledCommit, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), &builtRecipe); + } + break; + } + if (noteWrites && !writerKeyedEvidence()) noteWrittenBuffers(address, queueId, compiledCommit); + if (builtRecipe != nullptr) { + if (dataHit) { + auto own = std::make_shared(*builtRecipe); + own->dataWordsHash = Graphics::ShaderResources::DataWordsHash({ShaderRecompiler::ShaderStage::Compute, keepVariant->compiled.get(), 0}); + builtRecipe = std::move(own); + } + keepVariant->recipe.store(std::move(builtRecipe), std::memory_order_release); + VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Attach, false); + } + } catch (const std::exception& error) { + char where[64]; + std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast(address)); + throw std::runtime_error(where + std::string(error.what())); + } + }, std::move(itemWrites)); + ReportDispatchItems(profile); + return; + } + std::shared_ptr recipeHit; if (cached && keepVariant != nullptr && !stampValidate()) { recipeHit = localDevice->PrepareRecipe(keepVariant->recipe.load(std::memory_order_acquire), indirectArguments != 0); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index a6421e61c9..5dea7922ea 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -77,10 +77,26 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe std::shared_ptr decode; std::shared_ptr dataEntry; std::uint64_t shapeKey = 0; + std::uint64_t drawIdentity = 0, drawAlias = 0; + std::vector userPatches; if (registerKey) { const auto keyStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; - drawKey = drawRegisterKey(queue, *submission.shaders, localDevice->Serial(), &shapeKey, traceDrawCache() ? &drawKeyComponents : nullptr); - std::lock_guard cacheLock(drawCacheMutex); + std::shared_ptr relocation; + bool aliasFirst = false; + if (drawRelocationEnabled()) { + drawIdentity = drawIdentityKey(queue, *submission.shaders); + std::lock_guard cacheLock(drawCacheMutex); + learnPendingRelocation(); + const auto known = drawRelocations.find(drawIdentity); + if (known != drawRelocations.end()) { + auto& trial = drawRelocationTrials[drawIdentity]; + if (trial.idle != 0) --trial.idle; + else relocation = known->second; + aliasFirst = relocation != nullptr && trial.productive; + } + } + drawKey = drawRegisterKey(queue, *submission.shaders, localDevice->Serial(), &shapeKey, traceDrawCache() ? &drawKeyComponents : nullptr, aliasFirst ? relocation.get() : nullptr, aliasFirst ? &drawAlias : nullptr); + std::unique_lock cacheLock(drawCacheMutex); ++drawEntryCounters.lookups; ++drawEntryCounters.registerKeyLookups; if (profile) drawEntryCounters.keyUs += std::chrono::duration(std::chrono::steady_clock::now() - keyStart).count(); @@ -90,6 +106,22 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe decode = entry->decode; } else { ++drawEntryCounters.absent; + if (relocation != nullptr && !aliasFirst) { + cacheLock.unlock(); + drawRegisterKey(queue, *submission.shaders, localDevice->Serial(), nullptr, nullptr, relocation.get(), &drawAlias); + cacheLock.lock(); + } + if (drawIdentity != 0) drawRelocationPending = {drawKey, drawAlias, drawIdentity}; + if (drawAlias != 0) relocateDrawEntry(queue, drawAlias, entry, decode, userPatches); + if (relocation != nullptr) { + auto& trial = drawRelocationTrials[drawIdentity]; + ++trial.lookups; + if (trial.lookups >= DrawRelocationTrial::Window) { + trial.productive = trial.hits * DrawRelocationTrial::MinimumHitShare >= trial.lookups; + if (!trial.productive) trial.idle = DrawRelocationTrial::IdleDraws; + trial.lookups = trial.hits = 0; + } + } if (traceDrawCache()) { auto& counters = drawEntryCounters; if (traceKeysEvicted.contains(drawKey)) ++counters.absentEvicted; @@ -97,7 +129,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe else if (traceShapes.contains(shapeKey)) ++counters.absentUserWords; else ++counters.absentNewRegisters; } - if (drawDataHits() && !drawParameters.indirect) { + if (decode == nullptr && drawDataHits() && !drawParameters.indirect) { if (const auto shape = drawShapes.find(shapeKey); shape != drawShapes.end()) { const auto kept = drawCache.find(shape->second); if (kept != drawCache.end() && kept->second->decode != nullptr && kept->second->shape == shapeKey) dataEntry = kept->second; @@ -113,6 +145,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe maybeReportDrawCache(profile); } if (decode != nullptr && !DecodeProgramsCurrent(*decode, *submission.shaders)) { + userPatches.clear(); std::lock_guard cacheLock(drawCacheMutex); eraseDrawEntry(drawKey, entry); ++drawEntryCounters.absent; @@ -144,6 +177,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe const auto& graphics = decode->state; auto pixel = decode->pixel; std::vector programs = decode->programs; + for (const auto& patch : userPatches) programs[patch.program].userData[patch.index] = patch.value; if (const auto skipped = SkippedShader(); skipped != 0 && std::any_of(programs.begin(), programs.end(), [skipped](const DrawProgram& program) { return program.binary.codeAddress == skipped; })) return drawn(); const auto setMeshIndexBuffer = [&](const Pm4::DrawParameters& parameters) { if (!graphics.stages.mesh) return; @@ -238,6 +272,23 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe bool verifyHit = false; const bool stageEntries = useDrawEntries && !(graphics.stages.tessellation && !drawParameters.indirect && drawParameters.instanceCount > 1u); lookupDraw(submission, localDevice, graphics, pixel, programs, roles, vertexInfos, stageEntries, registerKey, profile, drawKey, entry, matched, matchedRegions, drawHit, verifyHit, phaseTiming, phaseMs, dataEntry, dataCandidates, stageData); + for (auto& variant : matched) { + if (variant != nullptr) applyShaderDataPatches(*variant); + } + const auto relocatedDecode = decode; + if (!userPatches.empty()) { + std::lock_guard cacheLock(drawCacheMutex); + if (drawHit || verifyHit) { + ++drawRelocationCounters.hits; + ++drawRelocationTrials[drawIdentity].hits; + } + if (profile && drawRelocationCounters.lookups >= 100000) reportDrawRelocation(); + } + if (!userPatches.empty() && !drawHit) { + auto current = std::make_shared(*decode); + for (const auto& patch : userPatches) current->programs[patch.program].userData[patch.index] = patch.value; + decode = std::move(current); + } const bool patchedStages = std::any_of(stageData.begin(), stageData.end(), [](const StageDataHit& hit) { return hit.compiled != nullptr; }); std::uint64_t stageDataVerified = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp index 2120be06c4..948bc83d91 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp @@ -102,7 +102,22 @@ bool Driver::admitDrawKey(std::uint64_t key, std::uint64_t shape) { std::lock_guard cacheLock(drawCacheMutex); if (drawCache.contains(key)) return true; if (!drawKeySightings) drawKeySightings.emplace(drawCacheEntries() * DrawKeySightings::Ways); - const bool seen = drawKeySightings->SeenBefore(key); + bool seen = drawKeySightings->SeenBefore(key); + const auto& pending = drawRelocationPending; + if (pending.key == key && pending.identity != 0) { + if (pending.alias != 0) { + const auto trial = drawRelocationTrials.find(pending.identity); + if (drawKeySightings->SeenBefore(pending.alias ^ 0x9e3779b97f4a7c15ull) && trial != drawRelocationTrials.end()) { + auto& state = trial->second; + if (state.productive || state.hits != 0 || state.admissions < DrawRelocationTrial::ProbationAdmissions) { + if (!state.productive && state.hits == 0) ++state.admissions; + seen = true; + } + } + } else if (!drawRelocations.contains(pending.identity)) { + if (drawKeySightings->SeenBefore(pending.identity ^ 0xc2b2ae3d27d4eb4full)) seen = true; + } + } if (seen || shape == 0 || drawShapes.contains(shape)) return seen; if (!drawShapeSightings) drawShapeSightings.emplace(drawCacheEntries() * DrawKeySightings::Ways); return drawShapeSightings->SeenBefore(shape); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp index 2cab9c30e7..616d63c690 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp @@ -5,6 +5,7 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include "Optimization/ResourceProgram.hpp" +#include #include #include #include @@ -55,6 +56,7 @@ std::shared_ptr Driver::materializeDraw }(); stageCapture.regions = shaderMemory.TakeRecentRegions(); + stageCapture.userDataUse = capture->userDataUse; recompiled[i] = true; memory = shaderMemory.Regions(); @@ -159,13 +161,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP std::vector regions(stageCapture.regions.begin(), stageCapture.regions.end()); for (const auto& read : decodeReads[i]) regions.push_back({read.address, std::as_bytes(std::span(read.bytes))}); std::stable_sort(regions.begin(), regions.end(), [](const ShaderRecompiler::MemoryRegion& a, const ShaderRecompiler::MemoryRegion& b) { return a.guestAddress < b.guestAddress; }); - for (const auto& region : regions) { - variant->runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size()); - const auto count = region.bytes.size() / sizeof(std::uint32_t); - const auto offset = variant->words.size(); - variant->words.resize(offset + count); - std::memcpy(variant->words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t)); - } + tagVariantRuns(*variant, regions, stageCapture.userDataUse, decodeReads[i].empty() && (!vertexInfos[i] || vertexInfos[i]->resourcesNum == 0), programs[i].hiddenUserWords); if (drawStageDataHits() && stageCapture.capture != nullptr && !stageCapture.capture->readTrace.leaves.empty()) recordStageDataPositions(*variant, *stageCapture.capture, decodeReads[i]); if (traceDrawCache() && entry != nullptr && matched[i] == nullptr && i < entry->stages.size()) { const auto& variants = entry->stages[i]; @@ -177,7 +173,26 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP ++differingRunsChanged; } } - if (verifyHit && matched[i] != nullptr && (matched[i]->runs != variant->runs || matched[i]->words != variant->words)) { + const auto sameContent = [](const DispatchVariant& a, const DispatchVariant& b) { + const auto flatten = [](const DispatchVariant& source) { + std::vector> flat; + std::size_t word = 0; + for (const auto& [begin, end] : source.runs) { + for (auto address = begin; address < end && word < source.words.size(); address += sizeof(std::uint32_t)) flat.emplace_back(address, source.words[word++]); + } + std::sort(flat.begin(), flat.end()); + return flat; + }; + return (a.runs == b.runs && a.words == b.words) || flatten(a) == flatten(b); + }; + const auto sameForwardedWords = [](const ShaderRecompiler::RecompileResult& a, const ShaderRecompiler::RecompileResult& b, std::uint32_t hidden) { + if (a.pushConstants.size() != b.pushConstants.size() || a.shaderDataUserWords != b.shaderDataUserWords) return false; + for (std::size_t slot = 0; slot < a.shaderDataUserWords.size() && (slot + 1) * sizeof(std::uint32_t) <= a.pushConstants.size(); ++slot) { + if (a.shaderDataUserWords[slot] >= hidden && std::memcmp(a.pushConstants.data() + slot * sizeof(std::uint32_t), b.pushConstants.data() + slot * sizeof(std::uint32_t), sizeof(std::uint32_t)) != 0) return false; + } + return true; + }; + if (verifyHit && matched[i] != nullptr && (!sameContent(*matched[i], *variant) || !sameForwardedWords(*matched[i]->compiled, *variant->compiled, programs[i].hiddenUserWords))) { ++mismatches; static std::atomic reports{0}; if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: stage %zu (program 0x%llx) of a hit captured differently: %zu runs / %zu words matched, %zu / %zu fresh\n", i, static_cast(programs[i].binary.codeAddress), matched[i]->runs.size(), matched[i]->words.size(), variant->runs.size(), variant->words.size()); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp index 9079eb1c16..598176196b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp @@ -35,6 +35,7 @@ void InitializeMerged(const QueueState& queue, DrawProgram& program, std::uint32 program.mergedPointerRequired = pointerRequired; program.firstUserSgpr = 0; program.userData.insert(program.userData.begin(), 8, 0); + program.hiddenUserWords = 8; Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, pointerBase); Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, pointerBase + 1); if (staticAbi) return; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp index 366793d496..c64a860a7a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp @@ -22,7 +22,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr(opcode, 0x100)].fetch_add(1, std::memory_order_relaxed); if (!Busy()) return; if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline drain under the GPU lock (reason " + std::to_string(static_cast(reason)) + ", opcode " + std::to_string(opcode) + ")"); const auto start = std::chrono::steady_clock::now(); @@ -166,6 +173,10 @@ void DrawPipeline::Drain(DrainReason reason, std::uint32_t opcode) { rethrowFailure(); } +std::uint64_t DrawPipeline::DrainRequestsByOpcode(std::uint32_t opcode) { + return Queue0().drainRequests[std::min(opcode, 0x100)].load(std::memory_order_relaxed); +} + std::optional DrawPipeline::PendingLabel(std::uint64_t address, std::size_t bytes) { if (bytes == 0 || bytes > 8) return std::nullopt; const auto end = address + bytes; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp index d07b070444..56e4801c73 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp @@ -33,6 +33,20 @@ const auto& registerMasks() { return masks; } +template +void forEachKeyedUserWord(const QueueState& queue, bool allUserWords, TVisit&& visit) { + const auto words = [&](std::uint32_t base, std::uint32_t count) { + for (auto it = queue.shader.lower_bound(base); it != queue.shader.end() && it->first < base + count; ++it) visit(it->first, it->second); + }; + for (const auto base : {0x00cu, 0x08cu, 0x10cu}) { + const auto resources = queue.shader.find(base - 1); + const auto count = resources == queue.shader.end() ? 0u : ((resources->second >> 1u) & 0x1fu) | (((resources->second >> 27u) & 1u) << 5u); + words(base, allUserWords ? 32u : std::min(count, 32u)); + } + words(0x082u, 2); + words(0x102u, 2); +} + } std::uint64_t RegisterStateKey(const QueueState& queue, bool allUserWords, bool incremental, std::uint64_t* shapeKey, RegisterStateKeyParts* parts) { @@ -86,13 +100,59 @@ std::uint64_t DrawShaderSetKey(const QueueState& queue, const ShaderRegistry& re return key; } -std::uint64_t Driver::drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial, std::uint64_t* shape, std::array* components) { +std::uint64_t Driver::drawIdentityKey(const QueueState& queue, const ShaderRegistry& registry) { + std::uint64_t key = 0x84222325cbf29ce4ull; + const auto mix = [&](std::uint64_t value) { + key ^= value; + key *= 0x100000001b3ull; + }; + for (const auto base : {0x008u, 0x088u, 0x0c8u, 0x108u, 0x148u}) { + const auto low = queue.shader.find(base); + const auto high = queue.shader.find(base + 1); + if (low == queue.shader.end() || high == queue.shader.end()) { + mix(0); + continue; + } + mix(low->second); + mix(high->second); + const auto resources = queue.shader.find(base + 3); + mix(resources != queue.shader.end() ? resources->second : 0); + const auto address = (static_cast(low->second) << 8u) | (static_cast(high->second & 0xffu) << 40u); + auto it = registry.upper_bound(address); + if (it == registry.begin()) { + mix(1); + continue; + } + --it; + for (const auto& snapshot : it->second) mix(reinterpret_cast(snapshot.get())); + mix(address - it->first); + } + return key == 0 ? 1 : key; +} + +std::uint64_t Driver::drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial, std::uint64_t* shape, std::array* components, const DrawRelocation* relocation, std::uint64_t* alias) { + const bool aliased = alias != nullptr && relocation != nullptr && !relocation->words.empty(); + bool readable = true; + std::uint64_t other = 0x6c62272e07bb0142ull; + const auto mixOther = [&](std::uint64_t value) { + other ^= value; + other *= 0x100000001b3ull; + }; static const bool allUserWords = std::getenv("APS5_DRAW_KEY_ALL_USER_WORDS") != nullptr; static const bool verify = std::getenv("APS5_VERIFY_REGISTER_KEYS") != nullptr; std::uint64_t shapeKey = 0; RegisterStateKeyParts parts; auto key = RegisterStateKey(queue, allUserWords, true, &shapeKey, components != nullptr ? &parts : nullptr); if (verify && key != RegisterStateKey(queue, allUserWords, false)) throw std::runtime_error("incremental register fingerprint disagrees with full state"); + if (aliased) { + mixOther(shapeKey); + forEachKeyedUserWord(queue, allUserWords, [&](std::uint32_t offset, std::uint32_t value) { + mixOther(offset); + const auto word = std::find_if(relocation->words.begin(), relocation->words.end(), [&](const RelocatableUserWord& candidate) { return candidate.reg == offset; }); + mixOther(word != relocation->words.end() ? relocatedContentKey(*word, value, readable) : value); + }); + mixOther(deviceSerial); + } const auto mix = [&](std::uint64_t value) { key ^= value; key *= 0x100000001b3ull; @@ -123,8 +183,13 @@ std::uint64_t Driver::drawRegisterKey(const QueueState& queue, const ShaderRegis --it; for (const auto& snapshot : it->second) mix(reinterpret_cast(snapshot.get())); mix(address - it->first); + if (aliased) { + for (const auto& snapshot : it->second) mixOther(reinterpret_cast(snapshot.get())); + mixOther(address - it->first); + } } if (shape != nullptr) *shape = shapeKey; + if (aliased && readable) *alias = other == 0 ? 1 : other; return key; } @@ -170,7 +235,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) { const auto sameMesh = [](const std::optional& x, const std::optional& y) { if (x.has_value() != y.has_value()) return false; if (!x) return true; - return x->inputPrimitive == y->inputPrimitive && x->primitivesPerGroup == y->primitivesPerGroup && x->verticesPerGroup == y->verticesPerGroup && x->maxVertices == y->maxVertices && x->maxPrimitives == y->maxPrimitives && x->threadsPerGroup == y->threadsPerGroup && x->ldsSizeDwords == y->ldsSizeDwords && x->provokingVertex == y->provokingVertex && x->esgsItemSize == y->esgsItemSize; + return x->inputPrimitive == y->inputPrimitive && x->primitivesPerGroup == y->primitivesPerGroup && x->verticesPerGroup == y->verticesPerGroup && x->maxVertices == y->maxVertices && x->maxPrimitives == y->maxPrimitives && x->threadsPerGroup == y->threadsPerGroup && x->ldsSizeDwords == y->ldsSizeDwords && x->provokingVertex == y->provokingVertex && x->esgsItemSize == y->esgsItemSize && x->outputPrimitiveSize == y->outputPrimitiveSize; }; const auto sameTess = [](const std::optional& x, const std::optional& y) { if (x.has_value() != y.has_value()) return false; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp new file mode 100644 index 0000000000..caff5d1bd1 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp @@ -0,0 +1,268 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +namespace { + +constexpr std::size_t MaxAliases = 16384; +constexpr std::size_t MaxContentWords = 64; + +std::uint64_t AlignedWord(std::uint32_t word) { + return word & ~std::uint32_t {3}; +} + +} + +bool Driver::drawRelocationEnabled() { + static const bool enabled = std::getenv("APS5_NO_DRAW_RELOCATION") == nullptr; + return enabled; +} + +std::uint64_t Driver::relocatedContentKey(const RelocatableUserWord& word, std::uint32_t value, bool& readable) { + std::uint64_t hash = 0x9e3779b97f4a7c15ull; + const auto mix = [&](std::uint64_t mixed) { + hash ^= mixed; + hash *= 0x100000001b3ull; + }; + mix(value & 3u); + if (value == 0) { + readable = false; + return hash; + } + std::array content; + for (const auto& [offset, words] : word.reads) { + mix(static_cast(offset)); + mix(words); + for (std::size_t done = 0; done < words; done += MaxContentWords) { + const auto count = std::min(words - done, MaxContentWords); + const auto address = AlignedWord(value) + static_cast(offset) + done * sizeof(std::uint32_t); + if (GuestMemory::CopyMapped(address, std::as_writable_bytes(std::span(content).first(count))) != GuestMemory::Compare::Equal) { + readable = false; + return hash; + } + for (std::size_t i = 0; i < count; ++i) mix(content[i]); + } + } + return hash; +} + +void Driver::tagVariantRuns(DispatchVariant& variant, std::span regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords) { + std::uint64_t relocatable = eligible && drawRelocationEnabled() ? use.asBase & ~use.asValue : 0; + if (hiddenWords != 0) relocatable &= hiddenWords >= 64 ? 0 : ~((std::uint64_t {1} << hiddenWords) - 1); + std::vector> reads; + const auto covered = [&](std::uint64_t address) { + return std::any_of(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& region) { return address >= region.guestAddress && address - region.guestAddress + sizeof(std::uint32_t) <= region.bytes.size(); }); + }; + if (relocatable != 0) { + reads = use.reads; + std::sort(reads.begin(), reads.end()); + reads.erase(std::unique(reads.begin(), reads.end()), reads.end()); + const auto drop = [&](std::int32_t origin) { + if (origin >= 0) relocatable &= ~(std::uint64_t {1} << origin); + }; + for (std::size_t i = 0; i < reads.size(); ++i) { + if (i + 1 < reads.size() && reads[i].first == reads[i + 1].first) { + drop(reads[i].second); + drop(reads[i + 1].second); + } + if (reads[i].second >= 0 && ((reads[i].first & 3u) != 0 || !covered(reads[i].first))) drop(reads[i].second); + } + } + const auto origin = [&](std::uint64_t address) -> std::int8_t { + const auto found = std::lower_bound(reads.begin(), reads.end(), std::make_pair(address, std::numeric_limits::min())); + if (found == reads.end() || found->first != address || found->second < 0) return -1; + return (relocatable >> found->second & 1u) != 0 ? static_cast(found->second) : std::int8_t {-1}; + }; + variant.relocatableWords = relocatable; + for (const auto& region : regions) { + const auto count = region.bytes.size() / sizeof(std::uint32_t); + const auto offset = variant.words.size(); + variant.words.resize(offset + count); + std::memcpy(variant.words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t)); + if (relocatable == 0) { + variant.runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size()); + continue; + } + std::size_t start = 0; + auto current = count != 0 ? origin(region.guestAddress) : std::int8_t {-1}; + for (std::size_t word = 1; word <= count; ++word) { + const auto next = word < count ? origin(region.guestAddress + word * sizeof(std::uint32_t)) : std::int8_t {-2}; + if (next == current) continue; + variant.runs.emplace_back(region.guestAddress + start * sizeof(std::uint32_t), region.guestAddress + word * sizeof(std::uint32_t)); + variant.runBases.push_back(current); + start = word; + current = next; + } + } +} + +namespace { + +template +std::size_t dropEveryOther(TMap& map) { + std::size_t dropped = 0; + bool drop = false; + for (auto it = map.begin(); it != map.end();) { + drop = !drop; + if (!drop) { + ++it; + continue; + } + it = map.erase(it); + ++dropped; + } + return dropped; +} + +} + +void Driver::learnPendingRelocation() { + const auto pending = drawRelocationPending; + drawRelocationPending = {}; + if (pending.key == 0) return; + const auto inserted = drawCache.find(pending.key); + if (inserted == drawCache.end() || inserted->second->decode == nullptr) return; + const auto& entry = *inserted->second; + const auto& programs = entry.decode->programs; + if (entry.stages.size() != programs.size()) return; + const auto known = drawRelocations.find(pending.identity); + auto shape = std::make_shared(); + if (known != drawRelocations.end()) *shape = *known->second; + bool changed = known == drawRelocations.end(); + for (std::size_t i = 0; i < programs.size(); ++i) { + if (entry.stages[i].empty()) continue; + const auto& variant = *entry.stages[i].front(); + const auto& program = programs[i]; + for (std::uint32_t index = program.hiddenUserWords; index < program.userData.size() && index < 64; ++index) { + const auto reg = program.userDataBase + index - program.hiddenUserWords; + const auto held = std::find_if(shape->words.begin(), shape->words.end(), [&](const RelocatableUserWord& word) { return word.reg == reg; }); + if ((variant.relocatableWords >> index & 1u) == 0) { + if (held == shape->words.end()) continue; + shape->words.erase(held); + shape->banned.push_back(reg); + changed = true; + continue; + } + if (held != shape->words.end() || std::find(shape->banned.begin(), shape->banned.end(), reg) != shape->banned.end()) continue; + RelocatableUserWord word{reg, {}}; + const auto base = AlignedWord(program.userData[index]); + for (std::size_t run = 0; run < variant.runs.size() && run < variant.runBases.size(); ++run) { + if (variant.runBases[run] != static_cast(index)) continue; + word.reads.emplace_back(static_cast(variant.runs[run].first - base), static_cast((variant.runs[run].second - variant.runs[run].first) / sizeof(std::uint32_t))); + } + shape->words.push_back(std::move(word)); + changed = true; + } + } + if (changed) { + std::sort(shape->words.begin(), shape->words.end(), [](const RelocatableUserWord& a, const RelocatableUserWord& b) { return a.reg < b.reg; }); + drawRelocations[pending.identity] = std::move(shape); + } + if (pending.alias == 0) return; + if (drawAliases.size() >= MaxAliases) drawRelocationCounters.evicted += dropEveryOther(drawAliases); + drawAliases[pending.alias] = inserted->second; +} + +void Driver::relocateDrawEntry(const QueueState& queue, std::uint64_t alias, std::shared_ptr& entry, std::shared_ptr& decode, std::vector& patches) { + auto& counters = drawRelocationCounters; + ++counters.lookups; + const auto found = drawAliases.find(alias); + if (found == drawAliases.end() || found->second->decode == nullptr) return; + const auto donor = found->second; + const auto& programs = donor->decode->programs; + if (donor->stages.size() != programs.size()) return; + struct Move { + std::size_t program; + std::uint32_t index, from, to; + }; + std::vector moves; + for (std::size_t i = 0; i < programs.size(); ++i) { + const auto& program = programs[i]; + for (std::uint32_t index = program.hiddenUserWords; index < program.userData.size(); ++index) { + const auto current = readUserData(queue.shader, program.userDataBase + index - program.hiddenUserWords); + if (current != program.userData[index]) moves.push_back({i, index, program.userData[index], current}); + } + } + if (moves.empty()) return; + ++counters.donors; + auto relocated = std::make_shared(); + relocated->decode = donor->decode; + relocated->stages.resize(donor->stages.size()); + for (std::size_t i = 0; i < donor->stages.size(); ++i) { + const bool moved = std::any_of(moves.begin(), moves.end(), [&](const Move& move) { return move.program == i; }); + if (!moved) { + relocated->stages[i] = donor->stages[i]; + continue; + } + for (const auto& variant : donor->stages[i]) { + if (variant == nullptr || variant->compiled == nullptr) continue; + const bool allowed = std::all_of(moves.begin(), moves.end(), [&](const Move& move) { return move.program != i || (move.index < 64 && (variant->relocatableWords >> move.index & 1u) != 0 && move.from != 0 && move.to != 0 && ((move.from ^ move.to) & 3u) == 0); }); + if (!allowed) { + ++counters.refused; + continue; + } + auto copy = std::make_shared(); + copy->runs = variant->runs; + copy->runBases = variant->runBases; + copy->relocatableWords = variant->relocatableWords; + copy->words = variant->words; + copy->forgetSerial = variant->forgetSerial; + copy->compiled = variant->compiled; + copy->shader = variant->shader; + copy->dataPositions = variant->dataPositions; + copy->dataSlots = variant->dataSlots; + copy->flatBinding = variant->flatBinding; + copy->pushOffset = variant->pushOffset; + copy->vertexInfo = variant->vertexInfo; + for (const auto& move : moves) { + if (move.program != i) continue; + const auto delta = AlignedWord(move.to) - AlignedWord(move.from); + for (std::size_t run = 0; run < copy->runs.size() && run < copy->runBases.size(); ++run) { + if (copy->runBases[run] != static_cast(move.index)) continue; + copy->runs[run].first += delta; + copy->runs[run].second += delta; + } + const auto& forwarded = variant->compiled->shaderDataUserWords; + for (std::uint32_t slot = 0; slot < forwarded.size(); ++slot) { + if (forwarded[slot] == move.index) copy->shaderDataPatches.emplace_back(slot, move.to); + } + } + relocated->stages[i].push_back(std::move(copy)); + } + } + patches.clear(); + for (const auto& move : moves) patches.push_back({move.program, move.index, move.to}); + entry = std::move(relocated); + decode = donor->decode; +} + +void Driver::applyShaderDataPatches(DispatchVariant& variant) { + if (variant.shaderDataPatches.empty()) return; + auto rebased = std::make_shared(*variant.compiled); + for (const auto& [slot, value] : variant.shaderDataPatches) { + const auto at = static_cast(slot) * sizeof(std::uint32_t); + if (at + sizeof(std::uint32_t) <= rebased->pushConstants.size()) std::memcpy(rebased->pushConstants.data() + at, &value, sizeof(value)); + for (auto& binding : rebased->bindings) { + if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData && slot < binding.guestDescriptor.size()) binding.guestDescriptor[slot] = value; + } + } + variant.compiled = std::move(rebased); + variant.shaderDataPatches.clear(); +} + +void Driver::reportDrawRelocation() { + auto& counters = drawRelocationCounters; + AgcDriver::ProfilePrint_nid_no_patch("[draw-relocate] %llu alias lookups: %llu donors with moved user words, %llu refusals, %llu hits through a relocated entry; %zu shader sets learned, %zu aliases held (%llu dropped at the limit)\n", static_cast(counters.lookups), static_cast(counters.donors), static_cast(counters.refused), static_cast(counters.hits), drawRelocations.size(), drawAliases.size(), static_cast(counters.evicted)); + counters = {}; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp index 8fdd044922..7aa349ce2d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp @@ -1,3 +1,5 @@ +#include "prx/libSceAgcDriver/Execution/include/AsyncLog.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" @@ -38,6 +40,7 @@ void PrintFailure(std::exception_ptr error) { } catch (...) { std::fprintf(stderr, "[gpu] driver failure with a non-standard exception\n"); } + FlushAsyncLog_nid_no_patch(); std::fflush(stderr); } @@ -117,13 +120,16 @@ void Driver::stop() { std::lock_guard gpuLock(GuestMemory::GpuMutex()); device.Reset(); replacedDevices.clear(); + Graphics::FlushPipelineJournal(); Graphics::ShutdownGuestBufferWorkers(); PerformanceTrace::StopOutput(); + FlushAsyncLog_nid_no_patch(); StopProfileOutput_nid_no_patch(); stopped = true; } Driver::Driver() { + InstallAsyncLog_nid_no_patch(); static_cast(GuestMemory::TrackerGeneration()); ShaderMemory::SetWaitedMsProvider(&Graphics::Recorder::ThreadWaitedMs); try { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index 34b051bb28..47b88455e4 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -50,6 +50,31 @@ void Driver::noteWrittenBuffers(std::uint64_t program, std::uint32_t queue, cons while (writtenBuffers.size() > WrittenBufferRing) writtenBuffers.pop_front(); } +std::vector> DispatchWriteRanges(const ShaderRecompiler::RecompileResult& compiled, bool& bounded) { + bounded = compiled.bdaAbiVersion == 0; + std::vector> ranges; + constexpr std::uint64_t UnknownImageBytes = 64ull * 1024 * 1024; + for (const auto& binding : compiled.bindings) { + if (binding.role == ShaderRecompiler::DescriptorRole::Gds || binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer) bounded = false; + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) continue; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const bool written = element >= binding.imageWritten.size() || binding.imageWritten[element]; + if (!written) continue; + if (binding.guestDescriptor.size() < (static_cast(element) + 1) * 8) { + bounded = false; + continue; + } + const auto resource = Graphics::DecodeTextureResource(std::span(binding.guestDescriptor).subspan(static_cast(element) * 8, 8)); + if (resource.baseAddress == 0) continue; + const auto surfaceBytes = Graphics::DescribeSurface(resource).guestBytes; + ranges.emplace_back(resource.baseAddress, resource.baseAddress + (surfaceBytes != 0 ? surfaceBytes : UnknownImageBytes)); + if (resource.dccAddress != 0) ranges.emplace_back(resource.dccAddress, resource.dccAddress + (surfaceBytes != 0 ? surfaceBytes / 256 + 1 : UnknownImageBytes)); + } + } + visitWrittenBuffers(compiled, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); }); + return ranges; +} + void Driver::noteForeignWriter(std::uint64_t begin, std::uint64_t end, std::uint32_t queue) { if (!foreignWriters() || !(writeEvidenceEnabled() || traceCapSync()) || end <= begin) return; std::lock_guard lock(writtenBuffersMutex); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index 4bd279a45f..02458b6484 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -314,8 +314,10 @@ void Driver::execute(const Submission& submission) { currentPacketOffset() = cursor; if (gateOpened()) { gateOpened() = false; - if (pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode); - timed(&WorkerProfile::dispatchMs, [&] { resolveGroupAhead(submission, queue, cursor); }); + if (!pipelineDispatch() || submission.queue != 0) { + if (pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode); + timed(&WorkerProfile::dispatchMs, [&] { resolveGroupAhead(submission, queue, cursor); }); + } } if (opcode == 0x15) timed(&WorkerProfile::dispatchMs, [&] { dispatch(queue, packet, submission); }); else timed(&WorkerProfile::dispatchMs, [&] { dispatchIndirect(queue, packet, submission); }); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index e914fdd8bb..bbdc2b32d0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" @@ -233,10 +234,18 @@ void Driver::run(std::uint32_t id) noexcept { if (APS5_ENABLE_TIMING_LOG) submission.dequeuedAt = std::chrono::steady_clock::now(); if (id == 0) queue0Executing = submission.suspend ? 0 : submission.received; if (submission.waitFree) { - orderHolders.fetch_add(1, std::memory_order_acq_rel); - const auto released = [&] { return failure || stopping || orderReleased(id, submission.received); }; - while (!released()) poll(released); - orderHolders.fetch_sub(1, std::memory_order_acq_rel); + { + struct OrderHold { + std::atomic& holders; + explicit OrderHold(std::atomic& holders) : holders(holders) { holders.fetch_add(1, std::memory_order_acq_rel); } + ~OrderHold() { holders.fetch_sub(1, std::memory_order_acq_rel); } + OrderHold(const OrderHold&) = delete; + OrderHold& operator=(const OrderHold&) = delete; + } hold{orderHolders}; + const auto released = [&] { return failure || stopping || orderReleased(id, submission.received); }; + StallWatch stall(id); + while (!released() && !poll(released)) stall.Check("order behind queue 0 for event", submission.received); + } rethrowFailure(); } if (APS5_ENABLE_TIMING_LOG) submission.orderedAt = std::chrono::steady_clock::now(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp index b7cfef3d62..b1572bfa26 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -259,6 +260,8 @@ void Driver::reserveOutputs(Submission& submission) { void Driver::executeRewindTail(const Submission& stalled) { PerformanceTimer timing("Driver.Rewind"); + static const bool ring = ReadSubmissionRingSettings(); + if (ring) PrepareSubmissionRing(stalled.queue); std::atomic_ref control(*const_cast(stalled.rewindTail - 1)); if ((control.load(std::memory_order_acquire) & Pm4::RewindValid) == 0) { noteWaitBlocked(stalled.queue, reinterpret_cast(stalled.rewindTail - 1), true); @@ -267,10 +270,12 @@ void Driver::executeRewindTail(const Submission& stalled) { std::uint32_t queue; ~BlockedWait() { driver.noteWaitBlocked(queue, 0, false); } } blocked{*this, stalled.queue}; + StallWatch stall(stalled.queue); while ((control.load(std::memory_order_acquire) & Pm4::RewindValid) == 0) { CheckFailure(); checkStopping(); noteAwaitingTitle(stalled.queue, reinterpret_cast(stalled.rewindTail - 1)); + stall.Check("REWIND at", reinterpret_cast(stalled.rewindTail - 1)); PollSleep(); } } @@ -303,6 +308,7 @@ void Driver::executeRewindTail(const Submission& stalled) { tail.serial = stalled.serial; tail.received = ++eventSerial; if (capture != nullptr) capture->Take(captured, tail.serial, captureTime(), tail.heldAtSubmit); + if (ring) RecordSubmission(tail.queue, tail.serial, tail.commands, stalled.rewindTail, true); } if (capture != nullptr) writeCaptured(*this, *capture, captured); if (APS5_ENABLE_TIMING_LOG) tail.enqueuedAt = tail.dequeuedAt = tail.orderedAt = std::chrono::steady_clock::now(); @@ -313,7 +319,9 @@ void Driver::executeRewindTail(const Submission& stalled) { void Driver::Submit(const Packet* packet, std::uint32_t queue) { const auto receivedAt = APS5_ENABLE_TIMING_LOG ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; CheckFailure(); - require(queue == 0 || (queue >= 0x20 && queue < 0x58), "unsupported compute queue"); + require(queue == 0 || (queue >= 0x20 && queue < SubmissionQueueEnd), "unsupported compute queue"); + static const bool ring = ReadSubmissionRingSettings(); + if (ring) PrepareSubmissionRing(queue); GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet)); const auto descriptor = *packet; require(descriptor.flags == 0, "nonzero submission flags are not implemented"); @@ -355,6 +363,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { const auto now = std::chrono::steady_clock::now(); submission.enqueuedAt = now; if (capture != nullptr) capture->Take(captured, submission.serial, captureTime(), submission.heldAtSubmit); + if (ring) RecordSubmission(queue, submission.serial, submission.commands, descriptor.addr, false); if (profile) { ++costs.submissions; costs.validateNs += static_cast(std::chrono::duration_cast(validated - copied).count()); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/SubmissionRing.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/SubmissionRing.cpp new file mode 100644 index 0000000000..900cbd946d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/SubmissionRing.cpp @@ -0,0 +1,281 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include "prx/libc/include/FailureReport.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +namespace AgcDriver::DriverDetail { + +namespace { + +constexpr std::size_t KeptWords = 512; +constexpr std::size_t MaximumCapacity = 4096; +constexpr std::size_t MaximumStallSeconds = 86400; +constexpr std::size_t PayloadWordsShown = 8; + +struct Entry { + std::uint64_t tenthsOfMs = 0; + std::uint64_t serial = 0; + std::uint64_t guest = 0; + std::size_t dwords = 0; + std::size_t kept = 0; + bool rewindTail = false; + std::array words{}; +}; + +struct Ring { + std::mutex mutex; + std::once_flag allocated; + std::atomic recorded{0}; + std::unique_ptr entries; +}; + +void reportFailure() noexcept; + +struct Rings { + std::array queues; +}; + +Rings& rings() { + static auto* value = new Rings; + return *value; +} + +struct Registration { + Registration() { LibcRegisterFailureReport_nid_no_patch(reportFailure); } + ~Registration() { LibcUnregisterFailureReport_nid_no_patch(reportFailure); } + Registration(const Registration&) = delete; + Registration& operator=(const Registration&) = delete; +}; + +std::size_t environmentCount(const char* name, std::size_t fallback, std::size_t maximum, const char* reason) { + const char* text = std::getenv(name); + if (text == nullptr) return fallback; + const auto* end = text + std::strlen(text); + std::size_t value = 0; + const auto [last, error] = std::from_chars(text, end, value); + require(error == std::errc{} && last == end && value <= maximum, reason); + return value; +} + +bool writeLine(const char* text, std::size_t length) { +#ifdef _WIN32 + const HANDLE handle = GetStdHandle(STD_ERROR_HANDLE); + if (handle == nullptr || handle == INVALID_HANDLE_VALUE) return false; + DWORD written = 0; + return WriteFile(handle, text, static_cast(length), &written, nullptr) != 0 && written == length; +#else + while (length != 0) { + const auto written = ::write(2, text, length); + if (written < 0 && errno == EINTR) continue; + if (written <= 0) return false; + text += written; + length -= static_cast(written); + } + return true; +#endif +} + +class Line { +public: + void Add(const char* format, ...) { + if (length >= sizeof(text) - 2) return; + va_list arguments; + va_start(arguments, format); + const int added = std::vsnprintf(text + length, sizeof(text) - 1 - length, format, arguments); + va_end(arguments); + if (added > 0) length = std::min(length + static_cast(added), sizeof(text) - 2); + } + + bool Write() { + text[length] = '\n'; + const bool written = writeLine(text, length + 1); + length = 0; + return written; + } + +private: + char text[512]; + std::size_t length = 0; +}; + +void addName(Line& line, std::uint32_t header) { + const auto name = Pm4::KnownName(header); + if (name.empty()) line.Add("UNKNOWN_0x%02x", static_cast((header >> 8u) & 0xffu)); + else line.Add("%.*s", static_cast(name.size()), name.data()); +} + +bool writePackets(const Entry& entry, Line& line) { + bool written = true; + for (std::size_t cursor = 0; cursor < entry.kept;) { + const auto header = entry.words[cursor]; + line.Add("[pm4 ring] +0x%04zx ", cursor); + if (Pm4::FillerPacket(header)) { + std::size_t run = 1; + while (cursor + run < entry.kept && Pm4::FillerPacket(entry.words[cursor + run])) ++run; + addName(line, header); + line.Add(" x%zu", run); + written = line.Write() && written; + cursor += run; + continue; + } + if ((header >> 30u) != 3u) { + line.Add("not a type-3 header:"); + for (std::size_t i = 0; i <= PayloadWordsShown && cursor + i < entry.kept; ++i) line.Add(" %08x", entry.words[cursor + i]); + written = line.Write() && written; + break; + } + const auto words = Pm4::PacketWords(header); + const auto shown = std::min(words, PayloadWordsShown + 1); + addName(line, header); + for (std::size_t i = 1; i < shown && cursor + i < entry.kept; ++i) line.Add(" %08x", entry.words[cursor + i]); + if (cursor + shown > entry.kept) line.Add(" (cut)"); + else if (words > shown) line.Add(" (+%zu dwords)", words - shown); + written = line.Write() && written; + cursor += words; + } + return written; +} + +bool dump(std::string_view reason, bool failing) { + const auto capacity = SubmissionRingCapacity(); + const auto label = static_cast(std::min(reason.size(), 200)); + Line line; + if (capacity == 0) { + line.Add("[pm4 ring] %.*s: disabled (APS5_PM4_RING=0)", label, reason.data()); + return line.Write(); + } + auto& state = rings(); + std::size_t used = 0; + for (const auto& ring : state.queues) used += ring.recorded.load(std::memory_order_acquire) != 0 ? 1 : 0; + if (used == 0) { + line.Add("[pm4 ring] %.*s: no submissions recorded", label, reason.data()); + return line.Write(); + } + line.Add("[pm4 ring] %.*s: %zu queue(s), up to %zu submissions each, oldest first", label, reason.data(), used, capacity); + bool written = line.Write(); + for (std::size_t queue = 0; queue < state.queues.size(); ++queue) { + auto& ring = state.queues[queue]; + if (ring.recorded.load(std::memory_order_acquire) == 0) continue; + std::unique_lock lock(ring.mutex, std::defer_lock); + if (failing) { + for (int attempt = 0; attempt < 100 && !lock.try_lock(); ++attempt) std::this_thread::yield(); + } else { + lock.lock(); + } + const auto recorded = ring.recorded.load(std::memory_order_acquire); + if (!lock.owns_lock()) { + line.Add("[pm4 ring] queue 0x%zx: %llu submissions recorded, ring busy", queue, static_cast(recorded)); + written = line.Write() && written; + continue; + } + const auto first = recorded > capacity ? recorded - capacity : 0; + line.Add("[pm4 ring] queue 0x%zx: %llu submissions recorded, last %llu kept", queue, static_cast(recorded), static_cast(recorded - first)); + written = line.Write() && written; + for (auto ordinal = first; ordinal < recorded; ++ordinal) { + const auto& entry = ring.entries[ordinal % capacity]; + line.Add("[pm4 ring] queue 0x%zx #%llu serial %llu t=%llu.%llu ms: %zu dwords from 0x%llx%s, first %zu kept", queue, static_cast(ordinal + 1), static_cast(entry.serial), + static_cast(entry.tenthsOfMs / 10), static_cast(entry.tenthsOfMs % 10), + entry.dwords, static_cast(entry.guest), entry.rewindTail ? " (rewind tail)" : "", entry.kept); + written = line.Write() && written; + written = writePackets(entry, line) && written; + } + } + return written; +} + +void reportFailure() noexcept { + try { + dump("failure report", true); + } catch (...) { + static constexpr char text[] = "[pm4 ring] failure report: dump failed\n"; + writeLine(text, sizeof(text) - 1); + } +} + +} + +std::size_t SubmissionRingCapacity() { + static const std::size_t value = environmentCount("APS5_PM4_RING", 16, MaximumCapacity, "APS5_PM4_RING must be a number of submissions from 0 to 4096"); + return value; +} + +std::chrono::seconds StallWatchdogPeriod() { + static const std::chrono::seconds value(environmentCount("APS5_STALL_WATCHDOG_SECS", 0, MaximumStallSeconds, "APS5_STALL_WATCHDOG_SECS must be a number of seconds from 0 to 86400")); + return value; +} + +bool ReadSubmissionRingSettings() { + StallWatchdogPeriod(); + return SubmissionRingCapacity() != 0; +} + +void PrepareSubmissionRing(std::uint32_t queue) { + const auto capacity = SubmissionRingCapacity(); + require(capacity != 0, "the PM4 ring is disabled"); + require(queue < SubmissionQueueEnd, "PM4 ring queue out of range"); + static const Registration registration; + auto& ring = rings().queues[queue]; + std::call_once(ring.allocated, [&] { ring.entries = std::make_unique(capacity); }); +} + +void RecordSubmission(std::uint32_t queue, std::uint64_t serial, std::span commands, const void* guest, bool rewindTail) { + const auto capacity = SubmissionRingCapacity(); + require(capacity != 0, "the PM4 ring is disabled"); + require(queue < SubmissionQueueEnd, "PM4 ring queue out of range"); + auto& ring = rings().queues[queue]; + require(ring.entries != nullptr, "PM4 ring recorded before it was prepared"); + const auto tenthsOfMs = static_cast(std::max(TraceMs(), 0.0) * 10.0 + 0.5); + std::lock_guard lock(ring.mutex); + const auto recorded = ring.recorded.load(std::memory_order_relaxed); + auto& entry = ring.entries[recorded % capacity]; + entry.tenthsOfMs = tenthsOfMs; + entry.serial = serial; + entry.guest = reinterpret_cast(guest); + entry.dwords = commands.size(); + entry.kept = std::min(commands.size(), KeptWords); + entry.rewindTail = rewindTail; + std::copy_n(commands.begin(), entry.kept, entry.words.begin()); + ring.recorded.store(recorded + 1, std::memory_order_release); +} + +void DumpSubmissionRings(std::string_view reason) { + require(dump(reason, false), "cannot write the PM4 ring dump to stderr"); +} + +void StallWatch::Check(const char* waiter, std::uint64_t value) { + if (dumped) return; + const auto period = StallWatchdogPeriod(); + if (period.count() == 0) return; + const auto now = std::chrono::steady_clock::now(); + if (since == std::chrono::steady_clock::time_point{}) { + since = now; + return; + } + if (now - since < period) return; + dumped = true; + char reason[192]; + std::snprintf(reason, sizeof(reason), "queue 0x%x %s 0x%llx has not progressed for %lld s (APS5_STALL_WATCHDOG_SECS)", queue, waiter, static_cast(value), static_cast(period.count())); + DumpSubmissionRings(reason); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index b9a7b3ed9c..ab8df700c0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -1462,6 +1462,11 @@ ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device) { return null; } +std::shared_ptr Driver::RegisteredShaders() { + std::lock_guard lock(mutex); + return shaders; +} + void Driver::RegisterShader(const Shader* shader) { PerformanceContext timingContext(FrameTiming::Preparation()); PerformanceTimer timing("Shader.Register"); @@ -1514,7 +1519,11 @@ void Driver::RegisterShader(const Shader* shader) { registered.context = snapshot.registeredState->context; registered.userConfig = snapshot.registeredState->userConfig; std::shared_ptr preparation; - if (AsyncRegistrationPrepare()) { + static const bool lazy = std::getenv("APS5_LAZY_SHADER_PREPARE") != nullptr; + if (lazy) { + static_cast(PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred)); + snapshot.prepared->deferred = true; + } else if (AsyncRegistrationPrepare()) { preparation = PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred); if (preparation != nullptr) preparation->prebuildDevice = localDevice; if (preparation != nullptr && (snapshot.type == 0 || snapshot.type == 1)) BuildRegisteredAbiKey(registered, *localDevice, preparation->abiKey); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp index 3deebcb869..ef7a88bb2f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -330,6 +331,7 @@ void Driver::waitMemory(std::span packet, std::uint32_t que const auto spinStart = std::chrono::steady_clock::now(); bool spinning = pauseSpin; std::uint32_t polls = 0; + StallWatch stall(queue); while (!Pm4::WaitSatisfiedUnchecked(packet)) { if (std::uint32_t writer = queue; !requireMemory && storedSince(packet, awaited, awaitedBytes, received, &writer)) { if (writer != queue) notePendingWait(packet, queue, received); @@ -358,7 +360,9 @@ void Driver::waitMemory(std::span packet, std::uint32_t que if (const auto done = packetsDone.load(); done != lastDone || (!requireMemory && packetsInFlight.load() != 0)) { lastDone = done; start = std::chrono::steady_clock::now(); + stall.Progressed(); } + stall.Check(((packet[0] >> 8u) & 0xffu) == 0x93u ? "WAIT_REG_MEM_64 at" : "WAIT_REG_MEM at", awaited); if (!warned && std::chrono::steady_clock::now() - start > std::chrono::milliseconds(waitTimeoutMs())) { warned = true; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index bdee46ee16..2a8ef54378 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -175,7 +175,7 @@ std::vector copyDataSource(std::span packet, con } -std::string Name(std::uint32_t header) { +std::string_view KnownName(std::uint32_t header) { if (FillerPacket(header)) return "PM4_FILLER"; const auto opcode = (header >> 8u) & 0xffu; if (opcode == 0x10 && (header & 0xfcu) != 0) { @@ -194,9 +194,14 @@ std::string Name(std::uint32_t header) { default: return "UNKNOWN_CUSTOM"; } } - for (const auto& entry : Opcodes) if (entry.value == opcode) return std::string(entry.name); + for (const auto& entry : Opcodes) if (entry.value == opcode) return entry.name; + return {}; +} + +std::string Name(std::uint32_t header) { + if (const auto known = KnownName(header); !known.empty()) return std::string(known); char text[24]{}; - std::snprintf(text, sizeof(text), "UNKNOWN_0x%02x", opcode); + std::snprintf(text, sizeof(text), "UNKNOWN_0x%02x", (header >> 8u) & 0xffu); return text; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp index 7dc7a4640c..3ff6cc3d86 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp @@ -31,6 +31,7 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta const auto* mesh = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_FEATURES_EXT); const auto* clock = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR); const auto* drawParameters = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DRAW_PARAMETERS_FEATURES); + const auto* terminate = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_TERMINATE_INVOCATION_FEATURES_KHR); const auto* maintenance8 = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_8_FEATURES_KHR); Require(target.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version, "shader device profile has an incompatible BDA ABI"); Require(bda != nullptr && bda->bufferDeviceAddress == VK_TRUE && core.shaderInt64 == VK_TRUE, "shader runtime requires enabled bufferDeviceAddress and shaderInt64"); @@ -46,6 +47,7 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta Require(!hasCapability(capability) || enabled, std::string("shader device profile advertises a disabled feature: ") + name); }; checkFeature(spv::CapabilityFloat64, core.shaderFloat64 == VK_TRUE, "shaderFloat64"); + checkFeature(spv::CapabilityGeometry, core.geometryShader == VK_TRUE, "geometryShader"); checkFeature(spv::CapabilityTessellation, core.tessellationShader == VK_TRUE, "tessellationShader"); checkFeature(spv::CapabilityImageGatherExtended, core.shaderImageGatherExtended == VK_TRUE, "shaderImageGatherExtended"); checkFeature(spv::CapabilityClipDistance, core.shaderClipDistance == VK_TRUE, "shaderClipDistance"); @@ -76,6 +78,7 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta extensions.erase(std::unique(extensions.begin(), extensions.end()), extensions.end()); for (const auto& extension : extensions) extensionViews.emplace_back(extension); Require(std::ranges::find(extensionViews, "SPV_KHR_physical_storage_buffer") != extensionViews.end() && std::ranges::find(extensionViews, "SPV_KHR_8bit_storage") != extensionViews.end(), "shader device profile is missing runtime extensions"); + Require(std::ranges::find(extensionViews, "SPV_KHR_terminate_invocation") == extensionViews.end() || (terminate != nullptr && terminate->shaderTerminateInvocation == VK_TRUE), "shader device profile advertises a disabled feature: shaderTerminateInvocation"); this->target.supportedCapabilities = capabilities; this->target.supportedExtensions = extensionViews; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 694152052f..44c1dce86b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -6,6 +6,7 @@ #if ANYPS5_ENABLE_SPIRV_TOOLS #include "SpirvBackend/SpirvOptimizer.hpp" #endif +#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" @@ -215,6 +216,7 @@ struct ComputePipelineObjects { VkShaderModule module = VK_NULL_HANDLE; VkPipelineLayout layout = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + VkDescriptorSetLayout resourcesLayout = VK_NULL_HANDLE; ~ComputePipelineObjects() { if (pipeline != VK_NULL_HANDLE) destroyPipeline(device, pipeline, nullptr); if (layout != VK_NULL_HANDLE) destroyLayout(device, layout, nullptr); @@ -222,6 +224,16 @@ struct ComputePipelineObjects { } }; +// Pipelines are shared by dispatches of one module (the result's run-stable identity) with the +// same push constant use; descriptor set layouts built from the same bindings are compatible, so +// the pipeline layout of the first dispatch, or of the journal replay, serves them all. +struct ComputePipelineKey { + ShaderRecompiler::PipelineIdentity identity; + bool pushConstants = false; + auto operator<=>(const ComputePipelineKey&) const = default; + static ComputePipelineKey Of(const ShaderRecompiler::RecompileResult& shader, bool pushConstants) { return {shader.Identity(), pushConstants}; } +}; + constexpr std::uint64_t WorkgroupMemoryGranule = 1u << 20u; struct WorkgroupMemory { @@ -277,8 +289,10 @@ struct VulkanDevice::State { bool tessellationShader = false; bool meshShader = false; bool fragmentShaderBarycentric = false; + bool shaderTerminateInvocation = false; bool geometryShader = false; bool sampleRateShading = false; + bool storageImageMultisample = false; bool shaderClipDistance = false; bool shaderCullDistance = false; bool shaderClock = false; @@ -340,6 +354,8 @@ struct VulkanDevice::State { bool computeWave32 = false; bool meshWave32 = false; std::uint32_t maxComputeSubgroupSize = 0; + bool computeFullSubgroups = false; + bool spannedWaves = false; // Indirect draw features enabled (see Graphics::Context). bool drawIndirectFirstInstance = false; bool multiDrawIndirect = false; @@ -362,8 +378,12 @@ struct VulkanDevice::State { // dispatch's find-or-insert and the verify switch's lookups touch the map, recipes hold weak // references to its objects. std::mutex computePipelinesMutex; - std::map> computePipelines; - std::map>> computePipelinesBuilding; + std::map> computePipelines; + std::map>> computePipelinesBuilding; + // The journal replay building this device's pipelines on its own threads; drained first at teardown. + std::shared_ptr journalReplay; + std::atomic journalReplayPending{false}; + std::atomic computePipelinesCreated{0}; std::unique_ptr colorTransfer; std::shared_ptr bufferPool; std::unique_ptr emptyBuffer; @@ -671,6 +691,12 @@ struct VulkanDevice::State { } ~State() { + if (journalReplay != nullptr) { + journalReplay->Cancel(); + journalReplay->Wait(); + journalReplay.reset(); + Graphics::ReportPipelineJournal(); + } contextReady = false; if (deviceLostHook && Graphics::DeviceLostOwner() == this) { Graphics::DeviceLostHook() = nullptr; @@ -798,6 +824,7 @@ ComputePipelineReplayResult VulkanDevice::ReplayComputePipeline(const ComputePip VkPhysicalDeviceSubgroupSizeControlPropertiesEXT subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT}; VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &subgroup}; state->InstanceFunction("vkGetPhysicalDeviceProperties2")(state->physical, &properties); + require(capture.stageFlags == 0 || state->computeFullSubgroups, "captured full compute subgroups are unsupported"); require(state->computeWave32 && capture.requiredSubgroupSize >= subgroup.minSubgroupSize && capture.requiredSubgroupSize <= subgroup.maxSubgroupSize && (subgroup.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0, "captured required subgroup size is unsupported"); } const auto target = Target(); @@ -1133,6 +1160,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->fragmentShaderBarycentric = FragmentShaderBarycentricUsable(driver.driverID, state->fragmentShaderBarycentric); } } + VkPhysicalDeviceShaderTerminateInvocationFeaturesKHR terminateFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_TERMINATE_INVOCATION_FEATURES_KHR}; + if (hasExtension(VK_KHR_SHADER_TERMINATE_INVOCATION_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &terminateFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + state->shaderTerminateInvocation = terminateFeatures.shaderTerminateInvocation == VK_TRUE; + } VkPhysicalDeviceFragmentShaderInterlockFeaturesEXT interlockFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT}; if (hasExtension(VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME)) { VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &interlockFeatures}; @@ -1179,6 +1212,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->capabilities.push_back(spv::CapabilityFragmentBarycentricKHR); state->spirvExtensions.push_back("SPV_KHR_fragment_shader_barycentric"); } + if (state->shaderTerminateInvocation) { + deviceExtensions.push_back(VK_KHR_SHADER_TERMINATE_INVOCATION_EXTENSION_NAME); + state->spirvExtensions.push_back("SPV_KHR_terminate_invocation"); + } if (state->fragmentShaderPixelInterlock) { deviceExtensions.push_back(VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME); state->capabilities.push_back(spv::CapabilityFragmentShaderPixelInterlockEXT); @@ -1266,7 +1303,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); if (locations.sampleLocationSubPixelBits >= 4 && locations.sampleLocationCoordinateRange[0] <= 0.0f && locations.sampleLocationCoordinateRange[1] >= 15.0f / 16.0f) { const auto multisampleProperties = state->InstanceFunction("vkGetPhysicalDeviceMultisamplePropertiesEXT"); - for (const auto samples : {VK_SAMPLE_COUNT_2_BIT, VK_SAMPLE_COUNT_4_BIT, VK_SAMPLE_COUNT_8_BIT}) { + for (const auto samples : {VK_SAMPLE_COUNT_1_BIT, VK_SAMPLE_COUNT_2_BIT, VK_SAMPLE_COUNT_4_BIT, VK_SAMPLE_COUNT_8_BIT}) { if ((locations.sampleLocationSampleCounts & samples) == 0) continue; VkMultisamplePropertiesEXT multisample{VK_STRUCTURE_TYPE_MULTISAMPLE_PROPERTIES_EXT}; multisampleProperties(selected, samples, &multisample); @@ -1426,29 +1463,37 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : queueInfo.queueFamilyIndex = family; queueInfo.queueCount = 1; queueInfo.pQueuePriorities = &priority; -#ifdef _WIN32 const float priorities[2]{1.0f, 1.0f}; bool settleQueue = false; std::uint32_t settleFamily = family; - std::uint32_t families = 0; - auto getFamilies = state->InstanceFunction("vkGetPhysicalDeviceQueueFamilyProperties"); - getFamilies(selected, &families, nullptr); - std::vector queues(families); - getFamilies(selected, &families, queues.data()); - if (family < families && queues[family].queueCount >= 2) { - queueInfo.queueCount = 2; - queueInfo.pQueuePriorities = priorities; - settleQueue = true; - } else { - const auto copies = [&](std::uint32_t index) { return index != family && queues[index].queueCount != 0 && (queues[index].queueFlags & (VK_QUEUE_TRANSFER_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) != 0; }; - for (std::uint32_t i = 0; i < families && !settleQueue; ++i) { - if (copies(i) && (queues[i].queueFlags & (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) == 0) settleFamily = i, settleQueue = true; - } - for (std::uint32_t i = 0; i < families && !settleQueue; ++i) { - if (copies(i)) settleFamily = i, settleQueue = true; + std::uint32_t settleIndex = 0; + std::vector queues; +#ifdef _WIN32 + const bool settles = true; +#else + const bool settles = state->dmaBufImport; +#endif + if (settles) { + std::uint32_t families = 0; + auto getFamilies = state->InstanceFunction("vkGetPhysicalDeviceQueueFamilyProperties"); + getFamilies(selected, &families, nullptr); + queues.resize(families); + getFamilies(selected, &families, queues.data()); + if (family < families && queues[family].queueCount >= 2) { + queueInfo.queueCount = 2; + queueInfo.pQueuePriorities = priorities; + settleQueue = true; + settleIndex = 1; + } else { + const auto copies = [&](std::uint32_t index) { return index != family && queues[index].queueCount != 0 && (queues[index].queueFlags & (VK_QUEUE_TRANSFER_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) != 0; }; + for (std::uint32_t i = 0; i < families && !settleQueue; ++i) { + if (copies(i) && (queues[i].queueFlags & (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) == 0) settleFamily = i, settleQueue = true; + } + for (std::uint32_t i = 0; i < families && !settleQueue; ++i) { + if (copies(i)) settleFamily = i, settleQueue = true; + } } } -#endif VkDeviceCreateInfo deviceInfo{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; deviceInfo.queueCreateInfoCount = 1; deviceInfo.pQueueCreateInfos = &queueInfo; @@ -1482,7 +1527,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : // Guest shaders routinely read past descriptor ranges; robust access turns that into zeros // instead of a GPU fault that loses the device. enabled.robustBufferAccess = available.robustBufferAccess; - VkDeviceQueueCreateInfo queueInfos[2]{queueInfo, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}}; + VkDeviceQueueCreateInfo queueInfos[3]{queueInfo, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}}; if (available.sparseBinding && state->dmaBufImport) { std::uint32_t families = 0; auto getFamilies = state->InstanceFunction("vkGetPhysicalDeviceQueueFamilyProperties"); @@ -1510,19 +1555,22 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : enabled.sparseBinding = state->sparseBinding ? VK_TRUE : VK_FALSE; } deviceInfo.pQueueCreateInfos = queueInfos; -#ifdef _WIN32 if (settleQueue && settleFamily != family) { - if (deviceInfo.queueCreateInfoCount == 1) { - queueInfos[1] = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; - queueInfos[1].queueFamilyIndex = settleFamily; - queueInfos[1].queueCount = 1; - queueInfos[1].pQueuePriorities = &priority; - deviceInfo.queueCreateInfoCount = 2; + if (deviceInfo.queueCreateInfoCount == 2 && queueInfos[1].queueFamilyIndex == settleFamily) { + settleQueue = queues[settleFamily].queueCount >= 2; + if (settleQueue) { + queueInfos[1].queueCount = 2; + queueInfos[1].pQueuePriorities = priorities; + settleIndex = 1; + } } else { - settleQueue = false; + auto& settleInfo = queueInfos[deviceInfo.queueCreateInfoCount++]; + settleInfo = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + settleInfo.queueFamilyIndex = settleFamily; + settleInfo.queueCount = 1; + settleInfo.pQueuePriorities = &priority; } } -#endif // Indirect draw records with a non-zero first instance, and several records per call. enabled.drawIndirectFirstInstance = available.drawIndirectFirstInstance; enabled.multiDrawIndirect = available.multiDrawIndirect; @@ -1541,6 +1589,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->occlusionQueryPrecise = enabled.occlusionQueryPrecise == VK_TRUE; // Recompiled storage-image access declares no format (the guest descriptor decides it). enabled.shaderStorageImageMultisample = available.shaderStorageImageMultisample; + state->storageImageMultisample = enabled.shaderStorageImageMultisample == VK_TRUE; if (enabled.shaderStorageImageMultisample) state->capabilities.push_back(spv::CapabilityStorageImageMultisample); state->capabilities.push_back(spv::CapabilityImageMSArray); enabled.shaderStorageImageWriteWithoutFormat = available.shaderStorageImageWriteWithoutFormat; @@ -1651,6 +1700,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : barycentricFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &barycentricFeatures; } + if (state->shaderTerminateInvocation) { + terminateFeatures.shaderTerminateInvocation = VK_TRUE; + terminateFeatures.pNext = byteFeatures.pNext; + byteFeatures.pNext = &terminateFeatures; + } if (state->fragmentShaderPixelInterlock) { interlockFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &interlockFeatures; @@ -1709,10 +1763,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->meshWave32 = state->meshShader && subgroupSizeFeatures.subgroupSizeControl == VK_TRUE && subgroupSize.minSubgroupSize <= 32u && subgroupSize.maxSubgroupSize >= 32u && (subgroupSize.requiredSubgroupSizeStages & VK_SHADER_STAGE_MESH_BIT_EXT) != 0; state->maxComputeSubgroupSize = subgroupSize.maxSubgroupSize; + state->computeFullSubgroups = state->computeWave32 && subgroupSizeFeatures.computeFullSubgroups == VK_TRUE; } if (state->computeWave32 || state->meshWave32) { subgroupSizeFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT}; subgroupSizeFeatures.subgroupSizeControl = VK_TRUE; + subgroupSizeFeatures.computeFullSubgroups = state->computeFullSubgroups ? VK_TRUE : VK_FALSE; deviceExtensions.push_back(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); deviceInfo.enabledExtensionCount = static_cast(deviceExtensions.size()); deviceInfo.ppEnabledExtensionNames = deviceExtensions.data(); @@ -1720,6 +1776,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : byteFeatures.pNext = &subgroupSizeFeatures; APS5_LOG_OUT("Wave32 programs run on subgroups of %u: compute %d, mesh %d", 32u, state->computeWave32, state->meshWave32); } + if (std::getenv("APS5_SPANNED_WAVE") != nullptr) { + state->spannedWaves = state->computeFullSubgroups; + APS5_LOG_OUT("APS5_SPANNED_WAVE: single-wave wave64 compute spans two full 32-lane subgroups: %d", state->spannedWaves); + } timelineFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR}; timelineFeatures.timelineSemaphore = VK_TRUE; if (state->timelineSemaphores) { @@ -1807,12 +1867,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : } state->DeviceFunction("vkGetDeviceQueue")(state->device, family, 0, &state->queue); state->queueFamily = family; -#ifdef _WIN32 if (settleQueue) { state->settleFamily = settleFamily; - state->DeviceFunction("vkGetDeviceQueue")(state->device, settleFamily, settleFamily == family ? 1 : 0, &state->settleQueue); + state->DeviceFunction("vkGetDeviceQueue")(state->device, settleFamily, settleIndex, &state->settleQueue); } -#endif if (state->sparseFamily != VK_QUEUE_FAMILY_IGNORED) state->DeviceFunction("vkGetDeviceQueue")(state->device, state->sparseFamily, 0, &state->sparseQueue); if (checkpoints) state->checkpointMarker = state->DeviceFunction("vkCmdSetCheckpointNV"); if (deviceFault || checkpoints) { @@ -1928,10 +1986,63 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->scaler = std::make_unique(graphicsContext(), VK_FORMAT_B8G8R8A8_UNORM, state->surfaceFormat); state->presentation = std::make_unique(graphicsContext(), state->presentSlots.size()); } + // A device without a window is usually replaced by the window's device before the title draws + // (Driver::AttachWindow): its replay starts at its first draw or dispatch instead, so the + // journal is replayed once, on the device the title draws with. + state->journalReplayPending.store(!compileOnly, std::memory_order_release); + if (window != nullptr) startJournalReplay(); +} + +void VulkanDevice::startJournalReplay() { + if (!state->journalReplayPending.exchange(false, std::memory_order_acq_rel)) return; + state->journalReplay = Graphics::StartPipelineJournalReplay(graphicsContext(), Graphics::PipelineJournalDeviceFingerprint(state->properties), {[this](const ShaderRecompiler::RecompileResult& shader, bool pushConstants) { return ReplayComputePipeline(shader, pushConstants); }}); } VulkanDevice::~VulkanDevice() = default; +VkPipelineCreateFlags VulkanDevice::computePipelineFlags() const { + return (graphicsContext().pipelineExecutableInfo ? VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR : 0) | VK_PIPELINE_CREATE_DISPATCH_BASE_BIT; +} + +std::uint32_t VulkanDevice::computeSubgroupSize(const ShaderRecompiler::RecompileResult& shader) const { + return state->computeWave32 && (shader.hostSubgroupSize == 32u || shader.hostSubgroupSize > state->maxComputeSubgroupSize) ? std::min(shader.hostSubgroupSize, state->maxComputeSubgroupSize) : 0u; +} + +VkPipelineShaderStageCreateFlags VulkanDevice::computeStageFlags(const ShaderRecompiler::RecompileResult& shader) const { + if (!shader.fullSubgroups) return 0; + require(state->computeFullSubgroups && computeSubgroupSize(shader) == 32u, "a spanned wave64 program needs full 32-lane compute subgroups"); + return VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; +} + +bool VulkanDevice::ComputeFullSubgroups() const { + return state->computeFullSubgroups; +} + +void VulkanDevice::WaitForPipelineJournalReplay() { + startJournalReplay(); + if (state->journalReplay != nullptr) state->journalReplay->Wait(); +} + +std::uint64_t VulkanDevice::ComputePipelinesCreated() const { + return state->computePipelinesCreated.load(std::memory_order_relaxed); +} + +bool VulkanDevice::ReplayComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants) { + ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); + const auto key = ComputePipelineKey::Of(shader, pushConstants); + if (key.identity.Empty()) return false; + { + std::lock_guard pipelines(state->computePipelinesMutex); + if (state->computePipelines.contains(key) || state->computePipelinesBuilding.contains(key)) return false; + } + const auto context = graphicsContext(); + const std::array shaders{{{ShaderRecompiler::ShaderStage::Compute, &shader, 0}}}; + const auto layouts = Graphics::DescribeSetLayouts(context, shaders); + auto objects = createComputePipeline(context, shader, layouts.layouts.main, layouts.mainKey, pushConstants ? VK_SHADER_STAGE_COMPUTE_BIT : 0u, 0); + std::lock_guard pipelines(state->computePipelinesMutex); + return state->computePipelines.try_emplace(key, std::move(objects)).second; +} + bool VulkanDevice::ImportGuestMemory(const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice) { if (adoptDevice) GuestMemory::AssertGpuLockHeld("VulkanDevice::ImportGuestMemory"); return Graphics::ImportMappedRanges(graphicsContext(), ranges, generation, adoptDevice); @@ -1940,10 +2051,21 @@ bool VulkanDevice::ImportGuestMemory(const GuestAllocations::Mapped& ranges, std void VulkanDevice::PrepareForReplacement() { GuestMemory::AssertGpuLockHeld("VulkanDevice::PrepareForReplacement"); require(state->recorder != nullptr && Graphics::Recorder::Active() == state->recorder.get(), "device replacement requires its active recorder"); + state->journalReplayPending.store(false, std::memory_order_release); + if (state->journalReplay != nullptr) { + state->journalReplay->Cancel(); + state->journalReplay->Wait(); + state->journalReplay.reset(); + } Graphics::FlushCachedTextures(state->device); Graphics::PublishAllShadows(state->context, Graphics::PublishReason::Teardown); WaitIdle(); Graphics::DestroyShadows(state->device); + // The replacement's draws never find this device's pipelines (the store keys on the device), and + // the device lives on until the driver stops: destroy them now rather than then. + Graphics::ClearCachedPipelines(state->device); + std::lock_guard pipelines(state->computePipelinesMutex); + state->computePipelines.clear(); } void VulkanDevice::WaitIdle() { @@ -3364,6 +3486,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const { target.narrowSubgroupClock = state->narrowSubgroupClock; target.float64DenormPreserve = state->float64DenormPreserve; target.float16DenormPreserve = state->float16DenormPreserve; + target.spannedWaves = state->spannedWaves; target.fixedPushSlots = state->graphicsPipelineLibrary; target.registrationModules = state->graphicsPipelineLibrary; target.srgbDecodeFormats = state->srgbDecodeFormats; @@ -3488,8 +3611,10 @@ Graphics::Context VulkanDevice::buildContext() const { context.samplerTableCapacity = SamplerTableCapacity(context); context.geometryShader = state->geometryShader; context.sampleRateShading = state->sampleRateShading; + context.storageImageMultisample = state->storageImageMultisample; context.shaderClipDistance = state->shaderClipDistance; context.shaderCullDistance = state->shaderCullDistance; + context.shaderTerminateInvocation = state->shaderTerminateInvocation; context.nullDescriptors = state->shaderProfile != nullptr && state->shaderProfile->NullDescriptors(); context.bufferInt64Atomics = state->bufferInt64Atomics; context.placeholderImages = state->placeholderImages.get(); @@ -3558,6 +3683,7 @@ bool VulkanDevice::PrebuildGraphicsLibraries(const Graphics::ShaderStages& stage } void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, std::shared_ptr* recipe) { + startJournalReplay(); for (const auto& shader : shaders) { require(shader.program != nullptr, "missing compiled shader"); ShaderRecompiler::RuntimeAbi::RequireVersion(shader.program->runtimeAbiVersion); @@ -4417,11 +4543,76 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { recordStep(PhaseRecordCompletion); } -std::shared_ptr VulkanDevice::computePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress) { +// The module, pipeline layout and pipeline of a compute result over `setLayout` (the set layout +// its bindings describe): what a dispatch or its preparation builds at the first use of a module, +// and what the journal replay builds on its own threads before that use. +std::shared_ptr VulkanDevice::createComputePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - const std::uint64_t pipelineKey = shader.PipelineVariantId() != 0 ? (shader.PipelineVariantId() << 1u) | (pushStages != 0 ? 1u : 0u) : 0u; + const auto start = std::chrono::steady_clock::now(); + auto objects = std::make_shared(); + objects->device = state->device; + objects->destroyModule = state->DeviceFunction("vkDestroyShaderModule"); + objects->destroyLayout = state->DeviceFunction("vkDestroyPipelineLayout"); + objects->destroyPipeline = state->DeviceFunction("vkDestroyPipeline"); + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = shader.spirv.size() * sizeof(std::uint32_t); + moduleInfo.pCode = shader.spirv.data(); + check(state->DeviceFunction("vkCreateShaderModule")(state->device, &moduleInfo, nullptr, &objects->module), "vkCreateShaderModule"); + objects->resourcesLayout = setLayout; + const std::array setLayouts{setLayout, shader.workgroupMemoryDwords != 0 ? state->workgroupMemoryLayout : VK_NULL_HANDLE}; + const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes}; + VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + layoutInfo.setLayoutCount = shader.workgroupMemoryDwords != 0 ? ShaderRecompiler::WorkgroupMemoryDescriptorSet + 1 : 1; + layoutInfo.pSetLayouts = setLayouts.data(); + layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; + layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; + check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout"); + const VkPipelineCreateFlags pipelineFlags = computePipelineFlags(); + const Graphics::PipelineSpecialization specialization(shader); + const auto subgroupSize = computeSubgroupSize(shader); + const auto stageFlags = computeStageFlags(shader); + if (const char* directory = std::getenv("APS5_DUMP_COMPUTE_PIPELINES"); directory != nullptr && *directory != '\0' && CaptureComputePipeline(shader.spirv.size())) { + ComputePipelineCapture capture; + capture.spirv = shader.spirv.Copy(); + ComputeDescriptorSetLayout resourcesLayout; + const bool updateAfterBind = layoutKey.size() % 4u == 1u; + resourcesLayout.flags = updateAfterBind ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT : 0; + for (std::size_t index = 0; index + 4u <= layoutKey.size(); index += 4u) { + const auto type = static_cast(layoutKey[index + 1u]); + resourcesLayout.bindings.push_back({layoutKey[index], type, layoutKey[index + 2u], layoutKey[index + 3u], updateAfterBind && type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT : 0u}); + } + capture.sets.push_back(std::move(resourcesLayout)); + if (shader.workgroupMemoryDwords != 0) capture.sets.push_back({0, {{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, 0}}}); + if (pushStages != 0) capture.pushConstants.push_back(push); + for (const auto& value : shader.specialization) capture.specialization.push_back({value.id, value.value}); + capture.requiredSubgroupSize = subgroupSize; + capture.stageFlags = stageFlags; + capture.flags = pipelineFlags; + capture.deviceName = state->properties.deviceName; + capture.vendorId = state->properties.vendorID; + capture.deviceId = state->properties.deviceID; + capture.driverVersion = state->properties.driverVersion; + capture.apiVersion = state->properties.apiVersion; + std::copy_n(state->properties.pipelineCacheUUID, VK_UUID_SIZE, capture.pipelineCacheUuid.begin()); + const auto path = capture.Write(directory); + std::fprintf(stderr, "[compute-capture] %s\n", path.string().c_str()); + } + if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words (program 0x%llx)\n", shader.spirv.size(), static_cast(programAddress)); + CreateComputePipeline(state->device, state->DeviceFunction("vkCreateComputePipelines"), context.pipelineCache, objects->module, objects->layout, pipelineFlags, stageFlags, specialization.Info(), subgroupSize, &objects->pipeline, context.pipelineCacheSynchronization); + objects->destroyModule(objects->device, objects->module, nullptr); + objects->module = VK_NULL_HANDLE; + computePipelinesCreated.fetch_add(1, std::memory_order_relaxed); + state->computePipelinesCreated.fetch_add(1, std::memory_order_relaxed); + if (GuestMemory::GpuMutex().HeldByThisThread()) computePipelinesCreatedLocked.fetch_add(1, std::memory_order_relaxed); + Graphics::LogPipelineStatistics_nid_no_patch(context, objects->pipeline); + if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] created a pipeline for %zu SPIR-V words in %.0f ms (program 0x%llx)\n", shader.spirv.size(), std::chrono::duration(std::chrono::steady_clock::now() - start).count(), static_cast(programAddress)); + return objects; +} + +std::shared_ptr VulkanDevice::computePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress) { + const auto pipelineKey = ComputePipelineKey::Of(shader, pushStages != 0); std::promise> building; - if (pipelineKey != 0) { + if (!pipelineKey.identity.Empty()) { std::shared_future> other; { std::lock_guard pipelines(state->computePipelinesMutex); @@ -4432,7 +4623,7 @@ std::shared_ptr VulkanDevice::computePipeline(const Grap if (other.valid()) return other.get(); } const auto finish = [&](std::shared_ptr built, std::exception_ptr error) { - if (pipelineKey == 0) return built; + if (pipelineKey.identity.Empty()) return built; { std::lock_guard pipelines(state->computePipelinesMutex); if (built != nullptr) built = state->computePipelines.try_emplace(pipelineKey, built).first->second; @@ -4444,60 +4635,8 @@ std::shared_ptr VulkanDevice::computePipeline(const Grap }; std::shared_ptr objects; try { - const auto start = std::chrono::steady_clock::now(); - objects = std::make_shared(); - objects->device = state->device; - objects->destroyModule = state->DeviceFunction("vkDestroyShaderModule"); - objects->destroyLayout = state->DeviceFunction("vkDestroyPipelineLayout"); - objects->destroyPipeline = state->DeviceFunction("vkDestroyPipeline"); - VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; - moduleInfo.codeSize = shader.spirv.size() * sizeof(std::uint32_t); - moduleInfo.pCode = shader.spirv.data(); - check(state->DeviceFunction("vkCreateShaderModule")(state->device, &moduleInfo, nullptr, &objects->module), "vkCreateShaderModule"); - const std::array setLayouts{setLayout, shader.workgroupMemoryDwords != 0 ? state->workgroupMemoryLayout : VK_NULL_HANDLE}; - const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes}; - VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; - layoutInfo.setLayoutCount = shader.workgroupMemoryDwords != 0 ? ShaderRecompiler::WorkgroupMemoryDescriptorSet + 1 : 1; - layoutInfo.pSetLayouts = setLayouts.data(); - layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; - layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; - check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout"); - const VkPipelineCreateFlags pipelineFlags = (context.pipelineExecutableInfo ? VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR : 0) | VK_PIPELINE_CREATE_DISPATCH_BASE_BIT; - const Graphics::PipelineSpecialization specialization(shader); - const auto subgroupSize = state->computeWave32 && (shader.hostSubgroupSize == 32u || shader.hostSubgroupSize > state->maxComputeSubgroupSize) ? std::min(shader.hostSubgroupSize, state->maxComputeSubgroupSize) : 0u; - if (const char* directory = std::getenv("APS5_DUMP_COMPUTE_PIPELINES"); directory != nullptr && *directory != '\0' && CaptureComputePipeline(shader.spirv.size())) { - ComputePipelineCapture capture; - capture.spirv = shader.spirv.Copy(); - ComputeDescriptorSetLayout resourcesLayout; - const bool updateAfterBind = layoutKey.size() % 4u == 1u; - resourcesLayout.flags = updateAfterBind ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT : 0; - for (std::size_t index = 0; index + 4u <= layoutKey.size(); index += 4u) { - const auto type = static_cast(layoutKey[index + 1u]); - resourcesLayout.bindings.push_back({layoutKey[index], type, layoutKey[index + 2u], layoutKey[index + 3u], updateAfterBind && type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT : 0u}); - } - capture.sets.push_back(std::move(resourcesLayout)); - if (shader.workgroupMemoryDwords != 0) capture.sets.push_back({0, {{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, 0}}}); - if (pushStages != 0) capture.pushConstants.push_back(push); - for (const auto& value : shader.specialization) capture.specialization.push_back({value.id, value.value}); - capture.requiredSubgroupSize = subgroupSize; - capture.flags = pipelineFlags; - capture.deviceName = state->properties.deviceName; - capture.vendorId = state->properties.vendorID; - capture.deviceId = state->properties.deviceID; - capture.driverVersion = state->properties.driverVersion; - capture.apiVersion = state->properties.apiVersion; - std::copy_n(state->properties.pipelineCacheUUID, VK_UUID_SIZE, capture.pipelineCacheUuid.begin()); - const auto path = capture.Write(directory); - std::fprintf(stderr, "[compute-capture] %s\n", path.string().c_str()); - } - if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words (program 0x%llx)\n", shader.spirv.size(), static_cast(programAddress)); - CreateComputePipeline(state->device, state->DeviceFunction("vkCreateComputePipelines"), context.pipelineCache, objects->module, objects->layout, pipelineFlags, 0, specialization.Info(), subgroupSize, &objects->pipeline, context.pipelineCacheSynchronization); - objects->destroyModule(objects->device, objects->module, nullptr); - objects->module = VK_NULL_HANDLE; - computePipelinesCreated.fetch_add(1, std::memory_order_relaxed); - if (GuestMemory::GpuMutex().HeldByThisThread()) computePipelinesCreatedLocked.fetch_add(1, std::memory_order_relaxed); - Graphics::LogPipelineStatistics_nid_no_patch(context, objects->pipeline); - if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] created a pipeline for %zu SPIR-V words in %.0f ms (program 0x%llx)\n", shader.spirv.size(), std::chrono::duration(std::chrono::steady_clock::now() - start).count(), static_cast(programAddress)); + objects = createComputePipeline(context, shader, setLayout, layoutKey, pushStages, programAddress); + if (Graphics::PipelineJournalEnabled()) Graphics::RecordComputePipeline(shader, pushStages != 0); } catch (...) { finish(nullptr, std::current_exception()); throw; @@ -4506,6 +4645,7 @@ std::shared_ptr VulkanDevice::computePipeline(const Grap } VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipeOut) { + startJournalReplay(); state->recorder->BoundKeptBytes(); ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); PerformanceTimer timing("Vulkan.Dispatch"); @@ -4525,11 +4665,9 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } auto pushBytes = Graphics::AssemblePushConstants(shaders); const auto context = graphicsContext(); - // Pipelines are shared by dispatches of one compiled variant; descriptor set layouts built from the - // same bindings are compatible, so the pipeline layout of the first dispatch serves them all. - const std::uint64_t pipelineKey = shader.PipelineVariantId() != 0 ? (shader.PipelineVariantId() << 1u) | (pushStages != 0 ? 1u : 0u) : 0u; + const auto pipelineKey = ComputePipelineKey::Of(shader, pushStages != 0); std::shared_ptr objects = prepared != nullptr ? prepared->objects : nullptr; - if (objects == nullptr && pipelineKey != 0) { + if (objects == nullptr && !pipelineKey.identity.Empty()) { std::lock_guard pipelines(state->computePipelinesMutex); if (const auto found = state->computePipelines.find(pipelineKey); found != state->computePipelines.end()) objects = found->second; } @@ -4672,7 +4810,13 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } timing.Mark("shader_resources"); timer.phase(PhaseResources); + static const bool verifySetLayouts = std::getenv("APS5_VERIFY_SET_LAYOUTS") != nullptr; + if (verifySetLayouts && objects != nullptr) require(objects->resourcesLayout == resources->Layout(), "the resident compute pipeline's set layout differs from the dispatch's"); if (objects == nullptr) { + if (verifySetLayouts && context.descriptorCache != nullptr) { + const auto described = Graphics::DescribeSetLayouts(context, shaders); + require(described.layouts.main == resources->Layout() && !described.layouts.split, "the set layout described from the compute shader differs from the dispatch's"); + } const auto layouts = resources->SetLayouts(); objects = computePipeline(context, shader, layouts.main, layouts.mainKey, pushStages, programAddress); timing.Mark("pipeline_create"); @@ -4806,7 +4950,7 @@ RecipeOutcome VulkanDevice::DispatchRecipe(const ShaderRecompiler::RecompileResu return RecipeOutcome::Rebuild; } std::shared_ptr mapped; - const auto pipelineKey = (shader.PipelineVariantId() << 1u) | (recipe.pushes ? 1u : 0u); + const auto pipelineKey = ComputePipelineKey::Of(shader, recipe.pushes); { std::lock_guard pipelines(state->computePipelinesMutex); if (const auto found = state->computePipelines.find(pipelineKey); found != state->computePipelines.end()) mapped = found->second; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index cba5321c23..995eee4957 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -212,6 +212,7 @@ struct Context { std::uint32_t samplerTableCapacity = 0; bool geometryShader = false; bool sampleRateShading = false; + bool storageImageMultisample = false; bool shaderClipDistance = false; bool shaderCullDistance = false; bool nullDescriptors = false; @@ -232,6 +233,7 @@ struct Context { VkSampleCountFlags sampleLocationSampleCounts = 0; VkSampleCountFlags sampleLocationGridSampleCounts = 0; PipelineCacheSynchronization* pipelineCacheSynchronization = nullptr; + bool shaderTerminateInvocation = false; PipelineCacheSynchronization::PipelineUseLock LockPipelineCacheForUse() const { return pipelineCacheSynchronization != nullptr ? pipelineCacheSynchronization->LockForPipelineUse() : PipelineCacheSynchronization::PipelineUseLock{}; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp index d852d67432..0286db6e8e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp @@ -31,7 +31,7 @@ struct DrawInputCopy { DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uint64_t address, std::size_t bytes, std::size_t alignment, Recorder::SnapshotUse use); DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes); void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived); -const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment); +const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment, bool* behindGpuWrite = nullptr); struct InPlaceIndices { std::optional highest; bool reused = false; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 3d3a2c7f21..e2e43504a9 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -80,6 +80,8 @@ struct HostImportSettleCounts { std::uint64_t unmapWaits = 0; }; HostImportSettleCounts HostImportSettles(); +// Tests: how many imports settleImport has used on the settle queue. +std::uint64_t HostImportsSettled(); bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice); // Whether an existing import covers [address, address + bytes), without reconciling the imports // with the registry or making one (HostImportFor may take a registry lease): a hint for choices diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp index 237fd10ad0..c7b8e44aba 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp @@ -54,6 +54,8 @@ bool TakeMultisampledClear(MultisampledCmask& cmask, MultisampledColorFill& colo VkImageView MultisampleTargetView(const Context& context, const ColorTarget& target); std::shared_ptr MultisampleTargetTexture(const Context& context, const GuestTextureResource& resource, VkComponentMapping components); void MaterializeMultisampleTargetClear(const Context& context, const ColorTarget& target); +VkImageView MultisampleTargetStorageView(const Context& context, const GuestTextureResource& resource); +bool MultisampleTargetStorable(const Context& context, const ColorTarget& target); void ResolveMultisampleTarget(const Context& context, const ColorTarget& source, StorageTexture& destination, VkImageView destinationView); bool MultisampledColorFillAbsorbs(std::uint64_t address, std::size_t bytes, std::uint32_t pattern); void NoteColorMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32_t pattern); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp index a195283b06..6bd3a8dd82 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp @@ -94,14 +94,29 @@ private: std::vector framebuffers; }; -// The pipeline for this configuration, shared across draws and frames: keyed by the stages' variant -// ids (the rect-list tessellation pair by the vertex and fragment ids that generate it), the vertex -// input layout, the descriptor set layout key, push constant stages and every pipeline state field. -// A configuration whose stage has no variant id gets a private pipeline. Entries are LRU-bounded and -// only evicted once no recorded draw holds them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per -// draw as before. +// The pipeline for this configuration, shared across draws and frames: keyed by the stages' +// identities (the rect-list tessellation pair and the geometry stage by the vertex and fragment +// results that generate them), the vertex input layout, the descriptor set layout key, push constant +// stages and every pipeline state field. A configuration whose stage has no module gets a private +// pipeline. A miss builds the pipeline outside the store's mutex (a concurrent build of the same +// key keeps the first inserted) and records its recipe in the pipeline journal. Entries are +// LRU-bounded (APS5_PIPELINE_STORE_ENTRIES, 1024) and only evicted once no recorded draw holds +// them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per draw as before. std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); -std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); +// `created` reports whether this call built the pipeline (false: served from the store or private). +std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, bool* created = nullptr); +// Cumulative pipeline store outcomes since start-up (the [pipecache] line reports 10 s windows). +struct PipelineStoreCounters { + std::uint64_t hits = 0; + std::uint64_t misses = 0; + std::uint64_t uncached = 0; + std::uint64_t evicted = 0; + std::uint64_t cached = 0; + std::uint64_t bound = 0; +}; +PipelineStoreCounters PipelineStoreTotals(); +// The pipeline store's key of a configuration (empty when the configuration gets a private pipeline). +void BuildPipelineStoreKey(std::vector& key, const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout); struct DescribedSetLayouts { PipelineSetLayouts layouts; std::vector mainKey; @@ -126,7 +141,7 @@ bool ShadersWriteMemory(std::span shaders); bool DeclaresEarlyFragmentTests(const ShaderRecompiler::RecompileResult& program); void ValidateSampleMask(const Context& context, const State& state, std::span shaders); // Returns the color attachment locations the pixel shader writes. -std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false, bool shaderClipDistance = false, bool shaderCullDistance = false, bool viewportIndexLayer = false, bool runtimeDescriptorArray = false, bool demoteToHelperInvocation = false); +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false, bool shaderClipDistance = false, bool shaderCullDistance = false, bool viewportIndexLayer = false, bool runtimeDescriptorArray = false, bool demoteToHelperInvocation = false, bool shaderTerminateInvocation = false); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp new file mode 100644 index 0000000000..c2cf90476e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp @@ -0,0 +1,104 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINEJOURNAL_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINEJOURNAL_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +// What a graphics pipeline is built from besides the device: the stage results (named by their +// run-stable identity), the vertex input layout, the descriptor set layout keys and the state +// fields the Pipeline constructor reads, with every other State field cleared +// (CanonicalPipelineState). The pipeline store keys on exactly this, and the journal records it so +// the next run builds the same VkPipeline before the title asks for it. A generated stage (the +// rect-list control and evaluation pair, the barycentric geometry stage) is built from the vertex +// and fragment results and the device's limits; its identity is in the key like any other stage's, +// so a replayed pipeline built from another device's generated module is missed, not used. +struct GraphicsPipelineRecipe { + struct Stage { + ShaderRecompiler::ShaderStage stage; + ShaderRecompiler::PipelineIdentity identity; + std::uint32_t pushConstantOffset; + bool generated; + }; + VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + std::vector stages; + VertexInputLayout vertexInput; + std::vector mainLayoutKey; + std::vector fragmentLayoutKey; + bool splitSets = false; + State state; +}; + +[[nodiscard]] State CanonicalPipelineState(const State& state); +[[nodiscard]] bool GeneratedPipelineStage(const State& state, ShaderRecompiler::ShaderStage stage); +[[nodiscard]] GraphicsPipelineRecipe MakeGraphicsPipelineRecipe(const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout); +[[nodiscard]] std::vector EncodeGraphicsPipelineRecipe(const GraphicsPipelineRecipe& recipe); +[[nodiscard]] bool DecodeGraphicsPipelineRecipe(std::span bytes, GraphicsPipelineRecipe& recipe); + +struct PipelineJournalCounters { + std::uint64_t loadedRecipes = 0; + std::uint64_t loadedModules = 0; + std::uint64_t loadedBytes = 0; + std::uint64_t droppedRecords = 0; + double loadMs = 0; + std::uint64_t replayedGraphics = 0; + std::uint64_t replayedCompute = 0; + std::uint64_t replayPresent = 0; + std::uint64_t replaySkipped = 0; + std::uint64_t replayRejected = 0; + std::uint64_t replayFailed = 0; + double replayMs = 0; + std::uint32_t replayThreads = 0; + std::uint64_t recordedGraphics = 0; + std::uint64_t recordedCompute = 0; + std::uint64_t recordedModules = 0; + std::uint64_t writtenBytes = 0; + std::uint64_t writeFailures = 0; +}; + +// A decimal switch of the pipeline store or journal, or its default when unset; anything else throws. +[[nodiscard]] std::uint64_t PipelineSwitch(const char* name, std::uint64_t fallback); +[[nodiscard]] bool PipelineJournalEnabled(); +// True on a replay thread while it builds an entry: what it builds is already in the file. +[[nodiscard]] bool PipelineJournalReplaying(); +[[nodiscard]] std::filesystem::path PipelineJournalPath(); +[[nodiscard]] PipelineJournalCounters PipelineJournalTotals(); +void RecordGraphicsPipeline(const GraphicsPipelineRecipe& recipe, std::span shaders); +void RecordComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants); +void FlushPipelineJournal(); +void ReportPipelineJournal(); + +// Cancel stops the threads taking further entries (the pipeline each is building completes); +// Wait joins them. A device cancels its replay before it is replaced or destroyed. +class PipelineJournalReplay { +public: + virtual ~PipelineJournalReplay() = default; + virtual void Cancel() = 0; + virtual void Wait() = 0; + [[nodiscard]] virtual bool Done() const = 0; +}; + +struct PipelineJournalHooks { + std::function compute; +}; + +// The journal belongs to the device that wrote it, named by the same properties as the driver's +// pipeline cache file; a replay for another device starts the file over. +[[nodiscard]] std::uint64_t PipelineJournalDeviceFingerprint(const VkPhysicalDeviceProperties& properties); +[[nodiscard]] std::shared_ptr StartPipelineJournalReplay(const Context& context, std::uint64_t deviceFingerprint, PipelineJournalHooks hooks); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp index b2ca74e468..a40cf15479 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include #include +#include #include #include #include @@ -32,7 +33,13 @@ struct OptimizedPipeline { PFN_vkDestroyPipeline destroy = nullptr; std::atomic handle{VK_NULL_HANDLE}; std::atomic released{false}; + // The relink of a pipeline the journal replay built waits for the first draw that binds it + // (Schedule), so the relink thread serves the title's own pipelines first and a replayed + // pipeline the title never draws is never relinked. + std::atomic deferred{false}; + std::function relink; void Release() noexcept; + void Schedule(); }; VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized = nullptr, bool partsOnly = false, std::span modules = {}); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index efefda3a7f..8e25d227c9 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -530,6 +530,7 @@ private: std::vector textureBaseLevel; std::vector textureSrgb; std::vector> storageTextures; + std::vector multisampleStorageViews; std::vector storageMips; std::vector storageKeys; std::vector storageFirstLayer; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index bc6efaf720..dfe074f738 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -34,6 +34,7 @@ struct HostImport; VkFormat StorageFormatForGuest(const Context& context, std::uint32_t guestFormat); // Whether StorageFormatForGuest would succeed, without throwing (unknown guest formats included). bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat); +VkFormat HostStorageFormat(const Context& context, VkFormat format); bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, DccKeys keys); std::uint64_t SampledTextureMemory(); bool ClearKeepsDenormals(const Context& context, VkFormat format); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index b15945ee89..555cacd4e0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -1139,7 +1139,7 @@ void RetireUnmappedDepth_nid_no_patch(std::uintptr_t address, std::size_t bytes) } bool SampleLocationsCompatibleDepth(const Context& context, VkFormat format, std::uint32_t samples) { - if (samples <= 1 || (context.sampleLocationSampleCounts & samples) == 0) return false; + if (samples == 0 || (context.sampleLocationSampleCounts & samples) == 0) return false; VkImageFormatProperties properties{}; const auto result = context.imageFormatProperties(context.physical, format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT, &properties); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 7fb4593c6e..c3ad0dcbff 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -646,6 +646,7 @@ bool ValidationKey(const Context& context, std::span shade add(context.shaderClipDistance); add(context.shaderCullDistance); add(context.viewportIndexLayer); + add(context.shaderTerminateInvocation); add(state.paClVsOutCntl); add(state.stages.mesh.has_value()); if (state.stages.mesh) { @@ -658,6 +659,7 @@ bool ValidationKey(const Context& context, std::span shade add(mesh.threadsPerGroup); add(mesh.ldsSizeDwords); add(mesh.provokingVertex); + add(mesh.outputPrimitiveSize); } add(state.stages.tessellation.has_value()); if (state.stages.tessellation) { @@ -701,7 +703,7 @@ std::set CachedFragmentOutputs(const Context& context, std::span< } std::set outputs; try { - outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics, context.shaderClipDistance, context.shaderCullDistance, context.viewportIndexLayer, context.runtimeDescriptorArray, context.demoteToHelperInvocation); + outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics, context.shaderClipDistance, context.shaderCullDistance, context.viewportIndexLayer, context.runtimeDescriptorArray, context.demoteToHelperInvocation, context.shaderTerminateInvocation); } catch (const std::exception& error) { if (keyed) { std::lock_guard lock(validationMutex()); @@ -925,11 +927,18 @@ void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCop recorder->KeepDrawSnapshot(address, copy.buffer->Bytes().size(), copy.generation, copy.registryGeneration, copy.buffer, use, derived); } -const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment) { +const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment, bool* behindGpuWrite) { auto* recorder = context.recorder; if (recorder == nullptr || bytes == 0) return nullptr; - GuestMemory::FlushGpuWrites(address, bytes); - if (recorder->PendingWriteOverlaps(address, bytes) || recorder->PendingLabelIn(address, bytes) || recorder->QueuedStoreOverlaps(address, bytes)) return nullptr; + bool behind = false; + if (behindGpuWrite == nullptr) { + GuestMemory::FlushGpuWrites(address, bytes); + if (recorder->PendingWriteOverlaps(address, bytes)) return nullptr; + } else { + StorageTexture::FlushPending(address, bytes, nullptr, "in-place vertex input"); + behind = recorder->PendingWriteOverlaps(address, bytes); + } + if (recorder->PendingLabelIn(address, bytes) || recorder->QueuedStoreOverlaps(address, bytes)) return nullptr; const auto overlaps = [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }; if (context.copiedWriters != nullptr && std::any_of(context.copiedWriters->begin(), context.copiedWriters->end(), overlaps)) return nullptr; const auto drawWriters = DrawCopiedWriters(); @@ -937,6 +946,7 @@ const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address const auto* import = HostImportFor(context, address, bytes); if (import == nullptr || (address - import->base) % alignment != 0) return nullptr; recorder->NotePendingRead(address, bytes, Recorder::ReadKind::DrawInput); + if (behindGpuWrite != nullptr && behind) *behindGpuWrite = true; return import; } @@ -978,6 +988,7 @@ struct DrawInputs { std::vector vertexHandles; std::vector vertexOffsets; std::vector> inPlaceReads; + bool vertexInputBehindGpuWrite = false; std::set fragmentOutputs; VkPipelineStageFlags shaderStages = 0; std::uint32_t meshGroups = 0; @@ -1180,7 +1191,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P const auto [begin, end] = plan.copies[range]; const auto bytes = static_cast(end - begin); GuestMemory::CheckRange(reinterpret_cast(begin), bytes, 1); - if (const auto* import = InPlaceDrawInput(context, begin, bytes, plan.alignments[range])) { + if (const auto* import = InPlaceDrawInput(context, begin, bytes, plan.alignments[range], &inputs.vertexInputBehindGpuWrite)) { rangeHandles.push_back(import->buffer); rangeOffsets.push_back(begin - import->base); inputs.inPlaceReads.emplace_back(begin, end); @@ -1793,6 +1804,10 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam countBarrier(1); } APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded"); + if (inputs.vertexInputBehindGpuWrite) { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT); + countBarrier(1); + } if (meshIndirect) { meshArguments = recordMeshArguments(context, commands, recorder, true, state, draw, *record.indirect, countBarrier, inputs.restartTable); argumentBuffer = meshArguments->Handle(); @@ -2347,6 +2362,10 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& countBarrier(); } APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded"); + if (inputs.vertexInputBehindGpuWrite) { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT); + countBarrier(); + } std::shared_ptr meshArguments; if (state.stages.mesh && args != nullptr) { meshArguments = recordMeshArguments(context, commands, recorder, recorded, state, draw, indirect, countBarrier, inputs.restartTable); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 0cf9b51eea..ddc156eff0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -205,7 +205,6 @@ struct HostImports { bool unwatchImports = false; bool unwatchDmaBufImports = false; bool watchPrivateImports = false; -#ifdef _WIN32 VkDevice settleDevice = VK_NULL_HANDLE; VkBuffer settleTarget = VK_NULL_HANDLE; VkDeviceMemory settleMemory = VK_NULL_HANDLE; @@ -214,6 +213,7 @@ struct HostImports { PFN_vkDestroyFence destroyFence = nullptr; PFN_vkDestroyBuffer settleDestroyBuffer = nullptr; PFN_vkFreeMemory settleFreeMemory = nullptr; +#ifdef _WIN32 PFN_vkWaitForFences settleWaitForFences = nullptr; VkBuffer settleBatchTarget = VK_NULL_HANDLE; VkDeviceMemory settleBatchMemory = VK_NULL_HANDLE; @@ -368,6 +368,28 @@ bool releaseSettleRetired(HostImports& state, std::uint64_t begin, std::uint64_t } #endif +#ifndef _WIN32 +struct ImportFamilies { + std::array indices{}; + std::uint32_t count = 0; +}; + +// A dma-buf import's buffer is used on the main queue, on the settle queue (settleImport) and, when +// sparse, on the sparse queue. +void shareDmaBufImport(const Context& context, VkBufferCreateInfo& info, ImportFamilies& families, bool sparse) { + const auto add = [&](std::uint32_t family) { + if (std::find(families.indices.begin(), families.indices.begin() + families.count, family) == families.indices.begin() + families.count) families.indices[families.count++] = family; + }; + add(context.queueFamily); + if (context.settleQueue != VK_NULL_HANDLE) add(context.settleFamily); + if (sparse && context.sparseQueue != VK_NULL_HANDLE) add(context.sparseFamily); + const bool concurrent = families.count > 1; + info.sharingMode = concurrent ? VK_SHARING_MODE_CONCURRENT : VK_SHARING_MODE_EXCLUSIVE; + info.queueFamilyIndexCount = concurrent ? families.count : 0u; + info.pQueueFamilyIndices = concurrent ? families.indices.data() : nullptr; +} +#endif + const char* bindImport(const Context& context, HostImport& entry, VkExternalMemoryHandleTypeFlagBits handleType, const void* import, std::uint32_t importTypes, bool& allocated, VkResult& failure) { const auto failed = [&](const char* step, VkResult result) -> const char* { if (entry.buffer) context.Function("vkDestroyBuffer")(context.device, entry.buffer, nullptr); @@ -384,6 +406,10 @@ const char* bindImport(const Context& context, HostImport& entry, VkExternalMemo // INDIRECT_BUFFER: DISPATCH_INDIRECT group counts are read in place (VulkanDevice::DispatchIndirect). info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; +#ifndef _WIN32 + ImportFamilies families; + if (handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT) shareDmaBufImport(context, info, families, false); +#endif if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &entry.buffer); result != VK_SUCCESS) return failed("vkCreateBuffer", result); VkMemoryRequirements requirements{}; context.Function("vkGetBufferMemoryRequirements")(context.device, entry.buffer, &requirements); @@ -562,7 +588,8 @@ const char* createChunkedDmaBufImport(const Context& context, HostImport& entry, VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; info.size = entry.bytes; info.usage = ChunkImportUsage; - info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + ImportFamilies families; + shareDmaBufImport(context, info, families, false); VkBuffer buffer = VK_NULL_HANDLE; if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer); result != VK_SUCCESS) { failure = result; @@ -615,11 +642,8 @@ VkMemoryRequirements dmaBufBufferRequirements(const Context& context, const Host info.flags = flags; info.size = size; info.usage = usage; - const std::uint32_t families[] = {context.queueFamily, context.sparseFamily}; - const bool concurrent = (flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) != 0 && context.sparseQueue != VK_NULL_HANDLE; - info.sharingMode = concurrent ? VK_SHARING_MODE_CONCURRENT : VK_SHARING_MODE_EXCLUSIVE; - info.queueFamilyIndexCount = concurrent ? 2u : 0u; - info.pQueueFamilyIndices = concurrent ? families : nullptr; + ImportFamilies families; + shareDmaBufImport(context, info, families, (flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) != 0); VkBuffer buffer = VK_NULL_HANDLE; if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer); result != VK_SUCCESS) throwSparseImport(entry, "vkCreateBuffer", result); VkMemoryRequirements requirements{}; @@ -748,33 +772,38 @@ ImportWatchRequest importWatchRequest() { bool importsStale(const Context& context, const HostImports& state); -#ifdef _WIN32 constexpr std::uint64_t settleFloorBytes = 0x100000; +#ifdef _WIN32 constexpr std::uint64_t settleBlockBytes = 0x10000; constexpr auto settlePollPeriod = std::chrono::microseconds(100); std::atomic deferredSettles{true}; +#endif void adoptSettleDevice(const Context& context, HostImports& state) { if (state.settleDevice == context.device) return; +#ifdef _WIN32 for (auto& batch : state.settleBatches) { waitSettleBatch(state, batch); for (const auto alias : batch.aliases) state.settleDestroyBuffer(state.settleDevice, alias, nullptr); batch = SettleBatch{}; } std::erase_if(state.settles, [](const auto& settle) { return settle.second.phase == SettlePhase::Submitted; }); + state.settleBatchTarget = VK_NULL_HANDLE; + state.settleBatchMemory = VK_NULL_HANDLE; + state.settleWaitForFences = context.Function("vkWaitForFences"); +#endif state.settleDevice = context.device; state.settleTarget = VK_NULL_HANDLE; state.settleMemory = VK_NULL_HANDLE; state.settleCommands = VK_NULL_HANDLE; state.settleFence = VK_NULL_HANDLE; - state.settleBatchTarget = VK_NULL_HANDLE; - state.settleBatchMemory = VK_NULL_HANDLE; state.destroyFence = context.Function("vkDestroyFence"); state.settleDestroyBuffer = context.Function("vkDestroyBuffer"); state.settleFreeMemory = context.Function("vkFreeMemory"); - state.settleWaitForFences = context.Function("vkWaitForFences"); } +std::atomic settledImports{0}; + const char* settleImport(const Context& context, HostImports& state, const HostImport& entry, VkResult& result) { std::unique_lock queueLock(*context.settleQueueMutex); adoptSettleDevice(context, state); @@ -816,6 +845,7 @@ const char* settleImport(const Context& context, HostImports& state, const HostI VkBuffer source = entry.buffer; VkBuffer alias = VK_NULL_HANDLE; const auto run = [&]() -> const char* { +#ifdef _WIN32 if (context.settleFamily != context.queueFamily) { VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT; @@ -826,6 +856,7 @@ const char* settleImport(const Context& context, HostImports& state, const HostI if ((result = context.Function("vkBindBufferMemory")(context.device, alias, entry.memory, 0)) != VK_SUCCESS) return "vkBindBufferMemory (settle alias)"; source = alias; } +#endif if ((result = context.Function("vkResetFences")(context.device, 1, &state.settleFence)) != VK_SUCCESS) return "vkResetFences (settle)"; if ((result = context.Function("vkResetCommandBuffer")(state.settleCommands, 0)) != VK_SUCCESS) return "vkResetCommandBuffer (settle)"; VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; @@ -838,16 +869,20 @@ const char* settleImport(const Context& context, HostImports& state, const HostI submit.commandBufferCount = 1; submit.pCommandBuffers = &state.settleCommands; if ((result = context.Function("vkQueueSubmit")(context.settleQueue, 1, &submit, state.settleFence)) != VK_SUCCESS) return "vkQueueSubmit (settle)"; +#ifdef _WIN32 ++state.settlesWaited; +#endif if ((result = context.Function("vkWaitForFences")(context.device, 1, &state.settleFence, VK_TRUE, 10'000'000'000ull)) != VK_SUCCESS) return "vkWaitForFences (settle)"; return nullptr; }; const char* step = run(); if (step != nullptr) context.Function("vkQueueWaitIdle")(context.settleQueue); if (alias != VK_NULL_HANDLE) context.Function("vkDestroyBuffer")(context.device, alias, nullptr); + if (step == nullptr) settledImports.fetch_add(1, std::memory_order_relaxed); return step; } +#ifdef _WIN32 const char* prepareSettleBatch(const Context& context, HostImports& state, SettleBatch& batch, VkResult& result) { if (state.settleBatchTarget == VK_NULL_HANDLE) { VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; @@ -1474,6 +1509,18 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import return step == nullptr; }, watchedDmaBuf); } + if (step == nullptr && entry.dmaBuf && bytes >= settleFloorBytes && context.settleQueue != VK_NULL_HANDLE && context.settleQueueMutex != nullptr) { + static const bool traceSettle = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; + const auto start = std::chrono::steady_clock::now(); + if ((step = settleImport(context, state, entry, result)) != nullptr) { + destroyImport(context, entry); + entry.buffer = VK_NULL_HANDLE; + entry.memory = VK_NULL_HANDLE; + entry.chunk.reset(); + } else if (traceSettle) { + std::fprintf(stderr, "[gpu] settled the dma-buf import of 0x%llx+0x%llx on the settle queue in %.2f ms\n", static_cast(base), static_cast(bytes), std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + } + } #endif entry.unwatched = step == nullptr && !(entry.dmaBuf ? watchedDmaBuf : watchedHostPointer); if (step != nullptr) { @@ -2625,8 +2672,8 @@ void ClearHostImports(VkDevice device) { auto& state = Imports(); std::lock_guard lock(state.mutex); state.parked.erase(device); -#ifdef _WIN32 if (state.settleDevice == device) { +#ifdef _WIN32 drainSettles(state); for (auto& batch : state.settleBatches) { if (batch.fence != VK_NULL_HANDLE) state.destroyFence(device, batch.fence, nullptr); @@ -2636,6 +2683,7 @@ void ClearHostImports(VkDevice device) { if (state.settleBatchMemory != VK_NULL_HANDLE) state.settleFreeMemory(device, state.settleBatchMemory, nullptr); state.settleBatchTarget = VK_NULL_HANDLE; state.settleBatchMemory = VK_NULL_HANDLE; +#endif if (state.settleFence != VK_NULL_HANDLE) state.destroyFence(device, state.settleFence, nullptr); if (state.settleTarget != VK_NULL_HANDLE) state.settleDestroyBuffer(device, state.settleTarget, nullptr); if (state.settleMemory != VK_NULL_HANDLE) state.settleFreeMemory(device, state.settleMemory, nullptr); @@ -2645,7 +2693,6 @@ void ClearHostImports(VkDevice device) { state.settleCommands = VK_NULL_HANDLE; state.settleFence = VK_NULL_HANDLE; } -#endif if (state.device != device) return; for (const auto& [address, entry] : state.imports) { state.destroyBuffer(state.device, entry.buffer, nullptr); @@ -2915,6 +2962,10 @@ HostImportSettleCounts HostImportSettles() { #endif } +std::uint64_t HostImportsSettled() { + return settledImports.load(std::memory_order_relaxed); +} + const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; ensurePinWaiter(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp index 13093680d6..9b92e9b857 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp @@ -55,7 +55,7 @@ bool IsNullTextureResource(std::span words) { Require(words.size() == 8, "guest texture descriptor must contain 8 dwords"); const auto base40 = (static_cast(words[0]) | (static_cast(words[1]) << 32u)) & 0xffffffffffull; const auto type = (words[3] >> 28u) & 0xfu; - return base40 == 0 || type == 0; + return base40 == 0 || type <= 1; } GuestTextureResource DecodeTextureResource(std::span words) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp index 15da911cfc..65e55692aa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp @@ -58,50 +58,29 @@ public: VkFormatProperties properties{}; context.formatProperties(context.physical, target.format, &properties); Require((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) != 0, "multisampled render-target format cannot be a color attachment"); - VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; VkImageFormatProperties supported{}; sampleable = (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT) != 0 && context.imageFormatProperties(context.physical, target.format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, usage | VK_IMAGE_USAGE_SAMPLED_BIT, 0, &supported) == VK_SUCCESS && (supported.sampleCounts & samples) != 0; if (sampleable) usage |= VK_IMAGE_USAGE_SAMPLED_BIT; Check(context.imageFormatProperties(context.physical, target.format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, usage, 0, &supported), "vkGetPhysicalDeviceImageFormatProperties multisampled target"); Require((supported.sampleCounts & samples) != 0 && target.extent.width <= supported.maxExtent.width && target.extent.height <= supported.maxExtent.height, "multisampled render target exceeds device image limits"); + storageFormat = HostStorageFormat(context, target.format); + storageFlags = storageFormat != target.format ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT : 0u; + const auto storageSupports = [&](VkFormat format, VkImageUsageFlags formatUsage, VkImageCreateFlags flags) { + VkImageFormatProperties properties{}; + return context.imageFormatProperties(context.physical, format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, formatUsage, flags, &properties) == VK_SUCCESS && (properties.sampleCounts & samples) != 0; + }; + if (storageFormat != VK_FORMAT_UNDEFINED && (!context.storageImageMultisample || !storageSupports(storageFormat, VK_IMAGE_USAGE_STORAGE_BIT, 0u) || !storageSupports(target.format, usage | VK_IMAGE_USAGE_STORAGE_BIT, storageFlags))) storageFormat = VK_FORMAT_UNDEFINED; try { - VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; - info.imageType = VK_IMAGE_TYPE_2D; - info.format = target.format; - info.extent = {target.extent.width, target.extent.height, 1}; - info.mipLevels = 1; - info.arrayLayers = 1; - info.samples = samples; - info.tiling = VK_IMAGE_TILING_OPTIMAL; - info.usage = usage; - info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - Check(context.Function("vkCreateImage")(context.device, &info, nullptr, &image), "vkCreateImage multisampled target"); - VkMemoryRequirements requirements{}; - context.Function("vkGetImageMemoryRequirements")(context.device, image, &requirements); - VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; - allocation.allocationSize = requirements.size; - allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory multisampled target"); - Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory multisampled target"); - VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; - viewInfo.image = image; - viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; - viewInfo.format = target.format; - viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView multisampled target"); + const auto created = allocate(usage, 0u); + image = created.image; + memory = created.memory; + view = created.view; recordCommands(context, [&](VkCommandBuffer commands, Recorder* recorder) { - VkImageMemoryBarrier toGeneral{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; - toGeneral.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - toGeneral.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; - toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; - toGeneral.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - toGeneral.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - toGeneral.image = image; - toGeneral.subresourceRange = viewInfo.subresourceRange; - context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + recordToGeneral(commands, image); const VkClearColorValue zero{}; - context.Resolved(&DeviceFunctions::cmdClearColorImage, "vkCmdClearColorImage")(commands, image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &toGeneral.subresourceRange); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + context.Resolved(&DeviceFunctions::cmdClearColorImage, "vkCmdClearColorImage")(commands, image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range); RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); if (recorder != nullptr) Recorder::CountBarriers(Recorder::CommandClass::Draw, 2); }); @@ -114,6 +93,88 @@ public: MultisampleTarget(const MultisampleTarget&) = delete; MultisampleTarget& operator=(const MultisampleTarget&) = delete; + struct Allocation { + VkImage image = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + VkImageView view = VK_NULL_HANDLE; + }; + + Allocation allocate(VkImageUsageFlags imageUsage, VkImageCreateFlags flags) { + Allocation created; + try { + VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + info.flags = flags; + info.imageType = VK_IMAGE_TYPE_2D; + info.format = target.format; + info.extent = {target.extent.width, target.extent.height, 1}; + info.mipLevels = 1; + info.arrayLayers = 1; + info.samples = static_cast(target.samples); + info.tiling = VK_IMAGE_TILING_OPTIMAL; + info.usage = imageUsage; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + Check(context.Function("vkCreateImage")(context.device, &info, nullptr, &created.image), "vkCreateImage multisampled target"); + VkMemoryRequirements requirements{}; + context.Function("vkGetImageMemoryRequirements")(context.device, created.image, &requirements); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &created.memory), "vkAllocateMemory multisampled target"); + Check(context.Function("vkBindImageMemory")(context.device, created.image, created.memory, 0), "vkBindImageMemory multisampled target"); + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = created.image; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = target.format; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &created.view), "vkCreateImageView multisampled target"); + } catch (...) { + destroy(created); + throw; + } + return created; + } + + void destroy(const Allocation& allocation) noexcept { + if (allocation.view) context.Function("vkDestroyImageView")(context.device, allocation.view, nullptr); + if (allocation.image) context.Function("vkDestroyImage")(context.device, allocation.image, nullptr); + if (allocation.memory) context.Function("vkFreeMemory")(context.device, allocation.memory, nullptr); + } + + void recordToGeneral(VkCommandBuffer commands, VkImage destination) { + VkImageMemoryBarrier toGeneral{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toGeneral.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + toGeneral.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toGeneral.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toGeneral.image = destination; + toGeneral.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + } + + void makeStorable(Recorder& recorder) { + if ((usage & VK_IMAGE_USAGE_STORAGE_BIT) != 0) return; + const auto storable = allocate(usage | VK_IMAGE_USAGE_STORAGE_BIT, storageFlags); + const auto source = image; + recordToGeneral(recorder.Commands(), storable.image); + RecordMemoryBarrier(context, recorder.Commands(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkImageCopy region{}; + region.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.dstSubresource = region.srcSubresource; + region.extent = {target.extent.width, target.extent.height, 1}; + context.Function("vkCmdCopyImage")(recorder.Commands(), source, VK_IMAGE_LAYOUT_GENERAL, storable.image, VK_IMAGE_LAYOUT_GENERAL, 1, ®ion); + RecordMemoryBarrier(context, recorder.Commands(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + Recorder::CountBarriers(Recorder::CommandClass::Draw, 3); + for (const auto& [key, texture] : textures) recorder.Keep(texture); + textures.clear(); + retired.push_back({image, memory, view}); + image = storable.image; + memory = storable.memory; + view = storable.view; + usage |= VK_IMAGE_USAGE_STORAGE_BIT; + } + void ApplyFastClear(const ColorTarget& use) { VkClearColorValue clear{}; if (!TakeMultisampledClear(cmask, colorFill, use, clear)) return; @@ -126,11 +187,50 @@ public: }); } + void SettleStorageWrites() { + if (!storageWritten) return; + storageWritten = false; + recordCommands(context, [&](VkCommandBuffer commands, Recorder* recorder) { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + if (recorder != nullptr) Recorder::CountBarriers(Recorder::CommandClass::Draw, 1); + }); + } + + VkImageView Storage(VkFormat format) { + Require(storageFormat != VK_FORMAT_UNDEFINED, "the device cannot write the multisampled color target's format as a storage image at its sample count"); + Require(format == storageFormat, "a storage image access to a multisampled color target in another format than the target's storage form is not implemented"); + Require(cmask.Refusal().empty(), cmask.Refusal()); + Require(cmask.Keys() != DccKeys::Clear0000, "a storage image access to a multisampled color target whose CMASK fast clear no draw has applied is not implemented"); + auto* recording = Recorder::Active(); + Require(recording != nullptr, "a multisampled color target is only written by recorded dispatches"); + makeStorable(*recording); + ApplyFastClear(target); + if (storageView == VK_NULL_HANDLE) { + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = image; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = storageFormat; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &storageView), "vkCreateImageView multisampled target storage"); + } + recordCommands(context, [&](VkCommandBuffer commands, Recorder* recorder) { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + if (recorder != nullptr) Recorder::CountBarriers(Recorder::CommandClass::Draw, 1); + }); + storageWritten = true; + return storageView; + } + + bool Storable() const { + return (usage & VK_IMAGE_USAGE_STORAGE_BIT) != 0; + } + std::shared_ptr Sampled(VkComponentMapping components) { Require(sampleable, "the device cannot sample the multisampled color target's format at its sample count"); Require(cmask.Refusal().empty(), cmask.Refusal()); Require(cmask.Keys() != DccKeys::Clear0000, "sampling a multisampled color target whose CMASK fast clear no draw has applied is not implemented"); Require(Recorder::Active() != nullptr, "a multisampled color target is only sampled by recorded draws and dispatches"); + SettleStorageWrites(); ApplyFastClear(target); recordCommands(context, [&](VkCommandBuffer commands, Recorder*) { RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); @@ -185,17 +285,24 @@ public: VkRenderPass resolvePass = VK_NULL_HANDLE; MultisampledCmask cmask; MultisampledColorFill colorFill; + bool fromTexture = false; private: bool sampleable = false; + VkImageUsageFlags usage = 0; + VkFormat storageFormat = VK_FORMAT_UNDEFINED; + VkImageCreateFlags storageFlags = 0; + std::vector retired; + VkImageView storageView = VK_NULL_HANDLE; + bool storageWritten = false; std::map, std::shared_ptr> textures; void release() noexcept { textures.clear(); if (resolvePass) context.Function("vkDestroyRenderPass")(context.device, resolvePass, nullptr); - if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); - if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); - if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); + if (storageView) context.Function("vkDestroyImageView")(context.device, storageView, nullptr); + destroy({image, memory, view}); + for (const auto& allocation : retired) destroy(allocation); } }; @@ -276,7 +383,12 @@ MultisampleTarget& acquire(const Context& context, const ColorTarget& use) { for (const auto& target : targets()) { if (!target->active->load(std::memory_order_acquire)) continue; if (target->context.device != context.device || !sameTarget(target->target, use)) continue; + if (target->fromTexture && use.cmaskAddress != 0) { + target->fromTexture = false; + target->cmask = MultisampledCmask(use.cmaskAddress, use.cmaskBytes); + } target->cmask.CheckAddress(use.cmaskAddress); + target->SettleStorageWrites(); target->ApplyFastClear(use); return *target; } @@ -288,10 +400,10 @@ MultisampleTarget& acquire(const Context& context, const ColorTarget& use) { return created; } -MultisampleTarget& createFilled(const Context& context, const GuestTextureResource& resource, VkFormat format) { +MultisampleTarget& createFilled(const Context& context, const GuestTextureResource& resource, VkFormat format, const char* access) { const auto refuse = [&](const std::string& reason) { - char text[160]; - std::snprintf(text, sizeof(text), "AGC graphics: sampling the %ux%u multisampled texture 0x%llx of %u samples, which no draw rendered to and ", resource.width, resource.height, static_cast(resource.baseAddress), resource.samples); + char text[192]; + std::snprintf(text, sizeof(text), "AGC graphics: %s the %ux%u multisampled texture 0x%llx of %u samples, which no draw rendered to and ", access, resource.width, resource.height, static_cast(resource.baseAddress), resource.samples); throw std::runtime_error(text + reason + ", is not implemented"); }; ColorTarget use{}; @@ -334,6 +446,7 @@ MultisampleTarget& createFilled(const Context& context, const GuestTextureResour auto& created = *targets().back(); guard(context, use, created.active); created.colorFill.NoteFill(use.address, static_cast(bytes), value); + created.fromTexture = true; return created; } @@ -478,7 +591,7 @@ std::shared_ptr MultisampleTargetTexture(const Context& context, const } if (other == nullptr) { Require(Recorder::Active() != nullptr, "a multisampled texture is only sampled by recorded draws and dispatches"); - return createFilled(context, resource, format).Sampled(components); + return createFilled(context, resource, format, "sampling").Sampled(components); } char text[320]; std::snprintf(text, sizeof(text), "AGC graphics: sampling multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast(format)); @@ -490,6 +603,33 @@ void MaterializeMultisampleTargetClear(const Context& context, const ColorTarget acquire(context, target); } +VkImageView MultisampleTargetStorageView(const Context& context, const GuestTextureResource& resource) { + std::lock_guard lock(targetsMutex()); + const auto format = ResolveTextureFormat(resource.format); + const MultisampleTarget* other = nullptr; + for (const auto& target : targets()) { + if (!target->active->load(std::memory_order_acquire)) continue; + if (target->context.device != context.device || target->target.surfaceAddress != resource.baseAddress) continue; + const auto& rendered = target->target; + if (rendered.format == format && rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0) return target->Storage(StorageFormatForGuest(context, resource.format)); + other = target.get(); + } + if (other == nullptr) { + Require(Recorder::Active() != nullptr, "a multisampled color target is only written by recorded dispatches"); + return createFilled(context, resource, format, "writing").Storage(StorageFormatForGuest(context, resource.format)); + } + char text[320]; + std::snprintf(text, sizeof(text), "AGC graphics: storage image access to multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast(format)); + throw std::runtime_error(text); +} + +bool MultisampleTargetStorable(const Context& context, const ColorTarget& target) { + std::lock_guard lock(targetsMutex()); + return std::any_of(targets().begin(), targets().end(), [&](const auto& candidate) { + return candidate->active->load(std::memory_order_acquire) && candidate->context.device == context.device && sameTarget(candidate->target, target) && candidate->Storable(); + }); +} + void ResolveMultisampleTarget(const Context& context, const ColorTarget& source, StorageTexture& destination, VkImageView destinationView) { std::lock_guard lock(targetsMutex()); auto& target = acquire(context, source); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 9fd5790a24..d9a9c96bdc 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -2,9 +2,11 @@ #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" +#include "prx/libc/include/HostThreadLocal.hpp" #include #include #include @@ -118,7 +120,7 @@ bool ValidateSampleLocations(const Context& context, const State& state) { const bool extensionSupportsSamples = (context.sampleLocationSampleCounts & state.samples) != 0; Require(!state.customSampleLocations || extensionSupportsSamples, "custom sample locations require VK_EXT_sample_locations support for the draw's sample count"); Require(!state.sampleLocationGrid || (context.sampleLocationGridSampleCounts & state.samples) != 0, "sample locations varying between pixels require a 2x2 sample location grid for the draw's sample count"); - const bool enabled = extensionSupportsSamples && (!state.depth || SampleLocationsCompatibleDepth(context, HostDepthFormat(context, state.depth->format), state.depth->samples)); + const bool enabled = extensionSupportsSamples && (state.samples > 1 || state.customSampleLocations) && (!state.depth || SampleLocationsCompatibleDepth(context, HostDepthFormat(context, state.depth->format), state.depth->samples)); Require(!state.customSampleLocations || enabled, "custom sample locations require a compatible depth format and sample count"); return enabled; } @@ -193,10 +195,12 @@ bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || SplitsAlternateAlpha(state)) return false; keys = {}; for (const auto& shader : shaders) { - if (shader.stage == Stage::Geometry || shader.program->PipelineVariantId() == 0) return false; + const auto identity = shader.program->Identity(); + if (shader.stage == Stage::Geometry || identity.Empty()) return false; auto& key = shader.stage == Stage::Fragment ? keys.fragmentShader : keys.preRasterization; appendKey(key, shader.stage); - appendKey(key, shader.program->PipelineVariantId()); + appendKey(key, identity.low); + appendKey(key, identity.high); appendKey(key, shader.pushConstantOffset); if (resources.split) { auto& sets = shader.stage == Stage::Fragment ? keys.fragmentSets : keys.preRasterizationSets; @@ -711,6 +715,7 @@ void Pipeline::Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, V } void Pipeline::Continue(VkCommandBuffer commands, const State& state) const { + if (optimized != nullptr) optimized->Schedule(); const auto best = optimized != nullptr ? optimized->handle.load() : VK_NULL_HANDLE; context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, best != VK_NULL_HANDLE ? best : pipeline); context.Resolved(&DeviceFunctions::cmdSetViewport, "vkCmdSetViewport")(commands, 0, 1, &state.viewport); @@ -779,25 +784,27 @@ void append(std::vector& key, const TValue& value) { key.insert(key.end(), bytes.begin(), bytes.end()); } -// Everything the Pipeline objects are built from, or empty when a stage's result has no variant id -// (the recompiler could not identify it, so nothing else may share its pipeline). The rect-list -// control and evaluation stages are generated from the vertex and fragment results, which the key -// already names, so they carry no id of their own. +// Everything the Pipeline objects are built from, or empty when a stage's result has no module +// (nothing else may share its pipeline). The stage identities and the state fields are the same on +// every run (CanonicalPipelineState keeps exactly the fields read here), which lets the journal's +// replay insert an entry a later draw finds; a generated stage (rect-list pair, geometry) is keyed +// by its identity like the others, so a replayed pipeline built from another device's generated +// module is missed rather than used. void pipelineKey(std::vector& key, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout) { - using Stage = ShaderRecompiler::ShaderStage; key.clear(); append(key, context.device); append(key, attachmentLayout); append(key, shaders.size()); for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); - const bool generated = (state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation)) || shader.stage == Stage::Geometry; - if (!generated && shader.program->PipelineVariantId() == 0) { + const auto identity = shader.program->Identity(); + if (identity.Empty()) { key.clear(); return; } append(key, shader.stage); - append(key, generated ? std::uint64_t{0} : shader.program->PipelineVariantId()); + append(key, identity.low); + append(key, identity.high); // Where the stage's push constants sit in the block (AssemblePushConstants). append(key, shader.pushConstantOffset); } @@ -840,18 +847,23 @@ void pipelineKey(std::vector& key, const Context& context, const Stat append(key, state.sampleLocations); } append(key, state.colors.size()); - for (const auto& color : state.colors) append(key, color.format); + for (const auto& color : state.colors) { + append(key, color.format); + append(key, color.componentMapping); + } if (state.blends.size() != state.colors.size()) { for (const auto& color : state.colors) append(key, color.exportIndex); } append(key, state.depth.has_value()); if (state.depth) { append(key, state.depth->format); + append(key, state.depth->samples); append(key, state.depthTest); append(key, state.depthWrite); append(key, state.depthCompare); append(key, state.depthBoundsTest); append(key, state.depthBias); + append(key, state.depthBias && state.depthBiasClamp != 0.0f); append(key, state.depthBiasPerFace); append(key, state.stencilTest); auto front = state.stencilFront; @@ -872,6 +884,7 @@ void pipelineKey(std::vector& key, const Context& context, const Stat append(key, mesh.threadsPerGroup); append(key, mesh.ldsSizeDwords); append(key, mesh.provokingVertex); + append(key, mesh.outputPrimitiveSize); } append(key, state.stages.tessellation.has_value()); if (state.stages.tessellation) { @@ -903,7 +916,6 @@ struct PipelineStore { std::shared_ptr pipeline; }; std::mutex mutex; - std::vector scratchKey; // Least recently used first. std::list entries; std::unordered_map::iterator> index; @@ -911,6 +923,7 @@ struct PipelineStore { std::uint64_t misses = 0; std::uint64_t uncached = 0; std::uint64_t evicted = 0; + PipelineStoreCounters totals; std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; @@ -936,6 +949,11 @@ std::list::iterator abandon(PipelineStore& store, std::lis return store.entries.erase(it); } +std::size_t storeBound() { + static const auto bound = static_cast(std::max(1u, PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 1024u))); + return bound; +} + void reportPipelines(PipelineStore& store) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; if (!profile) return; @@ -953,57 +971,117 @@ std::shared_ptr CachedPipeline(const Context& context, const State& st return CachedPipeline(context, state, vertexInput, resources.SetLayouts(), shaders, attachmentLayout); } -std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout) { +std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout, bool* created) { static const bool disabled = std::getenv("APS5_NO_PIPELINE_CACHE") != nullptr; + if (created != nullptr) *created = false; if (disabled) return std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); auto& store = Pipelines(); - std::lock_guard lock(store.mutex); - reportPipelines(store); - auto& key = store.scratchKey; + struct PipelineKeyStorage {}; + auto& key = HostThreadLocal, PipelineKeyStorage>(); pipelineKey(key, context, state, vertexInput, resources, shaders, attachmentLayout); if (key.empty()) { + std::lock_guard lock(store.mutex); ++store.uncached; + ++store.totals.uncached; return std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); } const auto hash = hashKey(key); - if (const auto found = store.index.find(hash); found != store.index.end()) { + // The store's entry of the key, or nullptr with `found` telling a miss (the key is new) from a + // collision (another configuration owns the hash, so this one stays private). + const auto lookup = [&](bool& collision) -> std::shared_ptr { + collision = false; + const auto found = store.index.find(hash); + if (found == store.index.end()) return nullptr; const auto it = found->second; - if (it->key == key) { - // The key names the device handle, which the loader may reuse for a device created after - // this one was destroyed without ClearCachedPipelines: such an entry is a miss. - if (alive(*it, context)) { - ++store.hits; - store.entries.splice(store.entries.end(), store.entries, it); - return it->pipeline; - } - abandon(store, it); - } else { + if (it->key != key) { + collision = true; + return nullptr; + } + // The key names the device handle, which the loader may reuse for a device created after + // this one was destroyed without ClearCachedPipelines: such an entry is a miss. + if (alive(*it, context)) { + store.entries.splice(store.entries.end(), store.entries, it); + return it->pipeline; + } + abandon(store, it); + return nullptr; + }; + { + std::lock_guard lock(store.mutex); + reportPipelines(store); + bool collision = false; + if (auto pipeline = lookup(collision)) { + ++store.hits; + ++store.totals.hits; + return pipeline; + } + if (collision) { // A different configuration with the same hash keeps the resident entry; this one stays private. ++store.uncached; + ++store.totals.uncached; return std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); } + ++store.misses; + ++store.totals.misses; } - ++store.misses; - // Entries of another device belong to one the driver replaced (it does so under the GpuMutex - // before any draw reaches the new device), whose objects went with it: forget them. - for (auto it = store.entries.begin(); it != store.entries.end();) { - it = alive(*it, context) ? std::next(it) : abandon(store, it); - } + // Built outside the mutex: a draw that finds its pipeline resident is not held behind another + // thread's build (the journal replay builds here too), and a device's driver compiles in + // parallel when several threads build at once. auto pipeline = std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); - store.entries.push_back({context.device, context.bufferPool, hash, key, pipeline}); - store.index[hash] = std::prev(store.entries.end()); - constexpr std::size_t bound = 1024; - while (store.entries.size() > bound) { - // Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence). - const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; }); - if (victim == store.entries.end()) break; - store.index.erase(victim->hash); - store.entries.erase(victim); - ++store.evicted; + const auto bound = storeBound(); + const bool replaying = PipelineJournalReplaying(); + { + std::lock_guard lock(store.mutex); + bool collision = false; + if (auto raced = lookup(collision)) return raced; + if (collision) { + ++store.uncached; + ++store.totals.uncached; + return pipeline; + } + // Entries of another device belong to one the driver replaced (it does so under the GpuMutex + // before any draw reaches the new device), whose objects went with it: forget them. + for (auto it = store.entries.begin(); it != store.entries.end();) { + it = alive(*it, context) ? std::next(it) : abandon(store, it); + } + // A replayed pipeline enters at the least recently used end and never displaces a draw's: + // the title's own pipelines evict it first, and a full store takes none. + if (replaying && store.entries.size() >= bound) return pipeline; + if (replaying) { + store.entries.push_front({context.device, context.bufferPool, hash, std::vector(key), pipeline}); + store.index[hash] = store.entries.begin(); + } else { + store.entries.push_back({context.device, context.bufferPool, hash, std::vector(key), pipeline}); + store.index[hash] = std::prev(store.entries.end()); + } + while (store.entries.size() > bound) { + // Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence). + const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; }); + if (victim == store.entries.end()) break; + store.index.erase(victim->hash); + store.entries.erase(victim); + ++store.evicted; + ++store.totals.evicted; + } } + if (created != nullptr) *created = true; + if (PipelineJournalEnabled() && !PipelineJournalReplaying()) RecordGraphicsPipeline(MakeGraphicsPipelineRecipe(state, vertexInput, resources, shaders, attachmentLayout), shaders); return pipeline; } +void BuildPipelineStoreKey(std::vector& key, const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout) { + pipelineKey(key, context, state, vertexInput, resources, shaders, attachmentLayout); +} + +PipelineStoreCounters PipelineStoreTotals() { + auto& store = Pipelines(); + std::lock_guard lock(store.mutex); + auto totals = store.totals; + totals.cached = store.entries.size(); + totals.bound = storeBound(); + return totals; +} + void ClearCachedPipelines(VkDevice device) { auto& store = Pipelines(); std::lock_guard lock(store.mutex); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp new file mode 100644 index 0000000000..77d4566132 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp @@ -0,0 +1,1141 @@ +#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" +#include "ShaderCacheDirectory.hpp" +#include "ShaderDiskCache.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#include +#include +#endif + +namespace AgcDriver::Graphics { + +namespace { + +static_assert(sizeof(VkPipelineColorBlendAttachmentState) == 32, "VkPipelineColorBlendAttachmentState changed: update the recipe codec"); +static_assert(sizeof(VkStencilOpState) == 28, "VkStencilOpState changed: update the recipe codec"); +static_assert(sizeof(VkVertexInputBindingDescription) == 12, "VkVertexInputBindingDescription changed: update the recipe codec"); +static_assert(sizeof(VkVertexInputAttributeDescription) == 16, "VkVertexInputAttributeDescription changed: update the recipe codec"); +#if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__) +static_assert(sizeof(State) == 752, "State changed: update CanonicalPipelineState and the recipe codec"); +static_assert(sizeof(ShaderStages) == 112, "ShaderStages changed: update the recipe codec"); +static_assert(sizeof(ShaderRecompiler::MeshConfiguration) == 40, "MeshConfiguration changed: update the recipe codec"); +static_assert(sizeof(ShaderRecompiler::TessellationConfiguration) == 40, "TessellationConfiguration changed: update the recipe codec"); +#endif + +constexpr std::uint32_t FileMagic = 0x4c4a5041u; +// Bumped when the recipe codec, CanonicalPipelineState or the store key change in a way the +// layout guards above do not catch (a field renumbered or reinterpreted at the same size). +constexpr std::uint32_t FileFormat = 3u; +constexpr std::uint32_t ModuleRecord = 1u; +constexpr std::uint32_t GraphicsRecord = 2u; +constexpr std::uint32_t ComputeRecord = 3u; +constexpr std::uint64_t RecordAlignment = 8u; + +struct FileHeader { + std::uint32_t magic; + std::uint32_t format; + std::uint32_t runtimeAbi; + std::uint32_t reserved; + std::uint64_t sourceVersion; + std::uint64_t device; +}; +static_assert(sizeof(FileHeader) == 32 && std::is_trivially_copyable_v); + +struct RecordHeader { + std::uint32_t kind; + std::uint32_t reserved; + std::uint64_t payloadBytes; + std::uint64_t payloadHash; +}; +static_assert(sizeof(RecordHeader) == 24 && std::is_trivially_copyable_v); + +class Writer { +public: + explicit Writer(std::vector& out) : out(out) {} + + template + void Raw(const TValue& value) requires std::is_trivially_copyable_v { + const auto offset = out.size(); + out.resize(offset + sizeof(value)); + std::memcpy(out.data() + offset, &value, sizeof(value)); + } + + void Bool(bool value) { Raw(value ? 1u : 0u); } + void U32(std::uint32_t value) { Raw(value); } + void U64(std::uint64_t value) { Raw(value); } + + template + void Raws(std::span values) requires std::is_trivially_copyable_v { + U64(values.size()); + for (const auto& value : values) Raw(value); + } + +private: + std::vector& out; +}; + +class Reader { +public: + explicit Reader(std::span bytes) : bytes(bytes) {} + + [[nodiscard]] bool Ok() const { return ok; } + [[nodiscard]] bool Done() const { return ok && position == bytes.size(); } + void Fail() { ok = false; } + + template + [[nodiscard]] TValue Raw() requires std::is_trivially_copyable_v { + TValue value{}; + if (!ok || bytes.size() - position < sizeof(value)) { + ok = false; + return value; + } + std::memcpy(&value, bytes.data() + position, sizeof(value)); + position += sizeof(value); + return value; + } + + [[nodiscard]] bool Bool() { + const auto value = Raw(); + if (value > 1u) ok = false; + return ok && value == 1u; + } + + [[nodiscard]] std::uint32_t U32() { return Raw(); } + [[nodiscard]] std::uint64_t U64() { return Raw(); } + + template + void Raws(std::vector& values, std::size_t limit) requires std::is_trivially_copyable_v { + const auto count = U64(); + if (!ok || count > limit || count > (bytes.size() - position) / sizeof(TValue)) { + ok = false; + values.clear(); + return; + } + values.resize(static_cast(count)); + for (auto& value : values) value = Raw(); + } + +private: + std::span bytes; + std::size_t position = 0; + bool ok = true; +}; + +template +void encodeOptional(Writer& writer, const std::optional& value, void (*encode)(Writer&, const TValue&)) { + writer.Bool(value.has_value()); + if (value) encode(writer, *value); +} + +void encodeMesh(Writer& writer, const ShaderRecompiler::MeshConfiguration& mesh) { + for (const auto value : {mesh.inputPrimitive, mesh.primitivesPerGroup, mesh.verticesPerGroup, mesh.maxVertices, mesh.maxPrimitives, mesh.threadsPerGroup, mesh.ldsSizeDwords, mesh.provokingVertex, mesh.esgsItemSize, mesh.outputPrimitiveSize}) writer.U32(value); +} + +void decodeMesh(Reader& reader, ShaderRecompiler::MeshConfiguration& mesh) { + for (auto* value : {&mesh.inputPrimitive, &mesh.primitivesPerGroup, &mesh.verticesPerGroup, &mesh.maxVertices, &mesh.maxPrimitives, &mesh.threadsPerGroup, &mesh.ldsSizeDwords, &mesh.provokingVertex, &mesh.esgsItemSize, &mesh.outputPrimitiveSize}) *value = reader.U32(); +} + +void encodeTessellation(Writer& writer, const ShaderRecompiler::TessellationConfiguration& tessellation) { + for (const auto value : {tessellation.inputControlPoints, tessellation.outputControlPoints, tessellation.domain, tessellation.partitioning, tessellation.outputTopology, tessellation.patchesPerGroup, tessellation.primitiveGroupSize}) writer.U32(value); + writer.Bool(tessellation.breakAtInstance); + writer.U32(tessellation.instancePatches); + writer.Bool(tessellation.drawParameters); +} + +void decodeTessellation(Reader& reader, ShaderRecompiler::TessellationConfiguration& tessellation) { + for (auto* value : {&tessellation.inputControlPoints, &tessellation.outputControlPoints, &tessellation.domain, &tessellation.partitioning, &tessellation.outputTopology, &tessellation.patchesPerGroup, &tessellation.primitiveGroupSize}) *value = reader.U32(); + tessellation.breakAtInstance = reader.Bool(); + tessellation.instancePatches = reader.U32(); + tessellation.drawParameters = reader.Bool(); +} + +void encodeState(Writer& writer, const State& state) { + writer.U32(static_cast(state.stages.path)); + writer.U32(state.stages.vertexWaveSize); + writer.U32(state.stages.fragmentWaveSize); + encodeOptional(writer, state.stages.mesh, encodeMesh); + encodeOptional(writer, state.stages.tessellation, encodeTessellation); + writer.Bool(state.depth.has_value()); + if (state.depth) { + writer.U32(static_cast(state.depth->format)); + writer.U32(state.depth->samples); + } + writer.Bool(state.depthTest); + writer.Bool(state.depthWrite); + writer.U32(static_cast(state.depthCompare)); + writer.Bool(state.depthBoundsTest); + writer.Bool(state.depthBias); + writer.Raw(state.depthBiasClamp); + writer.Bool(state.depthBiasPerFace); + writer.Bool(state.stencilTest); + writer.Raw(state.stencilFront); + writer.Raw(state.stencilBack); + writer.Bool(state.stencilReplayReference.has_value()); + writer.U64(state.colors.size()); + for (const auto& color : state.colors) { + writer.U32(static_cast(color.format)); + writer.U32(color.exportIndex); + writer.Raw(color.componentMapping); + } + writer.Raws(std::span(state.blends)); + for (const auto value : state.blendConstants) writer.Raw(value); + writer.Bool(state.hasColorTarget); + writer.Bool(state.rectList); + writer.Bool(state.dualSourceBlend); + writer.U32(state.reversedBlend); + writer.U32(static_cast(state.topology)); + writer.Bool(state.primitiveRestart); + writer.Bool(state.negativeOneToOne); + writer.Bool(state.depthClamp); + writer.U32(static_cast(state.conservativeRasterization)); + writer.U32(state.cullMode); + writer.U32(static_cast(state.frontFace)); + writer.U32(static_cast(state.provokingVertexMode)); + writer.U32(state.samples); + for (const auto value : state.sampleLocations) writer.Raw(value); + writer.Bool(state.customSampleLocations); + writer.Bool(state.sampleLocationGrid); + writer.Bool(state.perSampleShading); + for (const auto mask : state.quadSampleMask) writer.Raw(mask); +} + +void decodeState(Reader& reader, State& state) { + state = State{}; + state.stages.path = static_cast(reader.U32()); + state.stages.vertexWaveSize = reader.U32(); + state.stages.fragmentWaveSize = reader.U32(); + if (reader.Bool()) decodeMesh(reader, state.stages.mesh.emplace()); + if (reader.Bool()) decodeTessellation(reader, state.stages.tessellation.emplace()); + if (reader.Bool()) { + auto& depth = state.depth.emplace(); + depth.format = static_cast(reader.U32()); + depth.samples = reader.U32(); + } + state.depthTest = reader.Bool(); + state.depthWrite = reader.Bool(); + state.depthCompare = static_cast(reader.U32()); + state.depthBoundsTest = reader.Bool(); + state.depthBias = reader.Bool(); + state.depthBiasClamp = reader.Raw(); + state.depthBiasPerFace = reader.Bool(); + state.stencilTest = reader.Bool(); + state.stencilFront = reader.Raw(); + state.stencilBack = reader.Raw(); + if (reader.Bool()) state.stencilReplayReference = 0u; + const auto colors = reader.U64(); + if (!reader.Ok() || colors > 8u) { + reader.Fail(); + state = State{}; + return; + } + for (std::uint64_t index = 0; index < colors; ++index) { + ColorTarget color{}; + color.format = static_cast(reader.U32()); + color.exportIndex = reader.U32(); + color.componentMapping = reader.Raw(); + state.colors.push_back(color); + } + reader.Raws(state.blends, 8u); + for (auto& value : state.blendConstants) value = reader.Raw(); + state.hasColorTarget = reader.Bool(); + state.rectList = reader.Bool(); + state.dualSourceBlend = reader.Bool(); + state.reversedBlend = reader.U32(); + state.topology = static_cast(reader.U32()); + state.primitiveRestart = reader.Bool(); + state.negativeOneToOne = reader.Bool(); + state.depthClamp = reader.Bool(); + state.conservativeRasterization = static_cast(reader.U32()); + state.cullMode = reader.U32(); + state.frontFace = static_cast(reader.U32()); + state.provokingVertexMode = static_cast(reader.U32()); + state.samples = reader.U32(); + for (auto& value : state.sampleLocations) value = reader.Raw(); + state.customSampleLocations = reader.Bool(); + state.sampleLocationGrid = reader.Bool(); + state.perSampleShading = reader.Bool(); + for (auto& mask : state.quadSampleMask) mask = reader.Raw(); +} + +void encodeRecipe(Writer& writer, const GraphicsPipelineRecipe& recipe) { + writer.U32(static_cast(recipe.attachmentLayout)); + writer.U64(recipe.stages.size()); + for (const auto& stage : recipe.stages) { + writer.U32(static_cast(stage.stage)); + writer.U64(stage.identity.low); + writer.U64(stage.identity.high); + writer.U32(stage.pushConstantOffset); + writer.Bool(stage.generated); + } + writer.Raws(std::span(recipe.vertexInput.bindings)); + writer.Raws(std::span(recipe.vertexInput.attributes)); + writer.Raws(std::span(recipe.mainLayoutKey)); + writer.Raws(std::span(recipe.fragmentLayoutKey)); + writer.Bool(recipe.splitSets); + encodeState(writer, recipe.state); +} + +bool decodeRecipe(Reader& reader, GraphicsPipelineRecipe& recipe) { + recipe = GraphicsPipelineRecipe{}; + recipe.attachmentLayout = static_cast(reader.U32()); + const auto stages = reader.U64(); + if (!reader.Ok() || stages > 8u) return false; + for (std::uint64_t index = 0; index < stages; ++index) { + GraphicsPipelineRecipe::Stage stage{}; + stage.stage = static_cast(reader.U32()); + stage.identity.low = reader.U64(); + stage.identity.high = reader.U64(); + stage.pushConstantOffset = reader.U32(); + stage.generated = reader.Bool(); + recipe.stages.push_back(stage); + } + reader.Raws(recipe.vertexInput.bindings, 64u); + reader.Raws(recipe.vertexInput.attributes, 64u); + reader.Raws(recipe.mainLayoutKey, 4096u); + reader.Raws(recipe.fragmentLayoutKey, 4096u); + recipe.splitSets = reader.Bool(); + decodeState(reader, recipe.state); + return reader.Done(); +} + +std::uint64_t padded(std::uint64_t bytes) { + return (bytes + RecordAlignment - 1u) / RecordAlignment * RecordAlignment; +} + +bool profiling() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + +thread_local bool replayingHere = false; + +std::uint64_t configured(const char* name, std::uint64_t fallback) { + const char* text = std::getenv(name); + if (text == nullptr || *text == '\0') return fallback; + char* end = nullptr; + const auto value = std::strtoull(text, &end, 10); + if (end == text || *end != '\0') throw std::runtime_error(std::string("AGC graphics: ") + name + " must be a decimal integer"); + return value; +} + +// Held while the journal is read, cut or appended: another process sharing the directory (a +// second run of the same install) takes the same lock file, so its appends and this run's never +// interleave and neither cuts a record the other is writing. Taken before the journal's mutex, +// never while holding it. +class FileLock { +public: + explicit FileLock(const std::filesystem::path& path) { +#ifdef _WIN32 + handle = CreateFileW(path.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + if (handle == INVALID_HANDLE_VALUE) return; + OVERLAPPED overlapped{}; + if (!LockFileEx(handle, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &overlapped)) { + CloseHandle(handle); + handle = INVALID_HANDLE_VALUE; + } +#else + descriptor = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0644); + if (descriptor >= 0 && ::flock(descriptor, LOCK_EX) != 0) { + ::close(descriptor); + descriptor = -1; + } +#endif + } + + ~FileLock() { +#ifdef _WIN32 + if (handle == INVALID_HANDLE_VALUE) return; + OVERLAPPED overlapped{}; + UnlockFileEx(handle, 0, 1, 0, &overlapped); + CloseHandle(handle); +#else + if (descriptor < 0) return; + ::flock(descriptor, LOCK_UN); + ::close(descriptor); +#endif + } + + FileLock(const FileLock&) = delete; + FileLock& operator=(const FileLock&) = delete; + +#ifdef _WIN32 + bool Held() const { return handle != INVALID_HANDLE_VALUE; } +#else + bool Held() const { return descriptor >= 0; } +#endif + +private: +#ifdef _WIN32 + HANDLE handle = INVALID_HANDLE_VALUE; +#else + int descriptor = -1; +#endif +}; + +// The end of the last whole record in [from, size) of the file: records another process appended +// after this one loaded the file are kept, the torn tail of a record a crashed run left is cut. +std::uint64_t validTail(const std::filesystem::path& path, std::uint64_t from, std::uint64_t size); + +struct LoadedModule { + ShaderRecompiler::ShaderStage stage; + std::shared_ptr result; +}; + +struct LoadedEntry { + std::uint32_t kind; + GraphicsPipelineRecipe graphics; + ShaderRecompiler::PipelineIdentity compute; + bool pushConstants = false; +}; + +struct LoadedJournal { + std::map modules; + std::vector entries; + std::set indexModules; + std::set indexRecords; + std::uint64_t validBytes = 0; + std::uint64_t bytes = 0; + std::uint64_t dropped = 0; + bool headerValid = false; +}; + +class Journal { +public: + static Journal& Instance() { + static auto* journal = Create(); + return *journal; + } + + static Journal* Existing() { return created().load(std::memory_order_acquire); } + + [[nodiscard]] bool Enabled() const { return !path.empty() && !mismatched.load(std::memory_order_relaxed); } + [[nodiscard]] const std::filesystem::path& Path() const { return path; } + + // The journal belongs to one device per process: the first device to replay binds its + // fingerprint, the file header names it and a file written by another device starts over. A + // later device with another fingerprint (the window's device is the headless device's GPU, so a + // run does not see one) turns the journal off for the rest of the run. + bool Bind(std::uint64_t fingerprint) { + std::lock_guard lock(mutex); + if (!bound.load(std::memory_order_relaxed)) { + device.store(fingerprint, std::memory_order_relaxed); + bound.store(true, std::memory_order_release); + return true; + } + if (device.load(std::memory_order_relaxed) == fingerprint) return true; + if (!mismatched.exchange(true)) std::fprintf(stderr, "[pipeline-journal] %s belongs to another device; the journal is off for this run\n", path.string().c_str()); + return false; + } + + // Reads the whole file (modules keep views into the buffer, which lives as long as they do), + // fills the dedupe index, and remembers how many bytes parsed so a torn tail is cut before the + // next append. + LoadedJournal Load() { + LoadedJournal loaded; + if (!Enabled() || !bound.load(std::memory_order_acquire)) return loaded; + const auto started = std::chrono::steady_clock::now(); + auto file = std::make_shared>(); + bool read = false; + { + FileLock fileLock(lockPath); + read = ShaderRecompiler::ReadWholeFile(path, *file); + } + if (read) parse(*file, file, loaded, true); + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + { + std::lock_guard lock(mutex); + std::error_code error; + if (!indexed && (read || !std::filesystem::exists(path, error))) { + indexed = true; + validBytes = loaded.validBytes; + headerValid = loaded.headerValid; + writtenModules = loaded.indexModules; + writtenRecords = loaded.indexRecords; + } + counters.loadedRecipes.store(loaded.entries.size(), std::memory_order_relaxed); + counters.loadedModules.store(loaded.modules.size(), std::memory_order_relaxed); + counters.loadedBytes.store(loaded.bytes, std::memory_order_relaxed); + counters.droppedRecords.store(loaded.dropped, std::memory_order_relaxed); + loadMs.store(ms, std::memory_order_relaxed); + } + if (loaded.bytes != 0 || loaded.dropped != 0) std::fprintf(stderr, "[pipeline-journal] loaded %zu pipelines and %zu modules (%.1f MiB) from %s in %.0f ms%s\n", loaded.entries.size(), loaded.modules.size(), static_cast(loaded.bytes) / (1024.0 * 1024.0), path.string().c_str(), ms, !loaded.headerValid ? "; written by another build or device, starting over" : loaded.dropped != 0 ? "; the file's tail was unreadable and is dropped" : ""); + return loaded; + } + + void RecordGraphics(const GraphicsPipelineRecipe& recipe, std::span shaders) { + if (!Enabled() || !bound.load(std::memory_order_acquire)) return; + Pending pending; + pending.kind = GraphicsRecord; + for (std::size_t index = 0; index < shaders.size() && index < recipe.stages.size(); ++index) { + Require(shaders[index].program != nullptr, "a recorded pipeline stage has no program"); + pending.modules.push_back({recipe.stages[index].identity, recipe.stages[index].stage, std::make_shared(*shaders[index].program)}); + } + Writer writer(pending.payload); + encodeRecipe(writer, recipe); + enqueue(std::move(pending)); + } + + void RecordCompute(const ShaderRecompiler::RecompileResult& shader, bool pushConstants) { + if (!Enabled() || !bound.load(std::memory_order_acquire)) return; + const auto identity = shader.Identity(); + if (identity.Empty()) return; + Pending pending; + pending.kind = ComputeRecord; + pending.modules.push_back({identity, ShaderRecompiler::ShaderStage::Compute, std::make_shared(shader)}); + Writer writer(pending.payload); + writer.U64(identity.low); + writer.U64(identity.high); + writer.Bool(pushConstants); + enqueue(std::move(pending)); + } + + void Flush() { + std::unique_lock lock(mutex); + idle.wait(lock, [&] { return queue.empty() && !writing; }); + } + + PipelineJournalCounters Totals() const { + PipelineJournalCounters totals; + totals.loadedRecipes = counters.loadedRecipes.load(std::memory_order_relaxed); + totals.loadedModules = counters.loadedModules.load(std::memory_order_relaxed); + totals.loadedBytes = counters.loadedBytes.load(std::memory_order_relaxed); + totals.droppedRecords = counters.droppedRecords.load(std::memory_order_relaxed); + totals.loadMs = loadMs.load(std::memory_order_relaxed); + totals.recordedGraphics = counters.recordedGraphics.load(std::memory_order_relaxed); + totals.recordedCompute = counters.recordedCompute.load(std::memory_order_relaxed); + totals.recordedModules = counters.recordedModules.load(std::memory_order_relaxed); + totals.writtenBytes = counters.writtenBytes.load(std::memory_order_relaxed); + totals.writeFailures = counters.writeFailures.load(std::memory_order_relaxed); + return totals; + } + + struct Counters { + std::atomic loadedRecipes{0}, loadedModules{0}, loadedBytes{0}, droppedRecords{0}; + std::atomic recordedGraphics{0}, recordedCompute{0}, recordedModules{0}, writtenBytes{0}, writeFailures{0}; + }; + +private: + struct PendingModule { + ShaderRecompiler::PipelineIdentity identity; + ShaderRecompiler::ShaderStage stage; + std::shared_ptr result; + }; + + struct Pending { + std::uint32_t kind = 0; + std::vector modules; + std::vector payload; + }; + + static std::atomic& created() { + static std::atomic pointer{nullptr}; + return pointer; + } + + static Journal* Create() { + auto* journal = new Journal(); + created().store(journal, std::memory_order_release); + return journal; + } + + Journal() { + const char* off = std::getenv("APS5_NO_PIPELINE_JOURNAL"); + if ((off != nullptr && std::strcmp(off, "0") != 0) || !ShaderRecompiler::ShaderDiskCache::Enabled()) return; + path = ShaderRecompiler::ShaderDiskCache::EntryDirectory() / "pipelines.journal"; + lockPath = ShaderRecompiler::ShaderDiskCache::EntryDirectory() / "pipelines.journal.lock"; + maxBytes = configured("APS5_PIPELINE_JOURNAL_MAX_MIB", 1024u) << 20u; + std::thread([this] { run(); }).detach(); + } + + // Parses the file: with `loaded` set every record is decoded, otherwise only the index (the + // module identities and record hashes) is built. Stops at the first record that does not + // parse; the bytes before it stay valid. + void parse(const std::vector& file, const std::shared_ptr>& owner, LoadedJournal& loaded, bool decode) { + std::uint64_t valid = 0; + bool header = false; + std::uint64_t dropped = 0; + const auto bytes = std::span(file); + if (bytes.size() >= sizeof(FileHeader)) { + FileHeader fileHeader; + std::memcpy(&fileHeader, bytes.data(), sizeof(fileHeader)); + header = fileHeader.magic == FileMagic && fileHeader.format == FileFormat && fileHeader.runtimeAbi == ShaderRecompiler::RuntimeAbi::Version && fileHeader.sourceVersion == ShaderRecompiler::ShaderDiskCache::SourceVersion() && fileHeader.device == device.load(std::memory_order_relaxed); + } + if (header) { + valid = padded(sizeof(FileHeader)); + std::uint64_t cursor = valid; + while (cursor + sizeof(RecordHeader) <= bytes.size()) { + RecordHeader record; + std::memcpy(&record, bytes.data() + cursor, sizeof(record)); + const auto payloadOffset = cursor + sizeof(RecordHeader); + if (record.payloadBytes > bytes.size() - payloadOffset) break; + const auto end = padded(payloadOffset + record.payloadBytes); + if (end > bytes.size()) break; + const auto payload = bytes.subspan(static_cast(payloadOffset), static_cast(record.payloadBytes)); + if (ShaderRecompiler::HashBytes(payload) != record.payloadHash) break; + if (record.kind == ModuleRecord) { + if (payload.size() < 20u) break; + ShaderRecompiler::PipelineIdentity identity; + std::memcpy(&identity.low, payload.data(), sizeof(identity.low)); + std::memcpy(&identity.high, payload.data() + 8, sizeof(identity.high)); + std::uint32_t stage = 0; + std::memcpy(&stage, payload.data() + 16, sizeof(stage)); + bool decoded = !decode; + if (decode && !loaded.indexModules.contains(identity)) { + auto result = std::make_shared(); + decoded = ShaderRecompiler::ShaderDiskCache::DecodeResult(payload.subspan(20), *result, owner) && result->Identity() == identity; + if (decoded) loaded.modules.emplace(identity, LoadedModule{static_cast(stage), std::move(result)}); + else ++dropped; + } + if (decoded) loaded.indexModules.insert(identity); + } else if (record.kind == GraphicsRecord || record.kind == ComputeRecord) { + if (decode && !loaded.indexRecords.contains(record.payloadHash)) { + LoadedEntry entry; + entry.kind = record.kind; + bool decoded = false; + if (record.kind == GraphicsRecord) { + Reader reader(payload); + decoded = decodeRecipe(reader, entry.graphics); + } else { + Reader reader(payload); + entry.compute.low = reader.U64(); + entry.compute.high = reader.U64(); + entry.pushConstants = reader.Bool(); + decoded = reader.Done(); + } + if (decoded) loaded.entries.push_back(std::move(entry)); + else ++dropped; + } + loaded.indexRecords.insert(record.payloadHash); + } else { + break; + } + valid = end; + cursor = end; + } + } + if (valid < bytes.size()) ++dropped; + loaded.validBytes = valid; + loaded.bytes = bytes.size(); + loaded.dropped = dropped; + loaded.headerValid = header; + } + + // The index the writer dedupes against, replaced from the file: what it holds and where it + // ends. Under the file lock. False when the file exists but could not be read, so nothing is + // written until it can be (the next record tries again) and the file is never started over for + // a read failure. + bool indexLocked() { + std::vector file; + LoadedJournal index; + std::error_code error; + const bool exists = std::filesystem::exists(path, error); + if (exists && !ShaderRecompiler::ReadWholeFile(path, file)) return false; + if (exists) parse(file, nullptr, index, false); + std::lock_guard lock(mutex); + indexed = true; + validBytes = index.validBytes; + headerValid = index.headerValid; + writtenModules = std::move(index.indexModules); + writtenRecords = std::move(index.indexRecords); + return true; + } + + bool ensureIndexed() { + { + std::lock_guard lock(mutex); + if (indexed) return true; + } + FileLock fileLock(lockPath); + { + std::lock_guard lock(mutex); + if (indexed) return true; + } + return indexLocked(); + } + + void enqueue(Pending pending) { + { + std::lock_guard lock(mutex); + queue.push_back(std::move(pending)); + } + wake.notify_one(); + } + + void run() { + for (;;) { + Pending pending; + { + std::unique_lock lock(mutex); + wake.wait(lock, [&] { return !queue.empty(); }); + pending = std::move(queue.front()); + queue.pop_front(); + writing = true; + } + try { + write(pending); + } catch (const std::exception& error) { + counters.writeFailures.fetch_add(1, std::memory_order_relaxed); + static std::atomic reported{0}; + if (reported.fetch_add(1) < 4) std::fprintf(stderr, "[pipeline-journal] cannot write %s: %s\n", path.string().c_str(), error.what()); + } + { + std::lock_guard lock(mutex); + writing = false; + if (queue.empty()) idle.notify_all(); + } + } + } + + // One record per module the file lacks, then the pipeline record; a record whose payload the + // file already holds is skipped. The file is opened for appending per batch, after the torn + // tail of a crashed run (bytes past the last parsed record) was cut. + void write(const Pending& pending) { + if (!ensureIndexed()) throw std::runtime_error("the journal could not be read"); + for (unsigned attempt = 0; attempt < 2; ++attempt) { + if (append(pending)) return; + } + throw std::runtime_error("the journal keeps changing under this run"); + } + + // One attempt: false when the file was cut or replaced by someone else since it was indexed + // (it is indexed again, and the batch is rebuilt against that index). + bool append(const Pending& pending) { + const auto recordHash = ShaderRecompiler::HashBytes(pending.payload); + std::vector batch; + std::vector newModules; + { + std::lock_guard lock(mutex); + if (writtenRecords.contains(recordHash)) return true; + for (const auto& module : pending.modules) { + if (!writtenModules.contains(module.identity) && std::ranges::find(newModules, module.identity) == newModules.end()) newModules.push_back(module.identity); + } + } + const auto appendRecord = [&](std::uint32_t kind, std::span payload) { + const RecordHeader header{kind, 0u, payload.size(), ShaderRecompiler::HashBytes(payload)}; + Writer writer(batch); + writer.Raw(header); + batch.insert(batch.end(), payload.begin(), payload.end()); + batch.resize(static_cast(padded(batch.size()))); + }; + for (const auto& identity : newModules) { + const auto module = std::ranges::find_if(pending.modules, [&](const PendingModule& candidate) { return candidate.identity == identity; }); + std::vector payload; + Writer writer(payload); + writer.U64(identity.low); + writer.U64(identity.high); + writer.U32(static_cast(module->stage)); + ShaderRecompiler::ShaderDiskCache::EncodeResult(*module->result, payload); + appendRecord(ModuleRecord, payload); + } + appendRecord(pending.kind, pending.payload); + std::error_code error; + std::filesystem::create_directories(path.parent_path(), error); + FileLock fileLock(lockPath); + if (!fileLock.Held()) { + static std::atomic reported{false}; + if (!reported.exchange(true)) std::fprintf(stderr, "[pipeline-journal] cannot lock %s; pipelines are not recorded while it cannot be locked\n", lockPath.string().c_str()); + counters.writeFailures.fetch_add(1, std::memory_order_relaxed); + return true; + } + bool fresh = false; + { + std::lock_guard lock(mutex); + fresh = !headerValid; + if (!fresh) { + const auto size = std::filesystem::file_size(path, error); + if (error) fresh = true; + else if (size > validBytes) { + const auto tail = validTail(path, validBytes, size); + if (tail < size) std::filesystem::resize_file(path, tail, error); + if (error) fresh = true; + else validBytes = tail; + } + } + if (!fresh && validBytes + batch.size() > maxBytes) { + static std::atomic reported{false}; + if (!reported.exchange(true)) std::fprintf(stderr, "[pipeline-journal] %s reached APS5_PIPELINE_JOURNAL_MAX_MIB; later pipelines are not recorded\n", path.string().c_str()); + counters.writeFailures.fetch_add(1, std::memory_order_relaxed); + return true; + } + } + if (!fresh) { + const auto size = std::filesystem::file_size(path, error); + if (!error && size < validBytes) { + if (!indexLocked()) throw std::runtime_error("the journal could not be read"); + return false; + } + } + std::vector prefix; + if (fresh) { + const FileHeader header{FileMagic, FileFormat, ShaderRecompiler::RuntimeAbi::Version, 0u, ShaderRecompiler::ShaderDiskCache::SourceVersion(), device.load(std::memory_order_relaxed)}; + Writer writer(prefix); + writer.Raw(header); + prefix.resize(static_cast(padded(prefix.size()))); + } + std::ofstream file(path, fresh ? std::ios::binary | std::ios::trunc : std::ios::binary | std::ios::app); + if (!file) throw std::runtime_error("cannot open the journal for writing"); + if (!prefix.empty()) file.write(reinterpret_cast(prefix.data()), static_cast(prefix.size())); + file.write(reinterpret_cast(batch.data()), static_cast(batch.size())); + file.flush(); + if (file.fail()) throw std::runtime_error("the journal write failed"); + std::lock_guard lock(mutex); + headerValid = true; + validBytes = (fresh ? padded(sizeof(FileHeader)) : validBytes) + batch.size(); + for (const auto& identity : newModules) writtenModules.insert(identity); + writtenRecords.insert(recordHash); + counters.recordedModules.fetch_add(newModules.size(), std::memory_order_relaxed); + if (pending.kind == GraphicsRecord) counters.recordedGraphics.fetch_add(1, std::memory_order_relaxed); + else counters.recordedCompute.fetch_add(1, std::memory_order_relaxed); + counters.writtenBytes.fetch_add(prefix.size() + batch.size(), std::memory_order_relaxed); + return true; + } + + std::filesystem::path path; + std::filesystem::path lockPath; + std::uint64_t maxBytes = 0; + std::atomic device{0}; + std::atomic bound{false}; + std::atomic mismatched{false}; + std::mutex mutex; + std::condition_variable wake; + std::condition_variable idle; + std::deque queue; + bool writing = false; + bool indexed = false; + bool headerValid = false; + std::uint64_t validBytes = 0; + std::set writtenModules; + std::set writtenRecords; + Counters counters; + std::atomic loadMs{0}; +}; + +std::uint64_t validTail(const std::filesystem::path& path, std::uint64_t from, std::uint64_t size) { + if (size <= from) return from; + std::ifstream file(path, std::ios::binary); + if (!file) return from; + file.seekg(static_cast(from)); + std::vector bytes(static_cast(size - from)); + file.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + if (static_cast(file.gcount()) != bytes.size()) return from; + const auto span = std::span(bytes); + std::uint64_t cursor = 0; + std::uint64_t valid = 0; + while (cursor + sizeof(RecordHeader) <= span.size()) { + RecordHeader record; + std::memcpy(&record, span.data() + cursor, sizeof(record)); + const auto payloadOffset = cursor + sizeof(RecordHeader); + if (record.kind == 0 || record.kind > ComputeRecord || record.payloadBytes > span.size() - payloadOffset) break; + const auto end = padded(payloadOffset + record.payloadBytes); + if (end > span.size()) break; + if (ShaderRecompiler::HashBytes(span.subspan(static_cast(payloadOffset), static_cast(record.payloadBytes))) != record.payloadHash) break; + valid = end; + cursor = end; + } + return from + valid; +} + +struct ReplayCounters { + std::atomic graphics{0}, compute{0}, present{0}, skipped{0}, rejected{0}, failed{0}; + std::atomic threads{0}; + std::atomic ms{0}; +}; + +ReplayCounters& replayCounters() { + static ReplayCounters counters; + return counters; +} + +// Builds the journal's pipelines for one device on a pool: the first thread loads the file, every +// thread then takes the next entry. A pipeline the store already holds counts as present; an entry +// whose modules are missing or whose set layouts differ from what its modules describe is rejected. +class Replay : public PipelineJournalReplay { +public: + // The context keeps the device's buffer pool: a store entry is alive for a draw only while its + // pool is the draw's (the pool tells device instances apart), so the entries built here must + // name it. The pool is released with the replay, before the device's teardown resets it. + Replay(Context context, PipelineJournalHooks hooks, unsigned threads) : context(std::move(context)), hooks(std::move(hooks)), active(threads) { + started = std::chrono::steady_clock::now(); + replayCounters().threads.store(threads, std::memory_order_relaxed); + workers.reserve(threads); + for (unsigned index = 0; index < threads; ++index) workers.emplace_back([this] { run(); }); + } + + ~Replay() override { + Cancel(); + Wait(); + } + + void Cancel() override { cancelled.store(true, std::memory_order_relaxed); } + + void Wait() override { + for (auto& worker : workers) { + if (worker.joinable()) worker.join(); + } + workers.clear(); + } + + [[nodiscard]] bool Done() const override { return finished.load(std::memory_order_acquire); } + +private: + void run() { + std::call_once(loadOnce, [&] { + try { + auto journal = Journal::Instance().Load(); + const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 1024u); + if (journal.entries.size() > limit) journal.entries.resize(static_cast(limit)); + loaded = std::move(journal); + } catch (const std::exception& error) { + std::fprintf(stderr, "[pipeline-journal] cannot load %s: %s\n", Journal::Instance().Path().string().c_str(), error.what()); + } + }); + for (;;) { + if (cancelled.load(std::memory_order_relaxed)) break; + const auto index = cursor.fetch_add(1, std::memory_order_relaxed); + if (index >= loaded.entries.size()) break; + try { + replay(loaded.entries[index]); + } catch (const std::exception& error) { + replayCounters().failed.fetch_add(1, std::memory_order_relaxed); + static std::atomic reported{0}; + if (reported.fetch_add(1) < 8) std::fprintf(stderr, "[pipeline-journal] replay failed: %s\n", error.what()); + } + } + if (active.fetch_sub(1, std::memory_order_acq_rel) == 1) { + loaded = LoadedJournal{}; + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + replayCounters().ms.store(ms, std::memory_order_relaxed); + finished.store(true, std::memory_order_release); + ReportPipelineJournal(); + } + } + + void replay(const LoadedEntry& entry) { + struct Replaying { + Replaying() { replayingHere = true; } + ~Replaying() { replayingHere = false; } + } replaying; + auto& counters = replayCounters(); + if (entry.kind == ComputeRecord) { + const auto module = loaded.modules.find(entry.compute); + if (module == loaded.modules.end() || module->second.stage != ShaderRecompiler::ShaderStage::Compute || !hooks.compute) { + counters.rejected.fetch_add(1, std::memory_order_relaxed); + return; + } + (hooks.compute(*module->second.result, entry.pushConstants) ? counters.compute : counters.present).fetch_add(1, std::memory_order_relaxed); + return; + } + std::vector shaders; + for (const auto& stage : entry.graphics.stages) { + const auto module = loaded.modules.find(stage.identity); + if (module == loaded.modules.end() || module->second.stage != stage.stage) { + counters.rejected.fetch_add(1, std::memory_order_relaxed); + return; + } + shaders.push_back({stage.stage, module->second.result.get(), stage.pushConstantOffset}); + } + if (shaders.empty() || context.descriptorCache == nullptr) { + counters.rejected.fetch_add(1, std::memory_order_relaxed); + return; + } + const auto store = PipelineStoreTotals(); + if (store.cached >= store.bound) { + counters.skipped.fetch_add(1, std::memory_order_relaxed); + return; + } + const auto layouts = DescribeSetLayouts(context, shaders); + if (!std::ranges::equal(layouts.mainKey, entry.graphics.mainLayoutKey) || !std::ranges::equal(layouts.fragmentKey, entry.graphics.fragmentLayoutKey) || layouts.layouts.split != entry.graphics.splitSets) { + counters.rejected.fetch_add(1, std::memory_order_relaxed); + return; + } + bool created = false; + static_cast(CachedPipeline(context, entry.graphics.state, entry.graphics.vertexInput, layouts.View(), shaders, entry.graphics.attachmentLayout, &created)); + (created ? counters.graphics : counters.present).fetch_add(1, std::memory_order_relaxed); + } + + Context context; + PipelineJournalHooks hooks; + std::vector workers; + std::once_flag loadOnce; + LoadedJournal loaded; + std::atomic cursor{0}; + std::atomic active{0}; + std::atomic finished{false}; + std::atomic cancelled{false}; + std::chrono::steady_clock::time_point started; +}; + +} + +State CanonicalPipelineState(const State& state) { + State canonical{}; + canonical.stages.path = state.stages.path; + canonical.stages.vertexWaveSize = state.stages.vertexWaveSize; + canonical.stages.fragmentWaveSize = state.stages.fragmentWaveSize; + canonical.stages.mesh = state.stages.mesh; + canonical.stages.tessellation = state.stages.tessellation; + if (state.depth) { + DepthTarget depth{}; + depth.format = state.depth->format; + depth.samples = state.depth->samples; + canonical.depth = depth; + } + canonical.depthTest = state.depthTest; + canonical.depthWrite = state.depthWrite; + canonical.depthCompare = state.depthCompare; + canonical.depthBoundsTest = state.depthBoundsTest; + canonical.depthBias = state.depthBias; + canonical.depthBiasClamp = state.depthBiasClamp; + canonical.depthBiasPerFace = state.depthBiasPerFace; + canonical.stencilTest = state.stencilTest; + canonical.stencilFront = state.stencilFront; + canonical.stencilBack = state.stencilBack; + if (state.stencilReplayReference) canonical.stencilReplayReference = 0u; + for (const auto& color : state.colors) { + ColorTarget canonicalColor{}; + canonicalColor.format = color.format; + canonicalColor.exportIndex = color.exportIndex; + canonicalColor.componentMapping = color.componentMapping; + canonical.colors.push_back(canonicalColor); + } + canonical.blends = state.blends; + canonical.blendConstants = state.blendConstants; + canonical.hasColorTarget = state.hasColorTarget; + canonical.rectList = state.rectList; + canonical.dualSourceBlend = state.dualSourceBlend; + canonical.reversedBlend = state.reversedBlend; + canonical.topology = state.topology; + canonical.primitiveRestart = state.primitiveRestart; + canonical.negativeOneToOne = state.negativeOneToOne; + canonical.depthClamp = state.depthClamp; + canonical.conservativeRasterization = state.conservativeRasterization; + canonical.cullMode = state.cullMode; + canonical.frontFace = state.frontFace; + canonical.provokingVertexMode = state.provokingVertexMode; + canonical.samples = state.samples; + canonical.sampleLocations = state.sampleLocations; + canonical.customSampleLocations = state.customSampleLocations; + canonical.sampleLocationGrid = state.sampleLocationGrid; + canonical.perSampleShading = state.perSampleShading; + canonical.quadSampleMask = state.quadSampleMask; + return canonical; +} + +bool GeneratedPipelineStage(const State& state, ShaderRecompiler::ShaderStage stage) { + using Stage = ShaderRecompiler::ShaderStage; + return (state.rectList && (stage == Stage::TessellationControl || stage == Stage::TessellationEvaluation)) || stage == Stage::Geometry; +} + +GraphicsPipelineRecipe MakeGraphicsPipelineRecipe(const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout) { + GraphicsPipelineRecipe recipe; + recipe.attachmentLayout = attachmentLayout; + for (const auto& shader : shaders) { + Require(shader.program != nullptr, "missing compiled shader"); + recipe.stages.push_back({shader.stage, shader.program->Identity(), shader.pushConstantOffset, GeneratedPipelineStage(state, shader.stage)}); + } + recipe.vertexInput = vertexInput; + recipe.mainLayoutKey.assign(resources.mainKey.begin(), resources.mainKey.end()); + recipe.fragmentLayoutKey.assign(resources.fragmentKey.begin(), resources.fragmentKey.end()); + recipe.splitSets = resources.split; + recipe.state = CanonicalPipelineState(state); + return recipe; +} + +std::vector EncodeGraphicsPipelineRecipe(const GraphicsPipelineRecipe& recipe) { + std::vector bytes; + Writer writer(bytes); + encodeRecipe(writer, recipe); + return bytes; +} + +bool DecodeGraphicsPipelineRecipe(std::span bytes, GraphicsPipelineRecipe& recipe) { + Reader reader(bytes); + return decodeRecipe(reader, recipe); +} + +std::uint64_t PipelineSwitch(const char* name, std::uint64_t fallback) { + return configured(name, fallback); +} + +bool PipelineJournalEnabled() { + return Journal::Instance().Enabled(); +} + +bool PipelineJournalReplaying() { + return replayingHere; +} + +std::filesystem::path PipelineJournalPath() { + return Journal::Instance().Path(); +} + +PipelineJournalCounters PipelineJournalTotals() { + auto totals = Journal::Instance().Totals(); + const auto& replay = replayCounters(); + totals.replayedGraphics = replay.graphics.load(std::memory_order_relaxed); + totals.replayedCompute = replay.compute.load(std::memory_order_relaxed); + totals.replayPresent = replay.present.load(std::memory_order_relaxed); + totals.replaySkipped = replay.skipped.load(std::memory_order_relaxed); + totals.replayRejected = replay.rejected.load(std::memory_order_relaxed); + totals.replayFailed = replay.failed.load(std::memory_order_relaxed); + totals.replayMs = replay.ms.load(std::memory_order_relaxed); + totals.replayThreads = replay.threads.load(std::memory_order_relaxed); + return totals; +} + +void RecordGraphicsPipeline(const GraphicsPipelineRecipe& recipe, std::span shaders) { + Journal::Instance().RecordGraphics(recipe, shaders); +} + +void RecordComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants) { + Journal::Instance().RecordCompute(shader, pushConstants); +} + +void FlushPipelineJournal() { + auto* journal = Journal::Existing(); + if (journal != nullptr && journal->Enabled()) journal->Flush(); +} + +void ReportPipelineJournal() { + if (!profiling() || !Journal::Instance().Enabled()) return; + const auto totals = PipelineJournalTotals(); + AgcDriver::ProfilePrint_nid_no_patch("[pipeline-journal] loaded %llu pipelines, %llu modules (%.1f MiB, %.0f ms); replay on %u threads: %llu graphics and %llu compute built, %llu present, %llu skipped (store full), %llu rejected, %llu failed in %.0f ms; recorded this run: %llu graphics, %llu compute, %llu modules (%.1f MiB), %llu write failures\n", + static_cast(totals.loadedRecipes), static_cast(totals.loadedModules), static_cast(totals.loadedBytes) / (1024.0 * 1024.0), totals.loadMs, totals.replayThreads, + static_cast(totals.replayedGraphics), static_cast(totals.replayedCompute), static_cast(totals.replayPresent), static_cast(totals.replaySkipped), static_cast(totals.replayRejected), static_cast(totals.replayFailed), totals.replayMs, + static_cast(totals.recordedGraphics), static_cast(totals.recordedCompute), static_cast(totals.recordedModules), static_cast(totals.writtenBytes) / (1024.0 * 1024.0), static_cast(totals.writeFailures)); +} + +std::uint64_t PipelineJournalDeviceFingerprint(const VkPhysicalDeviceProperties& properties) { + std::array words{properties.vendorID, properties.deviceID, properties.driverVersion, 0u, 0u, 0u, 0u}; + static_assert(VK_UUID_SIZE == 16); + std::memcpy(words.data() + 3, properties.pipelineCacheUUID, VK_UUID_SIZE); + return ShaderRecompiler::HashSpirvWords(words, 0x6a6f75726e616c01ull); +} + +std::shared_ptr StartPipelineJournalReplay(const Context& context, std::uint64_t deviceFingerprint, PipelineJournalHooks hooks) { + if (!Journal::Instance().Enabled() || !Journal::Instance().Bind(deviceFingerprint)) return nullptr; + const auto threads = static_cast(configured("APS5_PIPELINE_JOURNAL_THREADS", std::max(1u, std::thread::hardware_concurrency() / 2u))); + if (threads == 0) return nullptr; + return std::make_shared(context, std::move(hooks), threads); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp index bd2a12ed32..7e8fec28b6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include #include @@ -204,13 +205,16 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip break; } VkPipeline handle = VK_NULL_HANDLE; - if (partsOnly) lock.unlock(); + // Built without the store's mutex: a draw whose parts are resident is not held behind + // another thread's build, and builds from several threads (the registration prebuild, the + // journal replay) run at once; a part another thread built meanwhile is kept, this one dropped. + lock.unlock(); VkResult created; { auto pipelineCacheLock = context.LockPipelineCacheForUse(); created = context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &handle); } - if (partsOnly) lock.lock(); + lock.lock(); Check(created, "vkCreateGraphicsPipelines library"); if (const auto raced = cache.find(key); raced != cache.end()) { device.destroyPipeline(context.device, handle, nullptr); @@ -238,10 +242,12 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &linked}; create.layout = info.layout; VkPipeline pipeline = VK_NULL_HANDLE; + lock.unlock(); { auto pipelineCacheLock = context.LockPipelineCacheForUse(); Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &pipeline), "vkCreateGraphicsPipelines link"); } + lock.lock(); ++device.counters.linked; if (optimize && optimized != nullptr) { auto slot = std::make_shared(); @@ -252,7 +258,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip const auto cache = context.pipelineCache; const auto cacheSynchronization = context.pipelineCacheSynchronization; const auto libraryLayout = fullLayout; - enqueue(store, context.device, [slot, createPipelines, cache, cacheSynchronization, libraryLayout, libraries, &store, devicePointer = &device] { + std::function relink = [slot, createPipelines, cache, cacheSynchronization, libraryLayout, libraries, &store, devicePointer = &device] { if (slot->released.load()) return; VkPipelineLibraryCreateInfoKHR linked{VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR}; linked.libraryCount = static_cast(libraries.size()); @@ -272,7 +278,13 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip } std::lock_guard lock(store.mutex); ++devicePointer->counters.optimized; - }); + }; + if (PipelineJournalReplaying()) { + slot->relink = std::move(relink); + slot->deferred.store(true, std::memory_order_release); + } else { + enqueue(store, context.device, std::move(relink)); + } } return pipeline; } @@ -282,6 +294,15 @@ void OptimizedPipeline::Release() noexcept { if (const auto pipeline = handle.exchange(VK_NULL_HANDLE)) destroy(device, pipeline, nullptr); } +void OptimizedPipeline::Schedule() { + if (!deferred.load(std::memory_order_acquire)) return; + auto& store = Libraries(); + std::lock_guard lock(store.mutex); + if (!deferred.exchange(false, std::memory_order_acq_rel)) return; + if (relink && !released.load()) enqueue(store, device, std::move(relink)); + relink = nullptr; +} + void WaitForOptimizedPipelines(VkDevice device) { auto& store = Libraries(); std::unique_lock lock(store.mutex); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index d186a3876d..1f03eea443 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Execution/include/AsyncLog.hpp" #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libc/include/HostThreadLocal.hpp" @@ -3840,7 +3841,7 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source, bool try { action(); } catch (const std::exception& error) { - std::fprintf(stderr, "[gpu] deferred write-back failed: %s\n", error.what()); + AgcDriver::LogCoalescedStderr_nid_no_patch("[gpu] deferred write-back failed: %s\n", error.what()); } } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index e31084e0c4..d9c0124e45 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -1989,6 +1989,7 @@ void ShaderResources::buildComplete() { write.pImageInfo = images.data() + images.size(); for (const auto index : binding.imageAllocations) { if (index == std::numeric_limits::max()) images.push_back(absentImage(binding.layout.binding)); + else if (storageTextures[index] == nullptr) images.push_back({VK_NULL_HANDLE, multisampleStorageViews[index], VK_IMAGE_LAYOUT_GENERAL}); else images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL}); } Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding"); @@ -4051,6 +4052,21 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind const auto resource = record != nullptr && record->decoded ? record->resource : DecodeTextureResource(words); const bool firstLayer = binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k2DArray; if (binding.imageShape.has_value() && !firstLayer && !MatchesGuestDimension(*binding.imageShape, resource.dimension)) throw std::runtime_error("AGC graphics: guest storage texture dimension disagrees with the shader's declared image shape (shape " + std::to_string(static_cast(*binding.imageShape)) + ", dimension " + std::to_string(static_cast(resource.dimension)) + ")"); + if (resource.dimension == TextureDimension::k2DMsaa) { + Require(!(element < binding.imageAtomic.size() && binding.imageAtomic[element]) && !(element < binding.imageAtomic64.size() && binding.imageAtomic64[element]), "storage image atomics on a multisampled texture are not implemented"); + storageTextures.push_back(nullptr); + multisampleStorageViews.resize(storageTextures.size(), VK_NULL_HANDLE); + multisampleStorageViews.back() = MultisampleTargetStorageView(context, resource); + multisampleSources.push_back(resource.baseAddress); + storageMips.push_back(0); + storageKeys.push_back(0); + storageFirstLayer.push_back(false); + storageWritten.push_back(false); + storageAtomic.push_back(false); + storageAtomic64.push_back(false); + item.imageAllocations.push_back(storageTextures.size() - 1); + continue; + } const auto mip = std::min(resource.baseLevel + mipOffset, resource.mipCount - 1u); Require(resource.minLod <= mip * 256u, "guest storage texture descriptor clamps its minimum LOD above the level it addresses, which is not implemented"); const auto guestBytes = record != nullptr && record->decoded ? record->guestBytes : DescribeSurface(resource).guestBytes; @@ -4364,7 +4380,7 @@ std::vector> ShaderResources::PresyncSur } } for (std::size_t index = 0; index < storageTextures.size(); ++index) { - if (index != 0 && storageTextures[index] == storageTextures[index - 1]) continue; + if (storageTextures[index] == nullptr || (index != 0 && storageTextures[index] == storageTextures[index - 1])) continue; const auto& image = *storageTextures[index]; consider(image.Descriptor().format, image.Descriptor().baseAddress, image.GuestBytes(), false); } @@ -4649,7 +4665,7 @@ void ShaderResources::WriteBack() { WriteBackBuffers(); if (SkipWriteBack()) return; for (std::size_t index = 0; index < storageTextures.size(); ++index) { - if (!storageWritten[index]) continue; + if (!storageWritten[index] || storageTextures[index] == nullptr) continue; storageTextures[index]->MarkDirty(); static_cast(DepthSurfaceStoreBack(context, *storageTextures[index])); } @@ -4668,7 +4684,7 @@ void ShaderResources::MarkGpuWrites(Recorder& recorder) { } if (SkipWriteBack()) return; for (std::size_t index = 0; index < storageTextures.size(); ++index) { - if (!storageWritten[index]) continue; + if (!storageWritten[index] || storageTextures[index] == nullptr) continue; storageTextures[index]->MarkDirty(); static_cast(DepthSurfaceStoreBack(context, *storageTextures[index])); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index 9c7dcb033e..f2c1970381 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -47,7 +47,7 @@ struct Module { bool position = false; std::uint32_t clipDistances = 0; std::uint32_t cullDistances = 0; - bool primitiveIndices = false; + std::uint32_t primitiveIndices = 0; bool fragmentBarycentric = false; bool layerOutput = false; std::map> modes; @@ -134,8 +134,9 @@ struct Module { Require(input && ((value == spv::BuiltInPosition && signature == "f32x4") || (value == spv::BuiltInTessCoord && stage == Stage::TessellationEvaluation && signature == "f32x3") || ((value == spv::BuiltInInvocationId || value == spv::BuiltInPrimitiveId || value == spv::BuiltInPatchVertices) && (signature == "i32" || signature == "u32"))), "unsupported tessellation built-in"); return; } - if (!input && value == spv::BuiltInPrimitiveTriangleIndicesEXT) primitiveIndices = true; - Require((input && ((value == spv::BuiltInWorkgroupId || value == spv::BuiltInLocalInvocationId || value == spv::BuiltInGlobalInvocationId || value == spv::BuiltInNumWorkgroups) && signature == "u32x3")) || (input && value == spv::BuiltInLocalInvocationIndex && signature == "u32") || (!input && value == spv::BuiltInPrimitiveTriangleIndicesEXT && signature == "u32x3") || (!input && value == spv::BuiltInCullPrimitiveEXT && signature == "bool") || (!input && value == spv::BuiltInLayer && signature == "u32"), "unsupported mesh built-in"); + if (!input && value == spv::BuiltInPrimitiveLineIndicesEXT) primitiveIndices = 2u; + if (!input && value == spv::BuiltInPrimitiveTriangleIndicesEXT) primitiveIndices = 3u; + Require((input && ((value == spv::BuiltInWorkgroupId || value == spv::BuiltInLocalInvocationId || value == spv::BuiltInGlobalInvocationId || value == spv::BuiltInNumWorkgroups) && signature == "u32x3")) || (input && value == spv::BuiltInLocalInvocationIndex && signature == "u32") || (!input && value == spv::BuiltInPrimitiveLineIndicesEXT && signature == "u32x2") || (!input && value == spv::BuiltInPrimitiveTriangleIndicesEXT && signature == "u32x3") || (!input && value == spv::BuiltInCullPrimitiveEXT && signature == "bool") || (!input && value == spv::BuiltInLayer && signature == "u32"), "unsupported mesh built-in"); return; } const auto signature = Signature(type); @@ -151,7 +152,7 @@ struct Module { } }; -Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics, bool shaderClipDistance, bool shaderCullDistance, bool viewportIndexLayer, bool runtimeDescriptorArray, bool demoteToHelperInvocation) { +Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics, bool shaderClipDistance, bool shaderCullDistance, bool viewportIndexLayer, bool runtimeDescriptorArray, bool demoteToHelperInvocation, bool shaderTerminateInvocation) { using Stage = ShaderRecompiler::ShaderStage; Require(compiled.program != nullptr, "missing compiled shader"); const auto& shader = *compiled.program; @@ -185,6 +186,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi std::uint32_t entryPoint = 0; std::set executionModeTargets; bool upperLeft = false; + bool terminateExtension = false; for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> 16u; Require(count != 0 && count <= words.size() - cursor, "truncated SPIR-V instruction"); @@ -331,9 +333,19 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(fragment && fragmentShaderBarycentric, "SPV_KHR_fragment_shader_barycentric requires enabled fragmentShaderBarycentric in a fragment shader"); break; } + if (extension == "SPV_KHR_terminate_invocation") { + Require(fragment && shaderTerminateInvocation, "SPV_KHR_terminate_invocation requires enabled shaderTerminateInvocation in a fragment shader"); + terminateExtension = true; + break; + } Require(extension == "SPV_KHR_float_controls" || (imageInt64Atomics && extension == "SPV_EXT_shader_image_int64") || (viewportIndexLayer && extension == "SPV_EXT_shader_viewport_index_layer") || (fragment && demoteToHelperInvocation && extension == "SPV_EXT_demote_to_helper_invocation") || (mesh && (extension == "SPV_EXT_mesh_shader" || extension == "SPV_KHR_physical_storage_buffer")) || ((descriptorIndexing || runtimeDescriptorArray) && extension == "SPV_EXT_descriptor_indexing") || (shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && (extension == "SPV_KHR_physical_storage_buffer" || extension == "SPV_KHR_8bit_storage")), "unsupported SPIR-V extension"); break; } + case spv::OpTerminateInvocation: + Require(count == 1, "invalid OpTerminateInvocation instruction"); + Require(fragment && shaderTerminateInvocation, "OpTerminateInvocation requires enabled shaderTerminateInvocation in a fragment shader"); + Require(terminateExtension, "OpTerminateInvocation requires SPV_KHR_terminate_invocation"); + break; case spv::OpDecorateId: case spv::OpDecorationGroup: case spv::OpGroupDecorate: @@ -437,7 +449,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi mode(spv::ExecutionModeLocalSize, {paired ? config.threadsPerGroup / 2u : config.threadsPerGroup, 1, 1}); mode(spv::ExecutionModeOutputVertices, {config.maxVertices}); mode(spv::ExecutionModeOutputPrimitivesEXT, {config.maxPrimitives}); - mode(spv::ExecutionModeOutputTrianglesEXT, {}); + mode(config.outputPrimitiveSize == 2u ? spv::ExecutionModeOutputLinesEXT : spv::ExecutionModeOutputTrianglesEXT, {}); Require(module.modes.size() == 4, "unsupported mesh execution mode"); } else if (control) { Require(state.rectList || state.stages.tessellation.has_value(), "tessellation configuration is missing"); @@ -486,7 +498,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto length = module.constants.find(outer[3]); Require(length != module.constants.end() && length->second != 0, "invalid per-vertex interface array length"); if (mesh) { - const bool primitive = decoration.perPrimitive || (decoration.builtin && (*decoration.builtin == spv::BuiltInPrimitiveTriangleIndicesEXT || *decoration.builtin == spv::BuiltInCullPrimitiveEXT)); + const bool primitive = decoration.perPrimitive || (decoration.builtin && (*decoration.builtin == spv::BuiltInPrimitiveLineIndicesEXT || *decoration.builtin == spv::BuiltInPrimitiveTriangleIndicesEXT || *decoration.builtin == spv::BuiltInCullPrimitiveEXT)); Require(length->second == (primitive ? state.stages.mesh->maxPrimitives : state.stages.mesh->maxVertices), "mesh interface array disagrees with output limits"); } else if (geometry) { Require(length->second == 3u, "geometry interface array must hold the three vertices of a triangle"); @@ -594,7 +606,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(descriptors.size() == shader.bindings.size(), "recompiler binding metadata contains undeclared resources"); Require(push == (!shader.pushConstants.empty() || mesh || (control && !rectListControl)), "recompiler push constant metadata disagrees with SPIR-V"); for (const auto id : module.interface) Require(module.variables.contains(id), "entry point interface contains an unknown variable"); - if (mesh) Require(module.primitiveIndices, "mesh shader does not export primitive indices"); + if (mesh) Require(module.primitiveIndices == state.stages.mesh->outputPrimitiveSize, "mesh shader does not export primitive indices"); if (stage == Stage::Vertex || mesh || evaluation || geometry) Require(module.position, "vertex shader does not export position"); return module; } @@ -602,7 +614,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi } bool DeclaresEarlyFragmentTests(const ShaderRecompiler::RecompileResult& program) { - const auto& words = program.spirv.Words(); + const auto words = program.spirv.Words(); Require(words.size() >= 5 && words[0] == spv::MagicNumber, "the fragment shader is not a SPIR-V module"); for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> 16u; @@ -615,7 +627,7 @@ bool DeclaresEarlyFragmentTests(const ShaderRecompiler::RecompileResult& program return false; } -std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics, bool shaderClipDistance, bool shaderCullDistance, bool viewportIndexLayer, bool runtimeDescriptorArray, bool demoteToHelperInvocation) { +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics, bool shaderClipDistance, bool shaderCullDistance, bool viewportIndexLayer, bool runtimeDescriptorArray, bool demoteToHelperInvocation, bool shaderTerminateInvocation) { using Stage = ShaderRecompiler::ShaderStage; const bool tessellation = state.stages.path == ShaderPath::Tessellation; const bool mesh = state.stages.path == ShaderPath::Geometry; @@ -636,7 +648,7 @@ std::set ValidateShaders(std::span shaders, Require(shaders[i].program != nullptr, "missing compiled shader"); Require(shaders[i].stage == expected, "graphics stage order disagrees"); for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shaders[i].stage == ShaderRecompiler::ShaderStage::Fragment && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "graphics resource uses a descriptor set other than its stage's"); - const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric, descriptorIndexing, imageInt64Atomics, geometryShader, sampleRateShading, bufferInt64Atomics, shaderClipDistance, shaderCullDistance, viewportIndexLayer, runtimeDescriptorArray, demoteToHelperInvocation); + const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric, descriptorIndexing, imageInt64Atomics, geometryShader, sampleRateShading, bufferInt64Atomics, shaderClipDistance, shaderCullDistance, viewportIndexLayer, runtimeDescriptorArray, demoteToHelperInvocation, shaderTerminateInvocation); if (i != 0) { for (const auto& [location, signature] : current.inputs) { const auto output = previous.outputs.find(location); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index b767129e68..a3c166d395 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -256,7 +256,7 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint } void decodeDepth(const Registers& cx, std::uint32_t depthControl, bool testValExport, State& result) { - zero(cx, 0x000, 0x00001f9cu, "depth copy, resummarize or decompress draws (DB_RENDER_CONTROL)"); + zero(cx, 0x000, 0x0000101cu, "depth copy, resummarize or decompress draws (DB_RENDER_CONTROL)"); const bool depthClear = (read(cx, 0x000) & 1u) != 0; const bool stencilClear = (read(cx, 0x000) & 2u) != 0; const auto view = read(cx, 0x002); @@ -415,6 +415,8 @@ std::string singleSampleMsaaRejection(const QueueState& queue) { } constexpr std::uint32_t CoveredCentroidIsCenter = 1u << 29u; +constexpr std::uint32_t SampleZeroRefusedInputs = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCentroid) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::LinearCentroid) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::SampleCoverage); +constexpr std::uint32_t PixelCenterPositionInputs = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); constexpr std::uint32_t CentroidInputs = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCentroid) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::LinearCentroid); bool coveredCentroidSupported(const QueueState& queue) { @@ -489,6 +491,24 @@ void decodeSampleLocations(const Registers& cx, State& state) { } } +bool rasterizesSampleZeroOnly(const Registers& cx, const State& state) { + if (state.samples <= 1 || state.alphaToCoverage || (read(cx, 0x292) & 1u) == 0) return false; + const auto inputs = read(cx, 0x1b3) & read(cx, 0x1b4); + const auto refused = state.perSampleShading ? ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::SampleCoverage) : SampleZeroRefusedInputs; + const bool sampleIndexMovesPosition = (inputs & ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::Ancillary)) != 0 && (inputs & PixelCenterPositionInputs) != 0; + return (inputs & refused) == 0 && !sampleIndexMovesPosition; +} + +void rasterizeSampleZero(State& state) { + std::array sampleZero{}; + for (std::uint32_t pixel = 0; pixel < 4; ++pixel) sampleZero[pixel * 8u] = state.sampleLocations[pixel * 8u]; + state.samples = 1; + state.perSampleShading = false; + state.sampleLocations = sampleZero; + state.customSampleLocations = std::any_of(sampleZero.begin(), sampleZero.end(), [](std::uint8_t bits) { return bits != 0; }); + state.sampleLocationGrid = sampleZero[8] != sampleZero[0] || sampleZero[16] != sampleZero[0] || sampleZero[24] != sampleZero[0]; +} + bool colorControlSupported(std::uint32_t colorControl, bool hasColorTarget) { colorControl &= ~1u; return colorControl == 0xcc0010u || ((colorControl >> 4u) & 7u) == 0u || (!hasColorTarget && (colorControl & ~0x70u) == 0xcc0000u); @@ -876,7 +896,10 @@ ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup) { validate(!fastLaunch || (!vertexWorkgroup && primitive == 1 && vertices == 1 && primitives == 1), "fast launch other than one point per subgroup is unsupported"); const auto maxVertices = read(queue.context, 0x1ff); const auto verticesPerPrimitive = vertexWorkgroup ? 3u : read(queue.context, 0x2ce); - validate((primitive == 1 || primitive == 2 || primitive == 4 || primitive == 5 || primitive == 6) && read(queue.context, 0x29b) == 2 && verticesPerPrimitive >= 3, "unsupported geometry input or output assembly"); + const auto outputType = read(queue.context, 0x29b); + validate(outputType == 2 || (outputType == 1 && !vertexWorkgroup), "VGT_GS_OUT_PRIM_TYPE other than a triangle strip, or a line strip from a geometry program, is unsupported"); + const auto outputSize = outputType == 1 ? 2u : 3u; + validate((primitive == 1 || primitive == 2 || primitive == 4 || primitive == 5 || primitive == 6) && verticesPerPrimitive >= outputSize, "unsupported geometry input or output assembly"); const auto inputSize = primitive == 1 ? 1u : primitive == 2 ? 2u : 3u; validate(vertices >= inputSize && maxVertices != 0 && (!vertexWorkgroup || maxVertices >= inputSize) && maxVertices <= 256 && verticesPerPrimitive <= 256, "invalid geometry subgroup output"); const auto inputStep = primitive == 5 || primitive == 6 ? 1u : inputSize; @@ -886,11 +909,11 @@ ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup) { const auto resources = read(queue.shader, 0x8b, RegisterBank::Shader); const auto esgsItemSize = read(queue.context, 0x2ab); validate(esgsItemSize != 0 && esgsItemSize * vertices <= 0xffffu, "invalid VGT_ESGS_RING_ITEMSIZE"); - const auto amplification = fastLaunch ? read(queue.context, 0x2d3) & 0x1ffu : verticesPerPrimitive - 2u; + const auto amplification = fastLaunch ? read(queue.context, 0x2d3) & 0x1ffu : verticesPerPrimitive - outputSize + 1u; validate(amplification != 0, "fast launch subgroup amplifies to no primitives"); - const auto maxPrimitives = primitives * amplification; + const auto maxPrimitives = outputSize == 2u && !fastLaunch ? std::max(primitives * amplification, maxVertices) : primitives * amplification; const auto threads = std::max({(groupPrimitives - 1u) * inputStep + inputSize, primitives, maxVertices, maxPrimitives}); - result.mesh = ShaderRecompiler::MeshConfiguration{primitive, groupPrimitives, (groupPrimitives - 1u) * inputStep + inputSize, maxVertices, maxPrimitives, ((threads + result.vertexWaveSize - 1u) / result.vertexWaveSize) * result.vertexWaveSize, ((resources >> 19u) & 0xffu) * 128u, 0, esgsItemSize}; + result.mesh = ShaderRecompiler::MeshConfiguration{primitive, groupPrimitives, (groupPrimitives - 1u) * inputStep + inputSize, maxVertices, maxPrimitives, ((threads + result.vertexWaveSize - 1u) / result.vertexWaveSize) * result.vertexWaveSize, ((resources >> 19u) & 0xffu) * 128u, 0, esgsItemSize, outputSize}; } return result; } @@ -912,10 +935,10 @@ bool DepthResummarizeDrawsNothing(const QueueState& queue) { std::string DepthMaintenanceRejection(const QueueState& queue) { if (DepthResummarizeDrawsNothing(queue)) return {}; const auto control = find(queue.context, 0x000); - if (control == queue.context.end() || (control->second & ~0x2063u) == 0) return {}; + if (control == queue.context.end() || (control->second & ~0x2fe3u) == 0) return {}; const auto depthControl = find(queue.context, 0x200); const bool colorMasks = find(queue.context, 0x8e) != queue.context.end() && find(queue.context, 0x8f) != queue.context.end(); - if ((control->second & ~0x2063u) == 0x10u && depthControl != queue.context.end() && (depthControl->second & 3u) == 0 && colorMasks && ColorWriteMask(queue.context) == 0) return {}; + if ((control->second & ~0x2fe3u) == 0x10u && depthControl != queue.context.end() && (depthControl->second & 3u) == 0 && colorMasks && ColorWriteMask(queue.context) == 0) return {}; return zeroMessage(0x000, control->second, "DB_RENDER_CONTROL depth copy, resummarize or decompress"); } @@ -1031,7 +1054,8 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) { Require(ditheredAlphaSamplesSupported(cx), "dithered alpha-to-coverage requires eight matched raster/DB_EQAA samples and one pixel iteration"); Require(!result.customSampleLocations, "dithered alpha-to-coverage with custom sample locations is unsupported"); } - Require(!result.depth || result.depth->samples == result.samples, "the depth target's sample count differs from the rasterizer's"); + const bool depthKeepsSampleZero = result.depth && result.depth->samples == 1 && result.samples > 1; + Require(!result.depth || depthKeepsSampleZero || result.depth->samples == result.samples, "the depth target's sample count differs from the rasterizer's"); zero(cx, 0x292, ScanModeMask & ~1u, "scan conversion mode"); if (result.samples > 1 && (read(cx, 0x292) & 1u) == 0) { const bool filled = result.topology != VK_PRIMITIVE_TOPOLOGY_POINT_LIST && result.topology != VK_PRIMITIVE_TOPOLOGY_LINE_LIST && result.topology != VK_PRIMITIVE_TOPOLOGY_LINE_STRIP; @@ -1046,6 +1070,10 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) { } zero(cx, 0x293, ScanControlMask, "primitive discard or out-of-order rasterization"); result.perSampleShading = result.samples > 1 && (read(cx, 0x293) & PixelShaderIteratesSamples) != 0; + if (depthKeepsSampleZero) { + Require(rasterizesSampleZeroOnly(cx, result), "the depth target's sample count differs from the rasterizer's"); + rasterizeSampleZero(result); + } zero(cx, 0x80, ~0u, "window offset"); zero(cx, 0x8d, ScreenOffsetMask, "reserved PA_SU_HARDWARE_SCREEN_OFFSET bits"); Require(read(cx, 0x83) == 0xffffu, "clip rectangles are unsupported"); @@ -1060,7 +1088,7 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) { const auto raster = read(cx, 0x205); // Bits 5-10 give the front/back polygon type (2 = filled triangles), which POLY_MODE (bit 3) turns on // explicitly; KEEP_TOGETHER_ENABLE (bit 24) only affects primitive distribution across the chip. - const bool pointsOrLines = result.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || result.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || result.topology == VK_PRIMITIVE_TOPOLOGY_LINE_STRIP; + const bool pointsOrLines = result.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || result.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || result.topology == VK_PRIMITIVE_TOPOLOGY_LINE_STRIP || (result.stages.mesh.has_value() && result.stages.mesh->outputPrimitiveSize == 2u); const auto lineOffset = (raster & 0x1800u) != 0 && !pointsOrLines ? 0x2000u : 0u; const auto rasterMode = raster & ~0x7u & ~(1u << 24u) & ~0x1800u & ~lineOffset & ~(1u << 19u) & ~(1u << 16u); if (rasterMode != 0 && rasterMode != 0x240u && rasterMode != 0x248u) throw std::runtime_error("AGC graphics: " + zeroMessage(0x205, raster, "polygon mode, depth bias or nonstandard rasterization")); @@ -1111,6 +1139,7 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) { result.customSampleLocations = true; result.sampleLocations.fill(0); } + if (color.samples == 1 && result.colors.empty() && !result.depth && rasterizesSampleZeroOnly(cx, result)) rasterizeSampleZero(result); Require(color.samples == result.samples, "a color target's sample count differs from the rasterizer's"); APS5_LOG_OUT_DEBUG("Color %u address=0x%llx extent=%ux%u bytes=%llu VkFormat=%u", slot, static_cast(color.address), color.extent.width, color.extent.height, static_cast(color.bytes), static_cast(color.format)); if (result.colors.empty()) { @@ -1326,6 +1355,7 @@ std::array QuadSampleMasks(const State& state) { ShaderRecompiler::RasterPrimitive RasterPrimitiveOf(const State& state) { if (state.rectList) return ShaderRecompiler::RasterPrimitive::Rectangle; + if (state.stages.mesh.has_value() && state.stages.mesh->outputPrimitiveSize == 2u) return ShaderRecompiler::RasterPrimitive::Line; if (state.stages.mesh.has_value() || state.stages.tessellation.has_value()) return ShaderRecompiler::RasterPrimitive::Triangle; switch (state.topology) { case VK_PRIMITIVE_TOPOLOGY_POINT_LIST: return ShaderRecompiler::RasterPrimitive::Point; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index e71366c10b..ed284ea658 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Execution/include/AsyncLog.hpp" #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" @@ -259,9 +260,9 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex const auto& mips = geometry.mips; const auto arrayLayers = geometry.layers; const auto elementBytes = BytesPerElement(descriptor.format); - APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u layers=%u dim=%d tile=%d format=%u vk=%d element=%u", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, arrayLayers, + AGC_LOG_DEFERRED(stdout, "Texture address=0x%llx %ux%u mips=%u layers=%u dim=%d tile=%d format=%u vk=%d element=%u", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, arrayLayers, static_cast(descriptor.dimension), static_cast(descriptor.tileMode), descriptor.format, static_cast(vkFormat), elementBytes); - for (const auto& mip : mips) APS5_LOG_OUT(" mip %ux%u tiled=0x%llx+0x%llx linear=0x%llx+0x%llx blocksPerRow=%u pitch=%u tail=%d", mip.width, mip.height, static_cast(mip.tiledOffset), static_cast(mip.tiledSize), static_cast(mip.linearOffset), static_cast(mip.linearSize), mip.blocksPerRow, mip.pitchBytes, mip.tail ? 1 : 0); + for (const auto& mip : mips) AGC_LOG_DEFERRED(stdout, " mip %ux%u tiled=0x%llx+0x%llx linear=0x%llx+0x%llx blocksPerRow=%u pitch=%u tail=%d", mip.width, mip.height, static_cast(mip.tiledOffset), static_cast(mip.tiledSize), static_cast(mip.linearOffset), static_cast(mip.linearSize), mip.blocksPerRow, mip.pitchBytes, mip.tail ? 1 : 0); const auto guestBytes = geometry.guestBytes; Require(snapshot.size() == guestBytes, "texture snapshot size mismatch"); @@ -504,7 +505,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& copyLevels = descriptor.mipCount; copyLayers = geometry.imageLayers; copyElementBytes = BytesPerElement(descriptor.format); - APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u layers=%u block copy of storage image (guest format %u -> vk %d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, copyLayers, source->Descriptor().format, static_cast(blockFormat)); + AGC_LOG_DEFERRED(stdout, "Texture address=0x%llx %ux%u mips=%u layers=%u block copy of storage image (guest format %u -> vk %d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, copyLayers, source->Descriptor().format, static_cast(blockFormat)); VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; imageInfo.imageType = VK_IMAGE_TYPE_2D; @@ -560,7 +561,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& copyLevels = descriptor.mipCount; copyLayers = geometry.imageLayers; copyElementBytes = BytesPerElement(descriptor.format); - APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u layers=%u depth copy of storage image (vk=%d -> %d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, copyLayers, static_cast(colorFormat), static_cast(depthFormat)); + AGC_LOG_DEFERRED(stdout, "Texture address=0x%llx %ux%u mips=%u layers=%u depth copy of storage image (vk=%d -> %d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, copyLayers, static_cast(colorFormat), static_cast(depthFormat)); VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; imageInfo.flags = descriptor.dimension == TextureDimension::kCube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u; @@ -747,7 +748,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& Require(source != nullptr && CanCopyFrom(*source, descriptor), "storage image does not match the sampled texture"); const auto vkFormat = SampledTextureFormat(context, descriptor.format); const auto geometry = DescribeSurface(descriptor); - APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u viewed from storage image (vk=%d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, static_cast(vkFormat)); + AGC_LOG_DEFERRED(stdout, "Texture address=0x%llx %ux%u mips=%u viewed from storage image (vk=%d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, static_cast(vkFormat)); // Storage images stay in the general layout; the view samples them there. layout = VK_IMAGE_LAYOUT_GENERAL; const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -977,6 +978,10 @@ bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat) { } } +VkFormat HostStorageFormat(const Context& context, VkFormat format) { + return StorageFormatOrUndefined(context, format); +} + bool IntegerFormat(VkFormat format) { switch (format) { case VK_FORMAT_R8_UINT: case VK_FORMAT_R8_SINT: case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8_SINT: @@ -1087,7 +1092,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, Require(mipLevel < descriptor.mipCount, "storage texture mip level is outside the texture"); const auto vkFormat = StorageFormatFor(context, ResolveTextureFormat(descriptor.format)); storageFormat = vkFormat; - APS5_LOG_OUT("StorageTexture address=0x%llx %ux%u mips=%u mip=%u layers=%u base=%u dim=%d tile=%d format=%u vk=%d", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, mipLevel, + AGC_LOG_DEFERRED(stdout, "StorageTexture address=0x%llx %ux%u mips=%u mip=%u layers=%u base=%u dim=%d tile=%d format=%u vk=%d", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, mipLevel, descriptor.depthOrLastArray, descriptor.baseArray, static_cast(descriptor.dimension), static_cast(descriptor.tileMode), descriptor.format, static_cast(vkFormat)); geometry = DescribeSurface(descriptor); mips = geometry.mips; diff --git a/core/libs/prx/libSceAgcDriver/tests/AsyncLog.cpp b/core/libs/prx/libSceAgcDriver/tests/AsyncLog.cpp new file mode 100644 index 0000000000..0325795a41 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/AsyncLog.cpp @@ -0,0 +1,337 @@ +#include "prx/libSceAgcDriver/Execution/include/AsyncLog.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#endif + +namespace { + +using namespace std::chrono_literals; +using Clock = AgcDriver::AsyncLog::Clock; + +void Check(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +std::string ReadAll(std::FILE* file) { + std::fflush(file); + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t read = 0; (read = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, read); + return text; +} + +std::string ReadFile(const std::filesystem::path& path) { + std::ifstream stream(path, std::ios::binary); + return std::string(std::istreambuf_iterator(stream), std::istreambuf_iterator()); +} + +void ClearLogEnvironment() { +#ifdef _WIN32 + _putenv("APS5_LOG_SYNC="); + _putenv("APS5_LOG_TIMESTAMPS="); +#else + unsetenv("APS5_LOG_SYNC"); + unsetenv("APS5_LOG_TIMESTAMPS"); +#endif +} + +void LockStream(std::FILE* file) { +#ifdef _WIN32 + _lock_file(file); +#else + flockfile(file); +#endif +} + +void UnlockStream(std::FILE* file) { +#ifdef _WIN32 + _unlock_file(file); +#else + funlockfile(file); +#endif +} + +class HeldStream { +public: + explicit HeldStream(std::FILE* file) : file(file) { + holder = std::thread([this] { + LockStream(this->file); + held.store(true); + while (!releaseRequested.load()) std::this_thread::sleep_for(1ms); + released.store(true); + UnlockStream(this->file); + }); + while (!held.load()) std::this_thread::sleep_for(1ms); + } + + HeldStream(const HeldStream&) = delete; + HeldStream& operator=(const HeldStream&) = delete; + + ~HeldStream() { + Release(); + holder.join(); + } + + void Release() { + releaseRequested.store(true); + } + + bool Released() const { + return released.load(); + } + +private: + std::FILE* file; + std::atomic held{false}; + std::atomic releaseRequested{false}; + std::atomic released{false}; + std::thread holder; +}; + +std::string LargeLine(std::size_t index, std::size_t bytes) { + std::string line = "large " + std::to_string(index) + " "; + line.resize(bytes - 1, 'x'); + line += '\n'; + return line; +} + +void OrderPerThread() { + std::FILE* file = std::tmpfile(); + Check(file != nullptr, "temporary file"); + auto& log = AgcDriver::AsyncLog::Get(); + std::string expected; + for (int i = 0; i < 2000; ++i) { + log.Defer(file, "deferred %d %s %llu\n", i, AgcDriver::LogLiteral{"literal"}, static_cast(i) * 3ull); + expected += "deferred " + std::to_string(i) + " literal " + std::to_string(static_cast(i) * 3ull) + "\n"; + const std::string plain = "plain " + std::to_string(i) + "\n"; + log.Write(file, plain.data(), plain.size()); + expected += plain; + } + log.Flush(); + Check(ReadAll(file) == expected, "deferred and written lines changed order or content"); + std::fclose(file); +} + +void LongLine() { + std::FILE* file = std::tmpfile(); + Check(file != nullptr, "temporary file"); + const std::string longText(3000, 'x'); + AgcDriver::AsyncLog::Get().Defer(file, "%s|%d\n", AgcDriver::LogLiteral{longText.c_str()}, 7); + AgcDriver::AsyncLog::Get().Flush(); + Check(ReadAll(file) == longText + "|7\n", "a line longer than the stack buffer changed"); + std::fclose(file); +} + +void Coalescing() { + auto& log = AgcDriver::AsyncLog::Get(); + const std::string line = "[gpu] coalescing test line\n"; + int printed = 0; + for (int i = 0; i < 50; ++i) printed += log.Admit(line) ? 1 : 0; + Check(printed == 3, "identical lines in one window were not limited to the first three"); + Check(log.Admit("[gpu] another line\n"), "a different line was suppressed"); +} + +void FlushWaitsForHeldStream() { + std::FILE* file = std::tmpfile(); + Check(file != nullptr, "temporary file"); + auto& log = AgcDriver::AsyncLog::Get(); + { + HeldStream held(file); + log.Defer(file, "held %d %s\n", 7, AgcDriver::LogLiteral{"line"}); + std::thread releaser([&] { + std::this_thread::sleep_for(2400ms); + held.Release(); + }); + log.Flush(); + const bool waited = held.Released(); + releaser.join(); + Check(waited, "Flush returned while the logger still waited for the stream"); + } + Check(ReadAll(file) == "held 7 line\n", "the line queued behind a held stream changed"); + std::fclose(file); +} + +void BoundedFlushGivesUp() { + std::FILE* file = std::tmpfile(); + Check(file != nullptr, "temporary file"); + auto& log = AgcDriver::AsyncLog::Get(); + Check(log.FlushFor(0ms), "FlushFor waited with nothing queued"); + { + HeldStream held(file); + log.Defer(file, "bounded %d\n", 1); + Check(!log.FlushFor(100ms), "FlushFor reported a line written while its stream was held"); + held.Release(); + Check(log.FlushFor(10s), "FlushFor missed the line written after the stream was released"); + } + Check(ReadAll(file) == "bounded 1\n", "the line queued behind a held stream changed"); + std::fclose(file); +} + +void FullQueueBlocksCaller() { + std::FILE* file = std::tmpfile(); + Check(file != nullptr, "temporary file"); + auto& log = AgcDriver::AsyncLog::Get(); + constexpr std::size_t lineBytes = 1024 * 1024; + constexpr std::size_t lines = 2 * AgcDriver::AsyncLog::Capacity / lineBytes + 4; + constexpr std::size_t filled = AgcDriver::AsyncLog::Capacity / lineBytes / 2; + std::atomic queued{0}; + bool blocked = false; + { + HeldStream held(file); + std::thread producer([&] { + for (std::size_t i = 0; i < lines; ++i) { + const std::string line = LargeLine(i, lineBytes); + log.Write(file, line.data(), line.size()); + queued.store(i + 1); + } + }); + auto last = queued.load(); + auto changedAt = Clock::now(); + const auto deadline = changedAt + 10s; + while (last < lines && Clock::now() < deadline && (last < filled || Clock::now() - changedAt < 500ms)) { + std::this_thread::sleep_for(5ms); + const auto now = queued.load(); + if (now != last) { + last = now; + changedAt = Clock::now(); + } + } + blocked = last >= filled && last < lines && Clock::now() - changedAt >= 500ms; + held.Release(); + producer.join(); + } + Check(blocked, "the caller was not blocked by a full queue"); + log.Flush(); + std::fflush(file); + std::rewind(file); + std::string line; + for (std::size_t i = 0; i < lines; ++i) { + const std::string expected = LargeLine(i, lineBytes); + line.resize(expected.size()); + Check(std::fread(line.data(), 1, line.size(), file) == line.size() && line == expected, "a line written through a full queue changed or went missing"); + } + Check(std::fgetc(file) == EOF, "a full queue wrote extra output"); + std::fclose(file); +} + +constexpr int ChildLines = 50; + +[[noreturn]] void EndChild(const std::string& mode, const char* output) { + std::FILE* file = std::fopen(output, "wb"); + Check(file != nullptr, "child output file"); +#ifdef _WIN32 + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); +#else + const rlimit noCore{0, 0}; + setrlimit(RLIMIT_CORE, &noCore); +#endif + std::thread([] { + std::this_thread::sleep_for(8s); + std::_Exit(99); + }).detach(); + AgcDriver::InstallAsyncLog_nid_no_patch(); + auto& log = AgcDriver::AsyncLog::Get(); + HeldStream held(file); + for (int i = 0; i < ChildLines; ++i) { + log.Defer(file, "deferred %d\n", i); + WriteLogLine(file, "plain %d\n", i); + } + std::thread releaser([&] { + std::this_thread::sleep_for(100ms); + if (mode != "blocked") held.Release(); + }); + if (mode == "terminate") std::terminate(); +#ifdef _WIN32 + if (mode == "fault") RaiseException(EXCEPTION_ACCESS_VIOLATION, EXCEPTION_NONCONTINUABLE, 0, nullptr); +#else + if (mode == "raise") { + std::raise(SIGABRT); + std::_Exit(0); + } +#endif + std::abort(); +} + +std::string Quote(const std::string& text) { + return "\"" + text + "\""; +} + +void LinesSurviveEnd(const std::string& self, const std::string& mode) { + const auto output = std::filesystem::temp_directory_path() / ("aps5-async-log-" + mode + "-" + std::to_string(Clock::now().time_since_epoch().count()) + ".txt"); + std::string command = Quote(self) + " child " + mode + " " + Quote(output.string()); +#ifdef _WIN32 + command = Quote(command); +#endif + const auto start = Clock::now(); + const int status = std::system(command.c_str()); + const auto lived = Clock::now() - start; + std::error_code error; + const bool started = std::filesystem::exists(output, error); + const std::string text = started ? ReadFile(output) : std::string(); + std::filesystem::remove(output, error); + Check(started, "the child did not start"); + Check(status != 0, "the child did not end abnormally"); + Check(lived < 6s, "the child took too long to end"); + if (mode == "blocked") { + Check(lived >= 1900ms, "the child ended before the flush gave up"); + Check(text.empty(), "the child wrote through a held stream"); + return; + } + std::string expected; + for (int i = 0; i < ChildLines; ++i) expected += "deferred " + std::to_string(i) + "\nplain " + std::to_string(i) + "\n"; + Check(text == expected, "lines queued before the process ended were lost or changed"); +} + +} + +int main(int argc, char** argv) { + try { + ClearLogEnvironment(); + if (argc == 4 && std::strcmp(argv[1], "child") == 0) EndChild(argv[2], argv[3]); + if (argc == 2 && std::strcmp(argv[1], "fatal") == 0) { + LinesSurviveEnd(argv[0], "abort"); + LinesSurviveEnd(argv[0], "terminate"); +#ifdef _WIN32 + LinesSurviveEnd(argv[0], "fault"); +#else + LinesSurviveEnd(argv[0], "raise"); +#endif + LinesSurviveEnd(argv[0], "blocked"); + return 0; + } + OrderPerThread(); + LongLine(); + Coalescing(); + FlushWaitsForHeldStream(); + BoundedFlushGivesUp(); + FullQueueBlocksCaller(); + } catch (const std::exception& error) { + std::cerr << error.what() << std::endl; + return 1; + } + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ComputePipelineCapture.cpp b/core/libs/prx/libSceAgcDriver/tests/ComputePipelineCapture.cpp index 5324f2e432..51d13f59c5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ComputePipelineCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ComputePipelineCapture.cpp @@ -62,6 +62,10 @@ void CheckRoundTrip() { changed = decoded; changed.requiredSubgroupSize = 64; Require(changed.Identity() != capture.Identity(), "subgroup size did not affect capture identity"); + changed = decoded; + changed.stageFlags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; + Require(AgcDriver::ComputePipelineCapture::Decode(changed.Encode()).stageFlags == changed.stageFlags, "full subgroups were lost in the capture"); + Require(changed.Identity() != capture.Identity(), "full subgroups did not affect capture identity"); Directory directory; const auto file = capture.Write(directory.path); Require(AgcDriver::ComputePipelineCapture::Read(file).Encode() == encoded, "capture file changed pipeline inputs"); @@ -144,7 +148,11 @@ void CheckMalformedRecords() { invalid([](auto& copy) { copy.spirv[11] = 4; }); invalid([](auto& copy) { copy.spirv[13] = 0; }); invalid([](auto& copy) { copy.flags = VK_PIPELINE_CREATE_DERIVATIVE_BIT; }); - invalid([](auto& copy) { copy.stageFlags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; }); + invalid([](auto& copy) { copy.stageFlags = VK_PIPELINE_SHADER_STAGE_CREATE_ALLOW_VARYING_SUBGROUP_SIZE_BIT; }); + invalid([](auto& copy) { + copy.stageFlags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; + copy.requiredSubgroupSize = 0; + }); invalid([](auto& copy) { copy.requiredSubgroupSize = 3; }); invalid([](auto& copy) { copy.specialization.push_back(copy.specialization[0]); }); invalid([](auto& copy) { copy.pushConstants = {{VK_SHADER_STAGE_VERTEX_BIT, 0, 4}}; }); diff --git a/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp b/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp index 38131c019d..d660dc354d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp @@ -1,5 +1,6 @@ #include "IntermediateRepresentation/IrBuilder.hpp" #include "Optimization/ConstantFolder.hpp" +#include "Optimization/DeadCodeEliminator.hpp" #include #include #include @@ -44,6 +45,48 @@ bool reversesTo(std::uint32_t word, std::uint32_t expected) { return result->HasImmediate() && result->ImmediateBool(); } +bool vertexProjection(std::uint32_t low, std::uint32_t hostSubgroup, std::uint32_t version, bool expected, bool directLane = false, IrShaderStage stage = IrShaderStage::Vertex, bool ballotWords = false) { + IrProgram program; + program.Resources().stage = stage; + program.SetWaveSize(64u); + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder().push_back(&block); + IrBuilder builder(program); + builder.SetInsertionPoint(block); + auto& lane = builder.Emit(IrOpcode::LaneId, IrType::U32, {}); + auto& half = builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {&lane, &builder.Constant(32u)}); + IrValue* lowWord = &builder.Constant(low); + IrValue* highWord = &builder.Constant(0u); + if (ballotWords) { + auto& entry = builder.Emit(IrOpcode::Ballot, IrType::U32x4, {&builder.ConstantBool(true)}); + lowWord = &builder.Emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&entry, &builder.Constant(0u)}); + highWord = &builder.Emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&entry, &builder.Constant(1u)}); + } + auto& word = builder.Emit(IrOpcode::SelectU32, IrType::U32, {&half, lowWord, highWord}); + auto& amount = builder.Emit(IrOpcode::BitwiseAnd32, IrType::U32, {&lane, &builder.Constant(31u)}); + auto& shifted = builder.Emit(IrOpcode::ShiftRightLogical32, IrType::U32, {&word, &amount}); + auto& bit = builder.Emit(IrOpcode::BitwiseAnd32, IrType::U32, {&shifted, &builder.Constant(1u)}); + auto& test = builder.Emit(IrOpcode::INotEqual32, IrType::Bool, {&bit, &builder.Constant(0u)}); + auto& user = builder.Emit(IrOpcode::Reference, IrType::Void, {&test}); + if (directLane) static_cast(builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&lane})); + static_cast(builder.Emit(IrOpcode::Return, IrType::Void, {})); + ConstantFolder folder; + const bool folded = folder.FoldVertexLaneBounds(program, hostSubgroup, version); + const bool shouldFold = !ballotWords && stage == IrShaderStage::Vertex && hostSubgroup == 32u && version >= 0x00010000u && version < 0x00010600u; + if (folded != shouldFold) return false; + if (!shouldFold && (word.Opcode() != IrOpcode::SelectU32 || word.Argument(0)->Resolve() != &half || half.Opcode() != IrOpcode::ULessThan32)) return false; + if (shouldFold && (!word.Resolve()->HasImmediate() || word.Resolve()->ImmediateU32() != low)) return false; + folder.Fold(program); + DeadCodeEliminator{}.RemoveIdentities(program); + DeadCodeEliminator{}.Eliminate(program); + const auto* result = user.Argument(0)->Resolve(); + if (expected) return result->HasImmediate() && result->ImmediateBool(); + bool retainedLane = false; + for (const auto* instruction : block.Instructions()) retainedLane = retainedLane || instruction->Opcode() == IrOpcode::LaneId; + return retainedLane; +} + bool run(const char* name, bool passed) { if (!passed) std::fprintf(stderr, "%s\n", name); return passed; @@ -58,6 +101,18 @@ int main() { passed &= run("an all-one word must fold the lane bit test to true", foldsTo(0xffffffffu, true, true)); passed &= run("a mixed word must keep the lane bit test", foldsTo(0x0000ffffu, false, true)); passed &= run("an unmasked lane number must keep the lane bit test", foldsTo(0xffffffffu, true, false)); + passed &= run("a fixed 32-lane vertex subgroup removes a constant lower-half export guard", vertexProjection(0xffffffffu, 32u, 0x00010300u, true)); + passed &= run("a single set bit remains dependent on the vertex lane", vertexProjection(1u, 32u, 0x00010300u, false)); + passed &= run("a partial mask remains dependent on the vertex lane", vertexProjection(0x0000ffffu, 32u, 0x00010300u, false)); + passed &= run("a real vertex lane output stays observable after folding its guard", vertexProjection(0xffffffffu, 32u, 0x00010300u, false, true)); + passed &= run("a lane word selected from ballot words keeps its lower-half test", vertexProjection(0xffffffffu, 32u, 0x00010300u, false, false, IrShaderStage::Vertex, true)); + passed &= run("a 64-lane subgroup keeps a lower-half export guard", vertexProjection(0xffffffffu, 64u, 0x00010300u, false)); + passed &= run("an unknown subgroup keeps a lower-half export guard", vertexProjection(0xffffffffu, 0u, 0x00010300u, false)); + passed &= run("a single-lane request does not borrow the physical 32-lane proof", vertexProjection(0xffffffffu, 1u, 0x00010300u, false)); + passed &= run("SPIR-V 1.6 does not borrow a fixed subgroup-size proof", vertexProjection(0xffffffffu, 32u, 0x00010600u, false)); + passed &= run("an unknown module version keeps the lane guard", vertexProjection(0xffffffffu, 32u, 0u, false)); + passed &= run("the lane-bound optimization does not change compute shaders", vertexProjection(0xffffffffu, 32u, 0x00010300u, false, false, IrShaderStage::Compute)); + passed &= run("the lane-bound optimization does not change pixel shaders", vertexProjection(0xffffffffu, 32u, 0x00010300u, false, false, IrShaderStage::Pixel)); passed &= run("a reversed zero must fold to zero", reversesTo(0x00000000u, 0x00000000u)); passed &= run("a reversed single bit must fold to the opposite end", reversesTo(0x00000001u, 0x80000000u)); passed &= run("a reversed high bit must fold to bit zero", reversesTo(0x80000000u, 0x00000001u)); diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameDumpIo.cpp b/core/libs/prx/libSceAgcDriver/tests/FrameDumpIo.cpp index a23ee5531f..ec52e5f1d9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FrameDumpIo.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FrameDumpIo.cpp @@ -96,8 +96,8 @@ void PixelFailure() { } Close(reader); }); - const std::vector pixels(128 * 128 * 4); - const auto failure = AgcDriver::WriteBmpAndClose_nid_no_patch(file, 128, 128, pixels); + const std::vector pixels(512 * 512 * 4); + const auto failure = AgcDriver::WriteBmpAndClose_nid_no_patch(file, 512, 512, pixels); consumer.join(); Check(received == 54, "pixel failure did not follow a complete BMP header"); Check(failure && std::strcmp(failure.operation, "pixels") == 0, "closed pipe pixel write was accepted"); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index 2e1d26e11f..d157121472 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -3,6 +3,7 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" @@ -25,6 +26,7 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libc/include/GuestHeap.hpp" #include "ImageTableAbi.hpp" +#include "SpirvBackend/SpirvSpecialization.hpp" #include #include #include @@ -34,6 +36,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -816,6 +821,10 @@ void ShaderStageTests() { Require(state.stages.path == AgcDriver::Graphics::ShaderPath::Geometry && state.stages.vertexWaveSize == 32 && state.stages.mesh, "a fast-launch draw did not decode as wave32 geometry"); const auto& mesh = *state.stages.mesh; Require(mesh.inputPrimitive == 1 && mesh.primitivesPerGroup == 1 && mesh.verticesPerGroup == 1 && mesh.maxVertices == 32 && mesh.maxPrimitives == 64 && mesh.threadsPerGroup == 64 && mesh.ldsSizeDwords == 128, "a fast-launch subgroup did not decode as one point amplified by PRIM_AMP_FACTOR"); + fast.context[0x29b] = 1; + const auto fastLines = AgcDriver::Graphics::DecodeState(fast); + Require(fastLines.stages.mesh->outputPrimitiveSize == 2 && fastLines.stages.mesh->maxPrimitives == 64 && AgcDriver::Graphics::RasterPrimitiveOf(fastLines) == ShaderRecompiler::RasterPrimitive::Line, "a fast-launch line strip did not take its line count from PRIM_AMP_FACTOR"); + fast.context[0x29b] = 2; fast.context[0x2d5] = 0x502030; expectFailure([&] { AgcDriver::Graphics::DecodeState(fast); }, "unsupported geometry routing, fast launch or wave-ID state"); fast.context[0x2d5] = 0x482030; @@ -828,6 +837,40 @@ void ShaderStageTests() { fast.context[0x2d3] = 0; expectFailure([&] { AgcDriver::Graphics::DecodeState(fast); }, "amplifies to no primitives"); } + { + auto lines = makeState(); + lines.userConfig[0x242] = 1; + lines.context[0x2d5] = 0x2030; + lines.userConfig[0x25b] = 0x6633; + lines.context[0x1ff] = 255; + lines.context[0x2ce] = 5; + lines.context[0x29b] = 1; + lines.context[0x2ab] = 4; + lines.shader[0x8a] = 3u << 29u; + lines.shader[0x8b] = 3u << 16u; + auto state = AgcDriver::Graphics::DecodeState(lines); + Require(state.stages.path == AgcDriver::Graphics::ShaderPath::Geometry && state.stages.mesh && AgcDriver::Graphics::RasterPrimitiveOf(state) == ShaderRecompiler::RasterPrimitive::Line, "a geometry program that outputs line strips did not decode as geometry rasterized as lines"); + auto mesh = *state.stages.mesh; + Require(mesh.inputPrimitive == 1 && mesh.primitivesPerGroup == 51 && mesh.verticesPerGroup == 51 && mesh.maxVertices == 255 && mesh.maxPrimitives == 255 && mesh.threadsPerGroup == 256 && mesh.outputPrimitiveSize == 2, "a line-strip subgroup did not decode as 51 points with a line slot per output vertex"); + lines.context[0x29b] = 2; + state = AgcDriver::Graphics::DecodeState(lines); + mesh = *state.stages.mesh; + Require(mesh.maxPrimitives == 153 && mesh.threadsPerGroup == 256 && mesh.outputPrimitiveSize == 3 && AgcDriver::Graphics::RasterPrimitiveOf(state) == ShaderRecompiler::RasterPrimitive::Triangle, "the triangle-strip subgroup of the same draw changed"); + lines.context[0x29b] = 1; + lines.context[0x2ce] = 1; + expectFailure([&] { AgcDriver::Graphics::DecodeState(lines); }, "unsupported geometry input or output assembly"); + lines.context[0x2ce] = 5; + for (const auto output : {0u, 3u, 0x80000001u, 0x101u}) { + lines.context[0x29b] = output; + expectFailure([&] { AgcDriver::Graphics::DecodeState(lines); }, "VGT_GS_OUT_PRIM_TYPE other than a triangle strip, or a line strip from a geometry program, is unsupported"); + } + lines.userConfig[0x242] = 4; + lines.context[0x205] = 0x2a40; + lines.context[0x29b] = 2; + Require(AgcDriver::Graphics::DecodeState(lines).stages.mesh->primitivesPerGroup == 17, "triangle input with a triangle-strip output and a triangle polygon offset did not decode"); + lines.context[0x29b] = 1; + expectFailure([&] { AgcDriver::Graphics::DecodeState(lines); }, "polygon mode, depth bias or nonstandard rasterization"); + } queue.context[0x2ab] = 0; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "invalid VGT_ESGS_RING_ITEMSIZE"); queue.context[0x2ab] = 4; @@ -947,6 +990,9 @@ void NggVertexWorkgroupTests() { queue.context[0x2d5] = stages; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue, true); }, "requires PRIMGEN_EN without GS_EN, PRIMGEN_PASSTHRU_EN or tessellation"); } + queue.context[0x2d5] = 0x12010; + queue.context[0x29b] = 1; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue, true); }, "VGT_GS_OUT_PRIM_TYPE other than a triangle strip, or a line strip from a geometry program, is unsupported"); } constexpr std::array IdentityExports{0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u}; @@ -1886,7 +1932,12 @@ void DepthStencilTests() { queue.context[0x000] = 0x23; state = AgcDriver::Graphics::DecodeState(queue); Require(state.depthWrite && state.depthCompare == VK_COMPARE_OP_ALWAYS && clears(state.stencilFront), "a combined depth and stencil clear draw does not clear both planes"); - for (const auto control : {4u, 8u}) { + for (const auto select : {0xc0u, 0xf00u}) { + queue.context[0x000] = 0x23u | select; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depthWrite && state.depthCompare == VK_COMPARE_OP_ALWAYS && clears(state.stencilFront), "a copy sample selection without a copy changed a clear draw"); + } + for (const auto control : {4u, 8u, 0x84u, 0x108u}) { queue.context[0x000] = 0x22u | control; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "DB_RENDER_CONTROL"); } @@ -2060,7 +2111,7 @@ void memoryValueFillTests() { } void depthMaintenanceTests() { - for (const auto mode : {0x4u, 0x8u, 0x80u, 0x100u, 0x1000u, 0x4000u}) { + for (const auto mode : {0x4u, 0x8u, 0x84u, 0x108u, 0xf88u, 0x1000u, 0x4000u}) { for (const auto clear : {0u, 1u, 2u, 3u}) { auto queue = makeState(); queue.context[0x000] = mode | clear; @@ -2074,7 +2125,7 @@ void depthMaintenanceTests() { expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, reason); } } - for (const auto control : {0u, 1u, 2u, 3u, 0x20u, 0x40u, 0x2000u, 0x2063u}) { + for (const auto control : {0u, 1u, 2u, 3u, 0x20u, 0x40u, 0x80u, 0xc0u, 0x100u, 0xf00u, 0x2000u, 0x2063u, 0x2fe3u}) { auto queue = makeState(); queue.context[0x000] = control; Require(AgcDriver::Graphics::DepthMaintenanceRejection(queue).empty(), "ordinary depth controls were mistaken for maintenance"); @@ -2645,6 +2696,67 @@ void multisampleTests() { expectFailure([&] { AgcDriver::Graphics::ValidateSampleLocations(gridContext, state); }, "2x2 sample location grid"); gridContext.sampleLocationGridSampleCounts |= VK_SAMPLE_COUNT_2_BIT; Require(AgcDriver::Graphics::ValidateSampleLocations(gridContext, state), "a supported 2x2 sample location grid was refused"); + queue.context[0x31d] = 0; + state = DecodeState(queue); + Require(state.samples == 1 && state.colors.size() == 1 && state.colors[0].samples == 1 && state.customSampleLocations && state.sampleLocationGrid, "2x checkerboard rasterization into a one-sample target (PPSA27836) did not rasterize sample 0 alone"); + const auto sampleZero = AgcDriver::Graphics::SampleLocations(state); + const std::array expectedSampleZero{{{0.75f, 0.5f}, {0.75f, 0.5f}, {0.25f, 0.5f}, {0.25f, 0.5f}}}; + for (std::size_t pixel = 0; pixel < expectedSampleZero.size(); ++pixel) Require(sampleZero[pixel].x == expectedSampleZero[pixel].x && sampleZero[pixel].y == expectedSampleZero[pixel].y, "a one-sample target did not take sample 0's location in each pixel of the grid"); + gridContext.sampleLocationSampleCounts = VK_SAMPLE_COUNT_2_BIT; + gridContext.sampleLocationGridSampleCounts = VK_SAMPLE_COUNT_2_BIT; + expectFailure([&] { AgcDriver::Graphics::ValidateSampleLocations(gridContext, state); }, "VK_EXT_sample_locations"); + gridContext.sampleLocationSampleCounts |= VK_SAMPLE_COUNT_1_BIT; + expectFailure([&] { AgcDriver::Graphics::ValidateSampleLocations(gridContext, state); }, "2x2 sample location grid"); + gridContext.sampleLocationGridSampleCounts |= VK_SAMPLE_COUNT_1_BIT; + Require(AgcDriver::Graphics::ValidateSampleLocations(gridContext, state), "one-sample grid locations were refused on a device that supports them"); + for (std::uint32_t pixel = 0; pixel < 4; ++pixel) queue.context[0x2fe + pixel * 4u] = 0xcc44; + state = DecodeState(queue); + Require(state.samples == 1 && state.customSampleLocations && !state.sampleLocationGrid, "a uniform 2x pattern into a one-sample target did not use sample 0's location"); + auto stenciled = queue; + for (const auto [pixel, word] : {std::pair{0u, 0x0c04u}, std::pair{1u, 0x0c04u}, std::pair{2u, 0x040cu}, std::pair{3u, 0x040cu}}) stenciled.context[0x2fe + pixel * 4u] = word; + for (const auto [offset, word] : {std::pair{0x010u, 0x80000180u}, std::pair{0x011u, 0x20000181u}, std::pair{0x012u, 0x00001000u}, std::pair{0x013u, 0x00002000u}, std::pair{0x015u, 0x00002000u}, std::pair{0x007u, 0x003f003fu}, std::pair{0x000u, 0u}, std::pair{0x002u, 0u}, std::pair{0x00au, 0u}, std::pair{0x00bu, 0x3f800000u}, std::pair{0x10bu, 0u}, std::pair{0x10cu, 0x01ffff00u}, std::pair{0x10du, 0x01ffff00u}, std::pair{0x200u, 0x007007b3u}}) stenciled.context[offset] = word; + state = DecodeState(stenciled); + Require(state.samples == 1 && state.depth && state.depth->samples == 1 && state.stencilTest && state.sampleLocationGrid && state.colors.size() == 1 && state.colors[0].samples == 1, "2x checkerboard rasterization into one-sample stencil and color targets (PPSA27836) did not rasterize sample 0 alone"); + stenciled.context[0x31d] = 0x9000; + expectFailure([&] { DecodeState(stenciled); }, "a color target's sample count differs from the rasterizer's"); + stenciled.context[0x31d] = 0; + stenciled.context[0x1b3] |= ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCentroid); + stenciled.context[0x1b4] |= ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCentroid); + expectFailure([&] { DecodeState(stenciled); }, "the depth target's sample count differs from the rasterizer's"); + const auto withInputs = [](AgcDriver::QueueState base, std::initializer_list inputs) { + for (const auto input : inputs) { + base.context[0x1b3] |= ShaderRecompiler::PixelInputBit(input); + base.context[0x1b4] |= ShaderRecompiler::PixelInputBit(input); + } + return base; + }; + for (const auto input : {ShaderRecompiler::PixelInput::PerspectiveCentroid, ShaderRecompiler::PixelInput::LinearCentroid, ShaderRecompiler::PixelInput::SampleCoverage}) expectFailure([&] { DecodeState(withInputs(queue, {input})); }, "sample count differs from the rasterizer's"); + for (const auto input : {ShaderRecompiler::PixelInput::PerspectiveSample, ShaderRecompiler::PixelInput::LinearSample, ShaderRecompiler::PixelInput::Ancillary}) { + state = DecodeState(withInputs(queue, {input})); + Require(state.samples == 1 && !state.perSampleShading, "a sample or sample-index input into a one-sample target did not rasterize sample 0 alone"); + } + for (const auto position : {ShaderRecompiler::PixelInput::PositionX, ShaderRecompiler::PixelInput::PositionY}) expectFailure([&] { DecodeState(withInputs(queue, {ShaderRecompiler::PixelInput::Ancillary, position})); }, "sample count differs from the rasterizer's"); + auto iterated = queue; + iterated.context[0x293] = 0x06030000u; + state = DecodeState(iterated); + Require(state.samples == 1 && !state.perSampleShading, "PS_ITER_SAMPLE into a one-sample target (PPSA27836) did not rasterize sample 0 alone"); + state = DecodeState(withInputs(iterated, {ShaderRecompiler::PixelInput::PerspectiveSample, ShaderRecompiler::PixelInput::PositionX, ShaderRecompiler::PixelInput::PositionY})); + Require(state.samples == 1 && !state.perSampleShading, "PS_ITER_SAMPLE with sample and position inputs (PPSA27836) did not rasterize sample 0 alone"); + state = DecodeState(withInputs(iterated, {ShaderRecompiler::PixelInput::PerspectiveCentroid, ShaderRecompiler::PixelInput::LinearCentroid})); + Require(state.samples == 1, "PS_ITER_SAMPLE with centroid inputs did not rasterize sample 0 alone"); + expectFailure([&] { DecodeState(withInputs(iterated, {ShaderRecompiler::PixelInput::SampleCoverage})); }, "sample count differs from the rasterizer's"); + expectFailure([&] { DecodeState(withInputs(iterated, {ShaderRecompiler::PixelInput::Ancillary, ShaderRecompiler::PixelInput::PositionX})); }, "sample count differs from the rasterizer's"); + auto iteratedStencil = withInputs(stenciled, {}); + iteratedStencil.context[0x1b3] &= ~ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCentroid); + iteratedStencil.context[0x1b4] &= ~ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCentroid); + state = DecodeState(withInputs(iteratedStencil, {ShaderRecompiler::PixelInput::PerspectiveSample, ShaderRecompiler::PixelInput::Ancillary})); + Require(state.samples == 1 && state.depth && state.depth->samples == 1, "sample and sample-index inputs into a one-sample depth target (PPSA07411) did not rasterize sample 0 alone"); + iteratedStencil.context[0x293] = 0x06030000u; + state = DecodeState(iteratedStencil); + Require(state.samples == 1 && state.depth && state.depth->samples == 1 && !state.perSampleShading, "PS_ITER_SAMPLE into one-sample depth and color targets did not rasterize sample 0 alone"); + auto singleScan = queue; + singleScan.context[0x292] = 2; + expectFailure([&] { DecodeState(singleScan); }, "a color target's sample count differs from the rasterizer's"); queue = multisampled(); queue.context[0x2f8] = 0; queue.context[0x292] = 2; @@ -2785,7 +2897,8 @@ void multisampleTests() { queue.context[0x31d] = 0x1000; expectFailure([&] { DecodeState(queue); }, "fewer fragments than samples"); queue.context[0x31d] = 0; - expectFailure([&] { DecodeState(queue); }, "sample count differs"); + queue.context[0x293] = 0x10000u; + Require(DecodeState(queue).samples == 1 && !DecodeState(queue).perSampleShading, "PS_ITER_SAMPLE into a one-sample target did not rasterize sample 0 alone"); queue = multisampled(); queue.context[0x292] = 2; state = DecodeState(queue); @@ -3626,7 +3739,15 @@ VKAPI_ATTR VkResult VKAPI_CALL mockCreatePipelineLayout(VkDevice, const VkPipeli } VKAPI_ATTR VkResult VKAPI_CALL mockCreateGraphicsPipelines(VkDevice, VkPipelineCache, std::uint32_t count, const VkGraphicsPipelineCreateInfo* infos, const VkAllocationCallbacks*, VkPipeline* pipelines) { - Require(count == 1 && (infos[0].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) != 0, "mock expects a graphics pipeline library"); + Require(count == 1, "mock expects exactly one graphics pipeline per call"); + if ((infos[0].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) == 0) { + const auto* linked = static_cast(infos[0].pNext); + Require(linked != nullptr && linked->sType == VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR && linked->libraryCount != 0, "mock expects a graphics pipeline library or a pipeline linked from libraries"); + *pipelines = makeHandle(); + ++mock.pipelineCreateCount; + ++mock.live; + return VK_SUCCESS; + } const auto* library = static_cast(infos[0].pNext); Require(library != nullptr && library->sType == VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, "missing graphics pipeline library description"); mock.libraryLayouts[library->flags] = infos[0].layout; @@ -4783,6 +4904,8 @@ struct ModuleShape { std::uint32_t clipDistanceLength = 0; std::uint32_t cullDistanceLength = 0; bool layerOutput = false; + bool terminateInvocation = false; + bool omitTerminateExtension = false; }; void emit(std::vector& out, spv::Op op, std::initializer_list operands) { @@ -4977,7 +5100,7 @@ std::vector makeModule(const ModuleShape& shape) { } emit(function, spv::OpFunction, {voidType, main, 0, functionType}); emit(function, spv::OpLabel, {label}); - emit(function, spv::OpReturn, {}); + emit(function, shape.terminateInvocation ? spv::OpTerminateInvocation : spv::OpReturn, {}); emit(function, spv::OpFunctionEnd, {}); std::vector words{spv::MagicNumber, 0x10300, 0, next, 0}; emit(words, spv::OpCapability, {spv::CapabilityShader}); @@ -5003,6 +5126,14 @@ std::vector makeModule(const ModuleShape& shape) { words.resize(start + count, 0); for (std::size_t i = 0; i < extension.size(); ++i) words[start + i / 4] |= static_cast(static_cast(extension[i])) << ((i % 4) * 8); } + if (shape.terminateInvocation && !shape.omitTerminateExtension) { + const std::string extension = "SPV_KHR_terminate_invocation"; + const auto count = (extension.size() + 4) / 4; + words.push_back((static_cast(count + 1) << 16u) | spv::OpExtension); + const auto start = words.size(); + words.resize(start + count, 0); + for (std::size_t i = 0; i < extension.size(); ++i) words[start + i / 4] |= static_cast(static_cast(extension[i])) << ((i % 4) * 8); + } emit(words, spv::OpMemoryModel, {spv::AddressingModelLogical, spv::MemoryModelGLSL450}); const auto entryPointOffset = words.size(); if (shape.vertexInput) emit(words, spv::OpEntryPoint, {spv::ExecutionModelVertex, main, 0x6e69616du, 0, output, input}); @@ -5592,7 +5723,7 @@ std::string interlockShape(std::span words) { return shape; } -std::string interlockExits(std::span words, std::size_t& kills) { +std::string interlockExits(std::span words, std::size_t& kills, std::size_t& terminates) { struct Block { std::vector ops; std::vector next; @@ -5610,7 +5741,7 @@ std::string interlockExits(std::span words, std::size_t& ki if (op == spv::OpLabel) label = words[at + 1]; if (op == spv::OpLabel && function == entry && first == 0) first = label; if (function != entry) { - if (op == spv::OpBeginInvocationInterlockEXT || op == spv::OpEndInvocationInterlockEXT || op == spv::OpKill) return "an interlock instruction or OpKill outside the entry point"; + if (op == spv::OpBeginInvocationInterlockEXT || op == spv::OpEndInvocationInterlockEXT || op == spv::OpKill || op == spv::OpTerminateInvocation) return "an interlock instruction, OpKill or OpTerminateInvocation outside the entry point"; continue; } if (label == 0) continue; @@ -5634,6 +5765,7 @@ std::string interlockExits(std::span words, std::size_t& ki if (op == spv::OpEndInvocationInterlockEXT && stage++ != 1) return "block " + std::to_string(current) + " ends the interlock outside it"; if ((op == spv::OpKill || op == spv::OpReturn || op == spv::OpReturnValue || op == spv::OpUnreachable || op == spv::OpTerminateInvocation) && stage != 2) return "an exit in block " + std::to_string(current) + " skips OpEndInvocationInterlockEXT"; if (op == spv::OpKill) ++kills; + if (op == spv::OpTerminateInvocation) ++terminates; } for (const auto next : blocks[current].next) { const auto [known, added] = state.emplace(next, stage); @@ -5682,9 +5814,57 @@ void orderedPixelShaderTests() { auto faulting = orderedPixelRequest(0x30600u, bdaInterlock, OrderedBdaPixelCode); faulting.target.supportedExtensions = bdaExtensions; faulting.target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version; - std::size_t kills = 0; - const auto exits = interlockExits(ShaderRecompiler::Recompile(faulting).spirv.Words(), kills); - Require(exits.empty() && kills == 6u, "a primitive-ordered pixel shader with BDA reads: " + (exits.empty() ? std::to_string(kills) + " OpKill, not the valid-mask kill and 5 BDA fault kills" : exits)); + std::size_t kills = 0, terminates = 0; + const auto exits = interlockExits(ShaderRecompiler::Recompile(faulting).spirv.Words(), kills, terminates); + Require(exits.empty() && kills == 6u && terminates == 0u, "a primitive-ordered pixel shader with BDA reads: " + (exits.empty() ? std::to_string(kills) + " OpKill and " + std::to_string(terminates) + " OpTerminateInvocation, not the valid-mask kill and 5 BDA fault kills" : exits)); + const std::array terminateExtensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage", "SPV_KHR_terminate_invocation"}; + auto terminating = faulting; + terminating.target.supportedExtensions = terminateExtensions; + const auto terminated = ShaderRecompiler::Recompile(terminating).spirv.Copy(); + std::size_t terminatedKills = 0, terminatedStops = 0; + const auto terminatedExits = interlockExits(terminated, terminatedKills, terminatedStops); + Require(terminatedExits.empty() && terminatedKills == 1u && terminatedStops == 5u, "a primitive-ordered pixel shader with BDA reads and SPV_KHR_terminate_invocation: " + (terminatedExits.empty() ? std::to_string(terminatedKills) + " OpKill and " + std::to_string(terminatedStops) + " OpTerminateInvocation, not the valid-mask kill and 5 BDA fault terminations" : terminatedExits)); + std::size_t terminateDeclarations = 0; + for (std::size_t at = 5; at < terminated.size() && (terminated[at] >> 16u) != 0; at += terminated[at] >> 16u) { + if ((terminated[at] & 0xffffu) == spv::OpExtension && std::string_view(reinterpret_cast(&terminated[at + 1])) == "SPV_KHR_terminate_invocation") ++terminateDeclarations; + } + Require(terminateDeclarations == 1u, "the module declared SPV_KHR_terminate_invocation " + std::to_string(terminateDeclarations) + " times"); + const auto countOps = [](std::span words) { + std::map counts; + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) ++counts[static_cast(words[at] & 0xffffu)]; + return counts; + }; + const std::array bdaCapabilities{spv::CapabilityShader, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + auto unordered = orderedPixelRequest(0x20600u, bdaCapabilities, OrderedBdaPixelCode); + unordered.target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version; + unordered.target.supportedExtensions = terminateExtensions; + const auto unorderedOps = countOps(ShaderRecompiler::Recompile(unordered).spirv.Words()); + Require(!unorderedOps.contains(spv::OpBeginInvocationInterlockEXT), "a pixel shader without PRIMITIVE_ORDERED_PIXEL_SHADER began the interlock"); + Require(unorderedOps.at(spv::OpTerminateInvocation) == 5u && unorderedOps.at(spv::OpKill) == 1u, "a pixel shader with BDA reads and SPV_KHR_terminate_invocation did not terminate on a fault"); + unordered.target.supportedExtensions = bdaExtensions; + const auto unorderedKills = countOps(ShaderRecompiler::Recompile(unordered).spirv.Words()); + Require(!unorderedKills.contains(spv::OpTerminateInvocation) && unorderedKills.at(spv::OpKill) == 6u, "a pixel shader with BDA reads and no SPV_KHR_terminate_invocation did not kill on a fault"); + static constexpr std::array vertexBdaCode{0x7e040f00u, 0x7e060280u, 0xdc208001u, 0x047d0002u, 0xdc308008u, 0x057d0002u, 0xbf8c3f70u, 0x4a080b04u, 0x7e0e0d04u, 0xf80008cfu, 0x07070707u, 0xbf810000u}; + const std::array vertexCapabilities{spv::CapabilityShader, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityClipDistance}; + const auto vertexWords = [&](std::span extensions) { + ShaderRecompiler::RecompileRequest request{}; + request.shader = {ShaderRecompiler::ShaderStage::Vertex, 0x10000u, vertexBdaCode, 0, {}}; + request.context.waveSize = 64; + request.context.vertex = ShaderRecompiler::ShaderVertexStageInfo{}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 64; + request.target.supportedCapabilities = vertexCapabilities; + request.target.supportedExtensions = extensions; + request.target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return ShaderRecompiler::Recompile(request).spirv.Copy(); + }; + const auto vertexStops = vertexWords(bdaExtensions); + const auto vertexTerminate = vertexWords(terminateExtensions); + const auto vertexOps = countOps(vertexTerminate); + Require(std::ranges::equal(vertexStops, vertexTerminate) && !vertexOps.contains(spv::OpTerminateInvocation) && !vertexOps.contains(spv::OpKill) && vertexOps.at(spv::OpReturn) > 1u, "a vertex shader with BDA reads changed its fault exit when SPV_KHR_terminate_invocation was advertised"); expectFailure([&] { static_cast(ShaderRecompiler::Recompile(orderedPixelRequest(0x30600u, plain))); }, "fragmentShaderPixelInterlock"); } @@ -6677,6 +6857,34 @@ void validationTests() { vertex.spirv = makeModule({.barycentric = true}); pixel.spirv = makeModule({.fragment = true}); expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, true); }, "requires a fragment shader"); + const auto validateTerminate = [&](bool enabled) { + return AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false, false, false, false, false, false, false, false, false, false, false, enabled); + }; + vertex.spirv = makeModule({}); + pixel.spirv = makeModule({.fragment = true, .terminateInvocation = true}); + validateTerminate(true); + expectFailure([&] { validateTerminate(false); }, "shaderTerminateInvocation"); + vertex.spirv = makeModule({.terminateInvocation = true}); + pixel.spirv = makeModule({.fragment = true}); + expectFailure([&] { validateTerminate(true); }, "fragment shader"); + vertex.spirv = makeModule({}); + pixel.spirv = makeModule({.fragment = true, .terminateInvocation = true, .omitTerminateExtension = true}); + expectFailure([&] { validateTerminate(true); }, "OpTerminateInvocation requires SPV_KHR_terminate_invocation"); + expectFailure([&] { validateTerminate(false); }, "OpTerminateInvocation requires enabled shaderTerminateInvocation"); + vertex.spirv = makeModule({.terminateInvocation = true, .omitTerminateExtension = true}); + pixel.spirv = makeModule({.fragment = true}); + expectFailure([&] { validateTerminate(true); }, "OpTerminateInvocation requires enabled shaderTerminateInvocation in a fragment shader"); + vertex.spirv = makeModule({}); + pixel.spirv = ShaderRecompiler::SpecializeSpirv(makeModule({.fragment = true, .terminateInvocation = true})); + std::size_t terminates = 0; + std::size_t declarations = 0; + for (std::size_t at = 5; at < pixel.spirv.size(); at += pixel.spirv[at] >> 16u) { + const auto op = pixel.spirv[at] & 0xffffu; + if (op == spv::OpTerminateInvocation) ++terminates; + if (op == spv::OpExtension && std::string_view(reinterpret_cast(&pixel.spirv[at + 1])) == "SPV_KHR_terminate_invocation") ++declarations; + } + Require(terminates == 1u && declarations == 1u, "specialization dropped OpTerminateInvocation or its extension declaration"); + validateTerminate(true); } { ShaderRecompiler::RecompileResult vertex; @@ -7251,6 +7459,291 @@ void ResourceContentKeyTests(bool benchmark) { } } +// The pipeline journal without a GPU: the recipe codec and the store key agree with each other and +// with the canonical state, identities follow the module and the specialization, the journal +// records each pipeline once, and a replay into the recording context (the lavapipe test replays +// into a second device) rebuilds the recorded graphics pipeline so the same configuration hits the +// store, hands the compute result to its hook, recovers a tail torn inside a record's padding and +// stops when cancelled. +void pipelineJournalTests() { + using namespace AgcDriver::Graphics; + using Stage = ShaderRecompiler::ShaderStage; + const auto directory = std::filesystem::temp_directory_path() / ("aps5-pipeline-journal-mock-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(directory); + struct Cleanup { + std::filesystem::path directory; + ~Cleanup() { + std::error_code error; + std::filesystem::remove_all(directory, error); + } + } cleanup{directory}; +#ifdef _WIN32 + _putenv_s("ANYPS5_NO_SHADER_CACHE", "0"); + _putenv_s("ANYPS5_SHADER_CACHE_DIR", directory.string().c_str()); + _putenv_s("APS5_PIPELINE_JOURNAL_THREADS", "2"); +#else + setenv("ANYPS5_NO_SHADER_CACHE", "0", 1); + setenv("ANYPS5_SHADER_CACHE_DIR", directory.string().c_str(), 1); + setenv("APS5_PIPELINE_JOURNAL_THREADS", "2", 1); +#endif + Require(PipelineJournalEnabled(), "the journal is disabled with a shader cache directory set"); + const auto module = [](std::uint32_t salt) { + ShaderRecompiler::RecompileResult result; + result.spirv = std::vector{0x07230203u, 0x00010000u, 0u, 8u, 0u, 0x00020011u, 1u, 0x00020011u, 2u + salt}; + result.runtimeAbiVersion = ShaderRecompiler::RuntimeAbi::Version; + return result; + }; + { + auto first = module(1); + const auto& constFirst = first; + auto copy = first; + const auto& constCopy = copy; + Require(!constFirst.Identity().Empty() && constFirst.Identity() == constCopy.Identity() && constFirst.spirv.data() == constCopy.spirv.data(), "a copied result did not share its module and identity"); + copy.spirv[8] = 99; + Require(!(constCopy.Identity() == constFirst.Identity()) && constCopy.spirv.data() != constFirst.spirv.data() && constFirst.Identity() == module(1).Identity(), "writing a copied module did not give it an identity of its own"); + auto specialized = module(1); + specialized.specialization.push_back({3, 4}); + Require(!(specialized.Identity() == constFirst.Identity()), "specialization constants did not enter the identity"); + auto subgroup = module(1); + subgroup.hostSubgroupSize = 64; + Require(!(subgroup.Identity() == constFirst.Identity()), "the host subgroup size did not enter the identity"); + Require(ShaderRecompiler::RecompileResult{}.Identity().Empty(), "a result without a module has an identity"); + } + mock = MockVulkan{}; + auto context = mockContext(); + context.device = makeHandle(); + context.graphicsPipelineLibrary = true; + context.tessellationShader = true; + context.limits.maxTessellationPatchSize = 32; + context.limits.maxColorAttachments = 8; + context.limits.maxPushConstantsSize = 256; + context.limits.maxVertexInputBindings = 16; + context.limits.maxVertexInputAttributes = 16; + context.limits.maxFramebufferWidth = 4096; + context.limits.maxFramebufferHeight = 4096; + context.limits.maxFramebufferLayers = 1; + std::uint64_t replayedComputeIdentityLow = 0; + { + DescriptorCache cache(context); + context.descriptorCache = &cache; + auto vertex = module(1); + vertex.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 1, std::vector(4))); + auto fragment = module(2); + fragment.pushConstants.resize(8); + fragment.bindings.push_back(makeBinding(Role::ShaderData, 0, 1, std::vector(4))); + fragment.bindings.back().descriptorSet = ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet; + const std::array shaders{{{Stage::Vertex, &vertex, 0}, {Stage::Fragment, &fragment, PipelinePushSlotBytes}}}; + State state{}; + state.stages = {ShaderPath::Vertex, 7u, 64u, 32u, std::nullopt, std::nullopt}; + state.stages.tessellation = ShaderRecompiler::TessellationConfiguration{3, 3, 2, 1, 4, 128, 16, true}; + ColorTarget color{}; + color.address = 0x1000; + color.extent = {64, 32}; + color.format = VK_FORMAT_R8G8B8A8_UNORM; + color.bytes = 64 * 32 * 4; + color.componentMapping = 0xec; + state.colors = {color, color}; + state.colors[1].format = VK_FORMAT_B8G8R8A8_UNORM; + state.colors[1].exportIndex = 1; + state.color = color; + state.hasColorTarget = true; + VkPipelineColorBlendAttachmentState blend{}; + blend.srcColorBlendFactor = VK_BLEND_FACTOR_SRC_ALPHA; + blend.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA; + blend.colorWriteMask = 0xf; + state.blends = {blend, blend}; + state.blend = blend; + state.blendConstants = {0.25f, 0.5f, 0.75f, 1.0f}; + DepthTarget depth{}; + depth.address = 0x2000; + depth.format = VK_FORMAT_D32_SFLOAT; + depth.samples = 1; + state.depth = depth; + state.depthTest = true; + state.depthWrite = true; + state.depthCompare = VK_COMPARE_OP_LESS_OR_EQUAL; + state.stencilTest = true; + state.stencilFront = {VK_STENCIL_OP_KEEP, VK_STENCIL_OP_REPLACE, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_ALWAYS, 0xff, 0xff, 1}; + state.stencilBack = state.stencilFront; + state.renderExtent = {64, 32}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; + state.viewport = {0, 32, 64, -32, 0, 1}; + state.scissor = {{0, 0}, {64, 32}}; + state.cullMode = VK_CULL_MODE_BACK_BIT; + state.frontFace = VK_FRONT_FACE_CLOCKWISE; + state.samples = 1; + const VertexInputLayout input{{{0, 16, VK_VERTEX_INPUT_RATE_VERTEX}}, {{0, 0, VK_FORMAT_R32G32B32A32_SFLOAT, 0}}}; + const auto layouts = DescribeSetLayouts(context, shaders); + Require(layouts.layouts.split, "a fragment stage with the fragment set did not split the set layouts"); + const auto recipe = MakeGraphicsPipelineRecipe(state, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + Require(recipe.stages.size() == 2 && recipe.stages[0].identity == vertex.Identity() && recipe.stages[1].identity == fragment.Identity() && !recipe.stages[0].generated && recipe.stages[1].pushConstantOffset == PipelinePushSlotBytes && recipe.splitSets, "the recipe does not name its stages"); + Require(recipe.state.colors.size() == 2 && recipe.state.colors[0].address == 0 && recipe.state.colors[0].componentMapping == 0xec && recipe.state.depth && recipe.state.depth->address == 0 && recipe.state.viewport.width == 0 && recipe.state.stages.registerValue == 0 && recipe.state.stages.tessellation && recipe.state.stages.tessellation->breakAtInstance, "the canonical state kept or dropped the wrong fields"); + const auto bytes = EncodeGraphicsPipelineRecipe(recipe); + GraphicsPipelineRecipe decoded; + Require(DecodeGraphicsPipelineRecipe(bytes, decoded), "the recipe does not decode"); + Require(EncodeGraphicsPipelineRecipe(decoded) == bytes, "the recipe does not round-trip byte for byte"); + for (std::size_t length = 0; length < bytes.size(); length += 7) { + GraphicsPipelineRecipe scratch; + Require(!DecodeGraphicsPipelineRecipe(std::span(bytes).first(length), scratch), "a truncated recipe decoded"); + } + std::vector original, canonical, replayed, moved, mapped; + BuildPipelineStoreKey(original, context, state, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + BuildPipelineStoreKey(canonical, context, recipe.state, recipe.vertexInput, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + const PipelineSetLayouts decodedLayouts{layouts.layouts.main, layouts.layouts.fragment, decoded.splitSets, decoded.mainLayoutKey, decoded.fragmentLayoutKey}; + BuildPipelineStoreKey(replayed, context, decoded.state, decoded.vertexInput, decodedLayouts, shaders, decoded.attachmentLayout); + Require(!original.empty() && original == canonical && original == replayed, "the canonical or the decoded state keys another pipeline"); + auto gridState = state; + gridState.samples = 2; + gridState.customSampleLocations = true; + gridState.sampleLocationGrid = true; + gridState.perSampleShading = true; + gridState.sampleLocations[31] = 0x5a; + GraphicsPipelineRecipe gridDecoded; + Require(DecodeGraphicsPipelineRecipe(EncodeGraphicsPipelineRecipe(MakeGraphicsPipelineRecipe(gridState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL)), gridDecoded), "the sample grid recipe does not decode"); + Require(gridDecoded.state.sampleLocationGrid && gridDecoded.state.perSampleShading && gridDecoded.state.sampleLocations[31] == 0x5a, "the recipe dropped the sample grid, per-sample shading or the last sample location"); + std::vector grid, gridReplayed; + BuildPipelineStoreKey(grid, context, gridState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + BuildPipelineStoreKey(gridReplayed, context, gridDecoded.state, gridDecoded.vertexInput, layouts.View(), shaders, gridDecoded.attachmentLayout); + Require(grid != original && grid == gridReplayed, "the decoded sample grid state keys another pipeline"); + auto lineState = state; + lineState.stages.mesh = ShaderRecompiler::MeshConfiguration{1, 2, 2, 4, 4, 32, 64, 1, 4, 2}; + GraphicsPipelineRecipe lineDecoded; + Require(DecodeGraphicsPipelineRecipe(EncodeGraphicsPipelineRecipe(MakeGraphicsPipelineRecipe(lineState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL)), lineDecoded), "the line strip mesh recipe does not decode"); + Require(lineDecoded.state.stages.mesh && lineDecoded.state.stages.mesh->outputPrimitiveSize == 2u && lineDecoded.state.stages.mesh->esgsItemSize == 4u, "the recipe dropped the mesh output primitive size"); + auto fourSampleState = state; + fourSampleState.samples = 4; + auto maskState = fourSampleState; + maskState.quadSampleMask = {0x3u, 0x3u, 0x3u, 0x3u}; + maskState.stencilReplayReference = 7u; + maskState.reversedBlend = 1u; + GraphicsPipelineRecipe maskDecoded; + Require(DecodeGraphicsPipelineRecipe(EncodeGraphicsPipelineRecipe(MakeGraphicsPipelineRecipe(maskState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL)), maskDecoded), "the sample mask recipe does not decode"); + Require(maskDecoded.state.quadSampleMask == maskState.quadSampleMask && maskDecoded.state.stencilReplayReference.has_value() && maskDecoded.state.reversedBlend == 1u, "the recipe dropped the quad sample masks, the replayed stencil reference or the reversed blend"); + std::vector fourSamples, masked, maskReplayed; + BuildPipelineStoreKey(fourSamples, context, fourSampleState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + BuildPipelineStoreKey(masked, context, maskState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + BuildPipelineStoreKey(maskReplayed, context, maskDecoded.state, maskDecoded.vertexInput, layouts.View(), shaders, maskDecoded.attachmentLayout); + Require(masked != fourSamples && masked == maskReplayed, "the decoded sample mask state keys another pipeline"); + auto movedState = state; + movedState.viewport.x = 5; + movedState.colors[0].address = 0x9000; + movedState.renderLayers = 4; + BuildPipelineStoreKey(moved, context, movedState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + Require(moved == original, "a field the pipeline does not read changed the key"); + auto mappedState = state; + mappedState.colors[0].componentMapping = 0x24; + BuildPipelineStoreKey(mapped, context, mappedState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + Require(mapped != original, "the component mapping did not enter the key"); + std::vector unclamped, clamped, biased; + auto clampedState = state; + clampedState.depthBiasClamp = 1.0f; + BuildPipelineStoreKey(clamped, context, clampedState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + Require(clamped == original, "a depth bias clamp without depth bias changed the key"); + clampedState.depthBias = true; + BuildPipelineStoreKey(clamped, context, clampedState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + clampedState.depthBiasClamp = 0.0f; + BuildPipelineStoreKey(unclamped, context, clampedState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + Require(unclamped != original && clamped != unclamped && CanonicalPipelineState(clampedState).depthBias && CanonicalPipelineState(mappedState).depthBiasClamp == 0.0f, "the depth bias clamp did not enter the key"); + clampedState.depthBiasClamp = 2.0f; + BuildPipelineStoreKey(biased, context, clampedState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL); + Require(biased == clamped, "the depth bias clamp's value entered the key"); + auto compute = module(3); + compute.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 2, std::vector(8))); + compute.pushConstants.resize(16); + compute.hostSubgroupSize = 32; + constexpr std::uint64_t fingerprint = 0x5eed'0000'0000'0001ull; + { + auto bound = StartPipelineJournalReplay(context, fingerprint, {}); + Require(bound != nullptr, "the first replay did not start"); + bound->Wait(); + Require(bound->Done() && PipelineJournalTotals().loadedRecipes == 0, "an absent journal loaded pipelines"); + } + const auto before = PipelineJournalTotals(); + RecordGraphicsPipeline(recipe, shaders); + RecordComputePipeline(compute, true); + FlushPipelineJournal(); + const auto recorded = PipelineJournalTotals(); + Require(recorded.recordedGraphics == before.recordedGraphics + 1 && recorded.recordedCompute == before.recordedCompute + 1 && recorded.recordedModules == before.recordedModules + 3 && recorded.writeFailures == 0, "the journal did not record one graphics and one compute pipeline with three modules"); + RecordGraphicsPipeline(recipe, shaders); + RecordComputePipeline(compute, true); + FlushPipelineJournal(); + const auto repeated = PipelineJournalTotals(); + Require(repeated.recordedGraphics == recorded.recordedGraphics && repeated.recordedCompute == recorded.recordedCompute && repeated.recordedModules == recorded.recordedModules, "a repeated pipeline was recorded again"); + Require(std::filesystem::file_size(PipelineJournalPath()) > 0, "the journal file is empty"); + const auto storeBefore = PipelineStoreTotals(); + std::vector receivedWords; + ShaderRecompiler::RecompileResult received; + bool receivedPush = false; + std::uint32_t computeCalls = 0; + PipelineJournalHooks hooks{[&](const ShaderRecompiler::RecompileResult& shader, bool pushConstants) { + ++computeCalls; + received = shader; + receivedPush = pushConstants; + return true; + }}; + auto replay = StartPipelineJournalReplay(context, fingerprint, hooks); + Require(replay != nullptr, "the replay did not start"); + replay->Wait(); + Require(replay->Done(), "the replay did not finish"); + const auto totals = PipelineJournalTotals(); + Require(totals.loadedRecipes == 2 && totals.loadedModules == 3 && totals.droppedRecords == 0, "the replay did not load the two recorded pipelines and three modules"); + Require(totals.replayedGraphics == 1 && totals.replayedCompute == 1 && totals.replayRejected == 0 && totals.replayFailed == 0, "the replay did not build one graphics and one compute pipeline"); + Require(computeCalls == 1 && receivedPush && received.Identity() == compute.Identity() && received.spirv == compute.spirv && received.bindings.size() == 1 && received.bindings[0].count == 2 && received.pushConstants.size() == 16 && received.hostSubgroupSize == 32, "the compute hook did not receive the recorded result"); + replayedComputeIdentityLow = received.Identity().low; + const auto storeReplayed = PipelineStoreTotals(); + Require(storeReplayed.misses == storeBefore.misses + 1 && storeReplayed.cached == storeBefore.cached + 1, "the replayed graphics pipeline did not enter the store"); + const auto created = mock.pipelineCreateCount; + bool built = true; + const auto pipeline = CachedPipeline(context, state, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL, &built); + const auto storeHit = PipelineStoreTotals(); + Require(pipeline != nullptr && !built && mock.pipelineCreateCount == created && storeHit.hits == storeReplayed.hits + 1 && storeHit.misses == storeReplayed.misses, "the draw's configuration did not find the replayed pipeline"); + { + const auto journalPath = PipelineJournalPath(); + const auto whole = std::filesystem::file_size(journalPath); + Require(whole % 8 == 0 && whole > 3, "the journal file is not record aligned"); + std::filesystem::resize_file(journalPath, whole - 3); + auto later = module(4); + later.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 1, std::vector(4))); + RecordComputePipeline(later, false); + FlushPipelineJournal(); + Require(std::filesystem::file_size(journalPath) % 8 == 0 && PipelineJournalTotals().writeFailures == 0, "an append after a tail torn inside a record's padding left the file unaligned"); + auto torn = StartPipelineJournalReplay(context, fingerprint, hooks); + Require(torn != nullptr, "the replay after the torn tail did not start"); + torn->Wait(); + const auto reloaded = PipelineJournalTotals(); + Require(reloaded.droppedRecords == 0 && reloaded.loadedModules == 4 && reloaded.loadedRecipes == 2 && reloaded.replayFailed == 0, "the journal did not recover from a tail torn inside a record's padding"); + RecordComputePipeline(compute, true); + FlushPipelineJournal(); + Require(PipelineJournalTotals().recordedCompute == reloaded.recordedCompute + 1 && PipelineJournalTotals().recordedModules == reloaded.recordedModules, "the pipeline whose record was torn was not recorded again, or its module was"); + } + { + const auto before = PipelineJournalTotals(); + auto cancelled = StartPipelineJournalReplay(context, fingerprint, hooks); + Require(cancelled != nullptr, "the third replay did not start"); + cancelled->Cancel(); + cancelled->Wait(); + const auto after = PipelineJournalTotals(); + Require(cancelled->Done() && after.replayFailed == 0 && after.replayRejected == 0 && after.replayedGraphics + after.replayPresent <= before.replayedGraphics + before.replayPresent + 1, "a cancelled replay did not stop cleanly"); + } + { + const std::uint32_t meshWithoutOutputSize = 2u; + std::fstream file(PipelineJournalPath(), std::ios::in | std::ios::out | std::ios::binary); + file.seekp(4); + file.write(reinterpret_cast(&meshWithoutOutputSize), sizeof(meshWithoutOutputSize)); + file.close(); + Require(file.good(), "the journal header could not be rewritten"); + auto stale = StartPipelineJournalReplay(context, fingerprint, hooks); + Require(stale != nullptr, "the replay of a journal in the previous format did not start"); + stale->Wait(); + const auto discarded = PipelineJournalTotals(); + Require(discarded.loadedRecipes == 0 && discarded.loadedModules == 0, "a journal in format 2, whose mesh records lack the output primitive size, was loaded"); + } + ClearCachedPipelines(context.device); + } + Require(replayedComputeIdentityLow != 0, "the replayed compute identity is empty"); + Require(mock.live == 0, "the pipeline journal test leaked Vulkan objects"); +} + int main(int argc, char** argv) { #ifdef _WIN32 _putenv_s("APS5_PIN_WAIT_MS", "200"); @@ -7264,6 +7757,11 @@ int main(int argc, char** argv) { stencilReferenceUnusedTests(); return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--pipeline-journal") { + pipelineJournalTests(); + std::cout << "pipeline journal recipe, record and replay tests passed\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--sparse-buffer-holes") { bool passed = false; { diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp index 6a8e0d8e7a..34c28d4149 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp @@ -104,6 +104,10 @@ void RunGuestTextureResourceTests() { Fields nullType = base; nullType.typeRaw = 0; Require(IsNullTextureResource(pack(nullType)), "a type-0 texture with a nonzero base was not classified as null"); + nullType.typeRaw = 1; + Require(IsNullTextureResource(pack(nullType)), "a type-1 texture with a nonzero base was not classified as null"); + nullType.typeRaw = 2; + Require(!IsNullTextureResource(pack(nullType)), "a reserved type-2 texture was classified as null"); auto result = DecodeTextureResource(pack(base)); Require(result.baseAddress == (base.base40 << 8u), "decoded base address changed"); Require(result.width == 16 && result.height == 16, "decoded width or height changed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp b/core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp index de92d1672f..99f1f6e31c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp @@ -40,6 +40,7 @@ void Expect(bool condition, const std::string& what) { constexpr std::uint64_t Alignment = 65536; constexpr std::uint64_t ChunkBytes = std::uint64_t{64} << 20u; +constexpr std::uint64_t LargestTestedLimit = std::uint64_t{1} << 30u; struct Bind { VkDeviceSize resourceOffset; @@ -290,6 +291,10 @@ int main() { return 77; } close(udmabuf); + if (UdmabufLimitBytes() > LargestTestedLimit) { + std::printf("skipped, udmabuf size_limit_mb is above %llu: a range above it is too large to map\n", static_cast(LargestTestedLimit >> 20u)); + return 77; + } setenv("APS5_WRITE_WATCH_IMPORTS", "unwatch", 1); GuestArena::GuestArenaSetSharedBacking_nid_postfix(Resolve); try { diff --git a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp index 9a9b548f49..1e59006cd6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace { @@ -41,6 +42,13 @@ alignas(256) constexpr std::array FastLaunchCode{ 0xbf810000, }; +alignas(256) constexpr std::array LineStripCode{ + 0xd7650000, 0x000100c1, 0x7e000505, 0x93018300, 0x4a020001, 0xe0382000, 0x80020201, 0xe0382010, + 0x80020601, 0xb07c2003, 0xbf900009, 0xbf8c3f70, 0x7d880083, 0xbefe036a, 0xf80008cf, 0x05040302, + 0xf800020f, 0x09080706, 0x7d880082, 0xbefe036a, 0x161400ff, 0x00000401, 0x4a1414ff, 0x00000400, + 0xf8000941, 0x0000000a, 0xbf810000, +}; + alignas(256) constexpr std::array PixelCode{ 0xc8020002, 0xc8060102, 0xc80a0202, 0xc80e0302, 0xf800180f, 0x03020100, 0xbf810000, }; @@ -85,6 +93,13 @@ alignas(256) std::array Indices{}; alignas(256) std::array Ordered{}; alignas(256) std::array Strip{}; +constexpr std::uint32_t LineStrips = 4; +alignas(256) std::array LineStripVertices{}; + +std::uint32_t LineStripRow(std::uint32_t strip) { + return 16u + 28u * strip; +} + constexpr std::uint32_t FanRim = 8; constexpr float FanStep = 6.28318530718f / FanRim; alignas(256) std::array Fan{}; @@ -279,6 +294,17 @@ int main() { const float y = i % 2u == 0u ? -1.0f : 0.0f; Strip[i] = {{x, y, 0.5f, 1.0f}, stripColor}; } + for (std::uint32_t strip = 0; strip < LineStrips; ++strip) { + const auto color = TriangleColor(strip); + const std::array rgba{color[0] / 255.0f, color[1] / 255.0f, color[2] / 255.0f, 1.0f}; + const float top = 1.0f - (static_cast(LineStripRow(strip)) + 0.5f) / (Height / 2.0f); + const float bottom = top - 20.0f / (Height / 2.0f); + const float left = 24.5f / (Width / 2.0f) - 1.0f; + const float right = 168.5f / (Width / 2.0f) - 1.0f; + LineStripVertices[3 * strip] = {{left, top, 0.5f, 1.0f}, rgba}; + LineStripVertices[3 * strip + 1] = {{right, top, 0.5f, 1.0f}, rgba}; + LineStripVertices[3 * strip + 2] = {{right, bottom, 0.5f, 1.0f}, rgba}; + } Fan[0] = {{0.0f, 0.0f, 0.5f, 1.0f}, {1.0f, 1.0f, 1.0f, 1.0f}}; for (std::uint32_t k = 1; k <= FanRim; ++k) { const auto color = TriangleColor(k); @@ -360,6 +386,24 @@ int main() { } Require(fastLaunchIndexedRefused, "an indexed fast-launch draw was drawn"); + ClearPixels(); + auto lineSubgroup = FastLaunchSubgroup; + lineSubgroup.maxPrimitives = 2u; + lineSubgroup.outputPrimitiveSize = 2u; + DrawMesh(device, {lineSubgroup, {0, LineStrips, 0, 1, 0, false}, VertexBufferDescriptor(LineStripVertices.data(), static_cast(LineStripVertices.size())), ShaderRecompiler::MeshDrawPushOffsetBytes, false, 32u, VK_CULL_MODE_NONE, LineStripCode, 0x482030u}); + const auto linePixel = [](std::uint32_t x, std::uint32_t y) { return (static_cast(y) * Width + x) * 4u; }; + for (std::uint32_t strip = 0; strip < LineStrips; ++strip) { + const auto row = LineStripRow(strip); + const auto color = TriangleColor(strip); + const auto where = "line strip " + std::to_string(strip) + ": "; + for (std::uint32_t x = 32; x <= 160; x += 8) Require(PixelIs(linePixel(x, row), color), where + "horizontal line pixel (" + std::to_string(x) + ", " + std::to_string(row) + ") is " + PixelText(linePixel(x, row))); + for (std::uint32_t y = row + 4u; y <= row + 16u; y += 4) Require(PixelIs(linePixel(168, y), color), where + "vertical line pixel (168, " + std::to_string(y) + ") is " + PixelText(linePixel(168, y))); + for (const auto& [x, y, what] : {std::tuple{96u, row - 3u, "above the horizontal line"}, std::tuple{96u, row + 3u, "below the horizontal line"}, std::tuple{150u, row + 5u, "inside the triangle of the strip's three vertices"}, std::tuple{96u, row + 10u, "on the edge closing the strip"}, std::tuple{96u, row + 24u, "between this strip and the next"}, std::tuple{8u, row, "left of the strip"}, std::tuple{184u, row + 10u, "right of the strip"}}) { + Require(PixelIs(linePixel(x, y), Background), where + "pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") " + what + " is " + PixelText(linePixel(x, y))); + } + } + Require(PixelIs(0, Background) && PixelIs(Pixels.size() - 4u, Background), "line strips: the corners changed"); + constexpr std::size_t recordBlockBytes = 65536; auto* record = static_cast(::operator new(recordBlockBytes, std::align_val_t{65536})); { diff --git a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp index 39131eb9ca..00359e947e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp @@ -3,16 +3,27 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawRegisterKey.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp" +#include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libSceAgcDriver/Submit/include/Acb.hpp" +#include "prx/libc/include/FailureReport.hpp" #include "prx/libc/include/Shutdown.hpp" +#include #include #include +#include #include #include +#include +#include #include +#include +#include #include #include +#include +#include #include #include #include @@ -21,6 +32,9 @@ #define NOMINMAX #endif #include +#else +#include +#include #endif extern "C" int APS5_VABI sceKernelAvailableFlexibleMemorySize(size_t* size); @@ -62,18 +76,22 @@ void testCatalog() { for (const auto& opcode : AgcDriver::Pm4::Opcodes) { check(values.insert(opcode.value).second, "duplicate PM4 opcode"); const auto packet = makePacket(opcode.value, {0}); - check(AgcDriver::Pm4::Name(packet[0]) == opcode.name, "opcode name mismatch"); + check(AgcDriver::Pm4::Name(packet[0]) == opcode.name && AgcDriver::Pm4::KnownName(packet[0]) == opcode.name, "opcode name mismatch"); const auto reason = AgcDriver::Pm4::UnsupportedReason(packet[0]); if (!reason.empty()) expectFailure([&] { AgcDriver::Pm4::Validate(packet, 0); }, std::string(reason).c_str()); } check(values.size() == 56, "reference opcode catalog is incomplete"); expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0xff, {0}), 0); }, "not known"); + check(AgcDriver::Pm4::KnownName(makePacket(0xff, {0})[0]).empty() && AgcDriver::Pm4::Name(makePacket(0xff, {0})[0]) == "UNKNOWN_0xff", "an unknown opcode is named"); const std::array, 11> custom{{ {5, "DRAW_RESET"}, {6, "WAIT_FLIP_DONE"}, {9, "DISPATCH_RESET"}, {11, "PUSH_MARKER"}, {12, "POP_MARKER"}, {20, "ACQUIRE_MEM_CUSTOM"}, {21, "WRITE_DATA_CUSTOM"}, {23, "FLIP"}, {24, "RELEASE_MEM_CUSTOM"}, {25, "DMA_DATA_CUSTOM"}, {26, "CONTEXT_STATE"} }}; - for (const auto& [id, name] : custom) check(AgcDriver::Pm4::Name(makePacket(0x10, {0}, id << 2)[0]) == name, "custom opcode name mismatch"); + for (const auto& [id, name] : custom) { + const auto header = makePacket(0x10, {0}, id << 2)[0]; + check(AgcDriver::Pm4::Name(header) == name && AgcDriver::Pm4::KnownName(header) == name, "custom opcode name mismatch"); + } } void testNopPad() { @@ -1237,6 +1255,370 @@ void testAsyncMemoryFailure() { expectFailure([] { LibcRunShutdown_nid_postfix(); }, "guest"); } +class CapturedStderr { +public: + CapturedStderr() { +#ifdef _WIN32 + path = std::filesystem::temp_directory_path() / ("aps5-pm4-ring-" + std::to_string(GetCurrentProcessId()) + ".txt"); + file = CreateFileW(path.c_str(), GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + check(file != INVALID_HANDLE_VALUE, "cannot create the stderr capture file"); + previous = GetStdHandle(STD_ERROR_HANDLE); + check(SetStdHandle(STD_ERROR_HANDLE, file) != 0, "cannot redirect stderr"); +#else + path = std::filesystem::temp_directory_path() / ("aps5-pm4-ring-" + std::to_string(::getpid()) + ".txt"); + file = ::open(path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0600); + check(file >= 0, "cannot create the stderr capture file"); + std::fflush(stderr); + previous = ::dup(2); + check(previous >= 0 && ::dup2(file, 2) >= 0, "cannot redirect stderr"); +#endif + } + + ~CapturedStderr() { +#ifdef _WIN32 + SetStdHandle(STD_ERROR_HANDLE, previous); + CloseHandle(file); +#else + std::fflush(stderr); + ::dup2(previous, 2); + ::close(previous); + ::close(file); +#endif + std::error_code ignored; + std::filesystem::remove(path, ignored); + } + + CapturedStderr(const CapturedStderr&) = delete; + CapturedStderr& operator=(const CapturedStderr&) = delete; + + std::vector RingLines() const { + std::ifstream input(path, std::ios::binary); + std::vector lines; + for (std::string line; std::getline(input, line);) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.starts_with("[pm4 ring]")) lines.push_back(line); + } + return lines; + } + +private: + std::filesystem::path path; +#ifdef _WIN32 + HANDLE file = INVALID_HANDLE_VALUE; + HANDLE previous = nullptr; +#else + int file = -1; + int previous = -1; +#endif +}; + +std::vector recordedWrites(const std::vector& lines) { + std::vector writes; + std::string entry; + for (const auto& line : lines) { + if (line.starts_with("[pm4 ring] queue ") && line.find(" #") != std::string::npos) { + entry = line.substr(17, line.find(" serial") - 17); + } else if (line.find("WRITE_DATA") != std::string::npos) { + check(!entry.empty() && line.size() >= 8, "a WRITE_DATA line is not under a submission"); + writes.push_back(entry + " " + line.substr(line.size() - 8)); + entry.clear(); + } + } + return writes; +} + +void submitWrites(std::uint32_t queue, std::uint32_t first, std::uint32_t count) { + alignas(8) static std::uint32_t target = 0; + for (std::uint32_t value = first; value < first + count; ++value) { + auto words = writeWord(target, value); + submitWords(words, queue); + } + AgcDriverWaitIdle_nid_postfix(); +} + +void testSubmissionRing() { + submitWrites(0, 0x5a000001u, 6); + submitWrites(0x20, 0x5b000001u, 2); + std::vector lines; + { + CapturedStderr capture; + AgcDriver::DriverDetail::DumpSubmissionRings("ring test"); + lines = capture.RingLines(); + } + check(!lines.empty() && lines.front() == "[pm4 ring] ring test: 2 queue(s), up to 4 submissions each, oldest first", "the ring dump header is wrong"); + check(std::find(lines.begin(), lines.end(), "[pm4 ring] queue 0x0: 6 submissions recorded, last 4 kept") != lines.end(), "the ring dump does not count every submission of queue 0"); + check(std::find(lines.begin(), lines.end(), "[pm4 ring] queue 0x20: 2 submissions recorded, last 2 kept") != lines.end(), "the ring dump does not count every submission of queue 0x20"); + const std::vector expected{"0x0 #3 5a000003", "0x0 #4 5a000004", "0x0 #5 5a000005", "0x0 #6 5a000006", "0x20 #1 5b000001", "0x20 #2 5b000002"}; + check(recordedWrites(lines) == expected, "the ring dump does not hold exactly the last submissions of each queue in order"); +} + +void testEmptySubmissionRing() { + CapturedStderr capture; + AgcDriver::DriverDetail::DumpSubmissionRings("ring test"); + check(capture.RingLines() == std::vector{"[pm4 ring] ring test: no submissions recorded"}, "the dump of empty rings is not one line"); +} + +void testDisabledSubmissionRing() { + submitWrites(0, 0x5a000001u, 2); + CapturedStderr capture; + AgcDriver::DriverDetail::DumpSubmissionRings("ring test"); + check(capture.RingLines() == std::vector{"[pm4 ring] ring test: disabled (APS5_PM4_RING=0)"}, "the dump of disabled rings is not one line"); +} + +void expectSettingRejected(const char* text) { + alignas(8) static std::uint32_t target = 0; + auto words = writeWord(target, 1); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + expectFailure([&] { sceAgcDriverSubmitDcb(&packet); }, text); + AgcDriverWaitIdle_nid_postfix(); + check(target == 0, "a submission rejected for its ring or watchdog setting was executed"); +} + +void testInvalidSubmissionRing() { + expectSettingRejected("APS5_PM4_RING must be a number"); +} + +void testInvalidStallWatchdog() { + expectSettingRejected("APS5_STALL_WATCHDOG_SECS must be a number"); +} + +void testSubmissionRingFailureReport() { + submitWrites(0, 0x5a000001u, 3); + std::vector lines; + { + CapturedStderr capture; + LibcRunFailureReports_nid_no_patch(); + lines = capture.RingLines(); + } + check(!lines.empty() && lines.front() == "[pm4 ring] failure report: 1 queue(s), up to 2 submissions each, oldest first", "the failure report does not dump the rings"); + check(recordedWrites(lines) == std::vector{"0x0 #2 5a000002", "0x0 #3 5a000003"}, "the failure report does not hold the last submissions in order"); +} + +#ifdef _WIN32 +std::atomic heapTrapThread{0}; +std::atomic heapTrapAllocations{0}; +decltype(&HeapAlloc) heapAllocOriginal = nullptr; +decltype(&HeapReAlloc) heapReAllocOriginal = nullptr; + +LPVOID WINAPI trappedHeapAlloc(HANDLE heap, DWORD flags, SIZE_T bytes) { + if (GetCurrentThreadId() == heapTrapThread.load()) heapTrapAllocations.fetch_add(1); + return heapAllocOriginal(heap, flags, bytes); +} + +LPVOID WINAPI trappedHeapReAlloc(HANDLE heap, DWORD flags, LPVOID block, SIZE_T bytes) { + if (GetCurrentThreadId() == heapTrapThread.load()) heapTrapAllocations.fetch_add(1); + return heapReAllocOriginal(heap, flags, block, bytes); +} + +void** importSlot(HMODULE module, const char* function) { + auto* base = reinterpret_cast(module); + const auto* headers = reinterpret_cast(base + reinterpret_cast(base)->e_lfanew); + const auto& directory = headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT]; + for (auto* import = reinterpret_cast(base + directory.VirtualAddress); import->Name != 0; ++import) { + const auto* names = reinterpret_cast(base + import->OriginalFirstThunk); + auto* slots = reinterpret_cast(base + import->FirstThunk); + for (; names->u1.AddressOfData != 0; ++names, ++slots) { + if (IMAGE_SNAP_BY_ORDINAL(names->u1.Ordinal)) continue; + const auto* byName = reinterpret_cast(base + names->u1.AddressOfData); + if (std::strcmp(reinterpret_cast(byName->Name), function) == 0) return reinterpret_cast(&slots->u1.Function); + } + } + return nullptr; +} + +void replaceSlot(void** slot, void* value) { + DWORD protection = 0; + check(VirtualProtect(slot, sizeof(void*), PAGE_READWRITE, &protection) != 0, "cannot unprotect a ucrtbase.dll import"); + *slot = value; + DWORD ignored = 0; + check(VirtualProtect(slot, sizeof(void*), protection, &ignored) != 0, "cannot protect a ucrtbase.dll import"); +} + +class CrtHeapTrap { +public: + CrtHeapTrap() { + const HMODULE runtime = GetModuleHandleW(L"ucrtbase.dll"); + check(runtime != nullptr, "ucrtbase.dll is not loaded"); + allocSlot = importSlot(runtime, "HeapAlloc"); + reAllocSlot = importSlot(runtime, "HeapReAlloc"); + check(allocSlot != nullptr && reAllocSlot != nullptr, "ucrtbase.dll does not import HeapAlloc and HeapReAlloc"); + heapAllocOriginal = reinterpret_cast(*allocSlot); + heapReAllocOriginal = reinterpret_cast(*reAllocSlot); + replaceSlot(allocSlot, reinterpret_cast(trappedHeapAlloc)); + replaceSlot(reAllocSlot, reinterpret_cast(trappedHeapReAlloc)); + } + + ~CrtHeapTrap() { + replaceSlot(allocSlot, reinterpret_cast(heapAllocOriginal)); + replaceSlot(reAllocSlot, reinterpret_cast(heapReAllocOriginal)); + } + + CrtHeapTrap(const CrtHeapTrap&) = delete; + CrtHeapTrap& operator=(const CrtHeapTrap&) = delete; + + template + std::uint32_t Allocations(TAction action) { + heapTrapAllocations.store(0); + heapTrapThread.store(GetCurrentThreadId()); + action(); + heapTrapThread.store(0); + return heapTrapAllocations.load(); + } + +private: + void** allocSlot = nullptr; + void** reAllocSlot = nullptr; +}; + +void testSubmissionRingFailureReportHeap() { + alignas(8) static std::uint32_t target = 0; + auto words = joinPackets({makePacket(0x13, {0x40}), writeWord(target, 0x5c000001u)}); + submitWords(words); + AgcDriverWaitIdle_nid_postfix(); + std::vector lines; + { + CrtHeapTrap trap; + check(trap.Allocations([&] { check(AgcDriver::Pm4::Name(words[0]) == "INDEX_BUFFER_SIZE", "INDEX_BUFFER_SIZE is misnamed"); }) != 0, "the heap trap does not see the allocation of a packet name"); + CapturedStderr capture; + const auto allocations = trap.Allocations([] { LibcRunFailureReports_nid_no_patch(); }); + lines = capture.RingLines(); + check(allocations == 0, "the failure report allocated from the C runtime heap"); + } + check(std::any_of(lines.begin(), lines.end(), [](const std::string& line) { return line.find("+0x0000 INDEX_BUFFER_SIZE 00000040") != std::string::npos; }), "the failure report does not name INDEX_BUFFER_SIZE"); + check(recordedWrites(lines) == std::vector{"0x0 #1 5c000001"}, "the failure report does not hold the submission"); +} +#endif + +bool hasRingLine(const std::vector& lines, std::string_view prefix, std::string_view suffix) { + return std::any_of(lines.begin(), lines.end(), [&](const std::string& line) { return line.size() >= prefix.size() + suffix.size() && line.starts_with(prefix) && line.ends_with(suffix); }); +} + +void awaitRingLine(const CapturedStderr& capture, std::string_view prefix, std::string_view suffix, const char* reason) { + for (int waited = 0; waited < 1000 && !hasRingLine(capture.RingLines(), prefix, suffix); ++waited) std::this_thread::sleep_for(std::chrono::milliseconds(10)); + check(hasRingLine(capture.RingLines(), prefix, suffix), reason); +} + +void checkOneStallDump(const std::vector& lines, std::uint32_t& done) { + check(std::atomic_ref(done).load() == 1, "the stalled submission did not finish after it was released"); + check(std::count_if(lines.begin(), lines.end(), [](const std::string& line) { return line.find("has not progressed") != std::string::npos; }) == 1, "a stalled waiter dumped the rings more than once"); +} + +std::string stallPrefix(const char* waiter, const void* address) { + char text[160]; + std::snprintf(text, sizeof(text), "[pm4 ring] queue 0x0 %s 0x%llx has not progressed", waiter, static_cast(reinterpret_cast(address))); + return text; +} + +void testStallWatchdog() { + alignas(8) static std::uint32_t gate = 0; + alignas(8) static std::uint32_t done = 0; + std::vector lines; + { + CapturedStderr capture; + auto words = joinPackets({makePacket(0x3c, {0x13, low(&gate), high(&gate), 1, 0xffffffffu, 0x19}), writeWord(done, 1)}); + submitWords(words); + awaitRingLine(capture, stallPrefix("WAIT_REG_MEM at", &gate), " for 1 s (APS5_STALL_WATCHDOG_SECS): 1 queue(s), up to 16 submissions each, oldest first", + "a WAIT_REG_MEM stalled past APS5_STALL_WATCHDOG_SECS did not dump the rings"); + std::this_thread::sleep_for(std::chrono::milliseconds(1500)); + check(std::atomic_ref(done).load() == 0, "the stalled submission finished before its gate was opened"); + std::atomic_ref(gate).store(1); + AgcDriverWaitIdle_nid_postfix(); + lines = capture.RingLines(); + } + checkOneStallDump(lines, done); + check(std::any_of(lines.begin(), lines.end(), [](const std::string& line) { return line.find("+0x0000 WAIT_REG_MEM 00000013") != std::string::npos; }), "the stall dump does not decode the stalled submission"); +} + +void testStallWatchdogRewind() { + alignas(8) static std::uint32_t done = 0; + static std::vector words; + words = joinPackets({makePacket(0x59, {0}), writeWord(done, 1)}); + std::vector lines; + { + CapturedStderr capture; + submitWords(words); + awaitRingLine(capture, stallPrefix("REWIND at", &words[1]), " for 1 s (APS5_STALL_WATCHDOG_SECS): 1 queue(s), up to 16 submissions each, oldest first", + "a REWIND stalled past APS5_STALL_WATCHDOG_SECS did not dump the rings"); + std::this_thread::sleep_for(std::chrono::milliseconds(1500)); + check(std::atomic_ref(done).load() == 0, "the tail of a REWIND ran before the title released it"); + std::atomic_ref(words[1]).store(0x80000000u); + AgcDriverWaitIdle_nid_postfix(); + lines = capture.RingLines(); + } + checkOneStallDump(lines, done); + check(std::any_of(lines.begin(), lines.end(), [](const std::string& line) { return line.find("+0x0000 REWIND 00000000") != std::string::npos; }), "the stall dump does not decode the stalled REWIND"); +} + +struct RenderingGateState { + std::mutex mutex; + std::condition_variable changed; + bool closed = true; + bool waiting = false; + + void Open() { + { + std::lock_guard lock(mutex); + closed = false; + } + changed.notify_all(); + } +}; + +class RenderingGate final : public AgcDriver::IRenderingWait { +public: + explicit RenderingGate(std::shared_ptr state) : state(std::move(state)) {} + + void Wait() override { + std::unique_lock lock(state->mutex); + state->waiting = true; + state->changed.notify_all(); + state->changed.wait(lock, [&] { return !state->closed; }); + } + +private: + std::shared_ptr state; +}; + +class GatedOutput final : public AgcDriver::IVideoOutput { +public: + std::shared_ptr Reserve(const AgcDriver::FlipInfo&) override { throw std::runtime_error("the gated test output takes no flips"); } + std::shared_ptr CaptureRenderingWait(std::uint32_t) override { return std::make_shared(state); } + void Fail(std::exception_ptr) noexcept override {} + + std::shared_ptr state = std::make_shared(); +}; + +void testStallWatchdogOrder() { + alignas(8) static std::uint32_t done = 0; + auto output = std::make_shared(); + AgcDriverRegisterVideoOutput_nid_postfix(7, output); + struct GateRelease { + std::shared_ptr state; + ~GateRelease() { state->Open(); } + } release{output->state}; + std::vector lines; + { + CapturedStderr capture; + std::vector wait{AgcDriver::RenderingWaitPacketHeader, 7, 0, 0}; + submitWords(wait); + { + std::unique_lock lock(output->state->mutex); + check(output->state->changed.wait_for(lock, std::chrono::seconds(5), [&] { return output->state->waiting; }), "queue 0 did not reach the rendering wait"); + } + auto words = writeWord(done, 1); + submitWords(words, 0x20); + awaitRingLine(capture, "[pm4 ring] queue 0x20 order behind queue 0 for event 0x", " has not progressed for 1 s (APS5_STALL_WATCHDOG_SECS): 2 queue(s), up to 16 submissions each, oldest first", + "a compute submission held behind queue 0 past APS5_STALL_WATCHDOG_SECS did not dump the rings"); + std::this_thread::sleep_for(std::chrono::milliseconds(1500)); + check(std::atomic_ref(done).load() == 0, "a compute submission passed queue 0 while it was held"); + output->state->Open(); + AgcDriverWaitIdle_nid_postfix(); + lines = capture.RingLines(); + } + AgcDriverUnregisterVideoOutput_nid_postfix(7, output); + checkOneStallDump(lines, done); +} + void testRefusedWorkerPriority() { alignas(8) static std::uint32_t result = 0; auto words = writeWord(result, 91); @@ -1268,6 +1650,28 @@ int main(int argc, char** argv) { std::puts("PM4 refused queue worker priority propagated to idle and shutdown"); return 0; } + const std::pair ringModes[]{ + {"ring", testSubmissionRing}, {"ring-empty", testEmptySubmissionRing}, {"ring-disabled", testDisabledSubmissionRing}, + {"ring-invalid", testInvalidSubmissionRing}, {"ring-report", testSubmissionRingFailureReport}, {"stall-watchdog", testStallWatchdog}, + {"stall-watchdog-invalid", testInvalidStallWatchdog}, {"stall-watchdog-rewind", testStallWatchdogRewind}, {"stall-watchdog-order", testStallWatchdogOrder}}; + if (argc == 2 && std::string(argv[1]) == "ring-report-heap") { +#ifdef _WIN32 + testSubmissionRingFailureReportHeap(); + LibcRunShutdown_nid_postfix(); + std::puts("PM4 ring-report-heap test passed"); + return 0; +#else + std::puts("the C runtime heap trap needs ucrtbase.dll"); + return 77; +#endif + } + for (const auto& [mode, test] : ringModes) { + if (argc != 2 || std::string(argv[1]) != mode) continue; + test(); + LibcRunShutdown_nid_postfix(); + std::printf("PM4 %s test passed\n", mode); + return 0; + } testCatalog(); testNopPad(); testWriteChangedKeepsUntouchedBytes(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 4c16ba2c3f..46aa951730 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -210,6 +210,8 @@ public: context.occlusionQueryPrecise = supported.occlusionQueryPrecise == VK_TRUE; enabled.shaderStorageImageReadWithoutFormat = supported.shaderStorageImageReadWithoutFormat; enabled.shaderStorageImageWriteWithoutFormat = supported.shaderStorageImageWriteWithoutFormat; + enabled.shaderStorageImageMultisample = supported.shaderStorageImageMultisample; + context.storageImageMultisample = supported.shaderStorageImageMultisample == VK_TRUE; context.singlePassStorage = supported.shaderStorageImageReadWithoutFormat == VK_TRUE && supported.shaderStorageImageWriteWithoutFormat == VK_TRUE; address.pNext = &bytes; std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; @@ -3499,10 +3501,16 @@ void drawInputInPlaceTests(const Device& device, Recorder& recorder) { Require(InPlaceDrawInput(context, vertices + 2, 256, 2) != nullptr, "an aligned 16-bit draw input was not bound in place"); recorder.NotePendingWrite(address + 8192, 16); Require(InPlaceDrawInput(context, address + 8192, 256, 4) != nullptr && !recorder.PendingWriteOverlaps(address + 8192, 256), "a draw input under a pending GPU write was bound before the write completed"); + bool behind = false; + Require(InPlaceDrawInput(context, address + 20480, 256, 4, &behind) != nullptr && !behind, "a vertex input free of pending GPU writes was marked as ordered behind one"); + recorder.NotePendingWrite(address + 20480, 16); + Require(InPlaceDrawInput(context, address + 20480, 256, 4, &behind) != nullptr && behind && recorder.PendingWriteOverlaps(address + 20480, 256), "a vertex input under a pending GPU write was not bound in place behind it"); const std::array label{std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}}; recorder.RecordStore(import->buffer, address + 12288 - import->base, label, address + 12288); Require(recorder.QueuedStoreOverlaps(address + 12288, 4), "the label store was not queued"); Require(InPlaceDrawInput(context, address + 12288, 256, 4) == nullptr, "a draw input under a queued label store was bound in place"); + behind = false; + Require(InPlaceDrawInput(context, address + 12288, 256, 4, &behind) == nullptr, "a vertex input under a queued label store was bound in place"); Require(InPlaceDrawInput(context, address + 16384, 256, 4) != nullptr, "a queued label store refused a disjoint draw input"); alignas(64) static std::array unregistered{}; Require(InPlaceDrawInput(context, reinterpret_cast(unregistered.data()), unregistered.size(), 4) == nullptr, "memory without a host import was bound in place"); @@ -6596,6 +6604,7 @@ void multisampleGuestAccessTests(const Device& device, Recorder& recorder) { { std::lock_guard gpu(GpuMutex()); Require(MultisampleTargetView(context, target) != VK_NULL_HANDLE, "the multisampled target has no view"); + Require(!MultisampleTargetStorable(context, target), "a multisampled target nothing stored to was created with storage usage"); recorder.Submit(); device.WaitQueue(); recorder.Sync(); @@ -6681,7 +6690,20 @@ void multisampleGuestAccessTests(const Device& device, Recorder& recorder) { refusedSample(ShaderRecompiler::DescriptorKind::SampledImage, multisampledWords(side, 3u, address), "as a 64x64 texture of 8 samples", "sampling a 4x target as an 8x texture"); refusedSample(ShaderRecompiler::DescriptorKind::SampledImage, multisampledWords(side / 2u, 2u, address), "as a 32x64 texture of 4 samples", "sampling a 4x target as a narrower texture"); refusedSample(ShaderRecompiler::DescriptorKind::SampledImage, multisampledWords(side, 2u, address + 4u * 65536u), "which no draw rendered to and whose tile mode 0 has no modeled multisampled color footprint", "sampling a linear multisampled texture no draw rendered to"); - refusedSample(ShaderRecompiler::DescriptorKind::StorageImage, multisampledWords(side, 2u, address), "storage image access to a multisampled texture", "storage access to a multisampled texture"); + if (context.storageImageMultisample) { + const auto stored = sampledProgram(ShaderRecompiler::DescriptorKind::StorageImage, multisampledWords(side, 2u, address)); + const CompiledShader storedCompute{ShaderRecompiler::ShaderStage::Compute, &stored, 0}; + std::lock_guard gpu(GpuMutex()); + const auto rendered = MultisampleTargetView(context, target); + Require(!MultisampleTargetStorable(context, target), "sampling the multisampled target gave it storage usage"); + ShaderResources resources(context, storedCompute); + Require(MultisampleTargetStorable(context, target) && MultisampleTargetView(context, target) != rendered, "a storage binding did not move the multisampled target to an image with storage usage"); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + } else { + refusedSample(ShaderRecompiler::DescriptorKind::StorageImage, multisampledWords(side, 2u, address), "as a storage image at its sample count", "storage access to a multisampled target without shaderStorageImageMultisample"); + } refused([&] { AgcDriver::GuestMemory::Read(address + 3u * 65536u + 256u, texels); }, "a CPU read", "a CPU read in the padding of a later sample plane"); AgcDriver::GuestMemory::Read(address + 4u * 65536u, texels); clearTargets(); @@ -7573,7 +7595,7 @@ bool vertexTargetRangeTests(const Device& device, Recorder& recorder, bool impor Draw(context, consumer, draw, shaders, {}, &recipe); WaitForOptimizedPipelines(context.device); const auto read = recorder.DescribePendingRead(address + unit, 16); - Require(read.has_value() == imported && (!read || read->kind == Recorder::ReadKind::DrawInput), "the vertex regression used the wrong copied/imported path"); + Require(read.has_value() == imported && (!read || read->kind == Recorder::ReadKind::DrawInput || read->kind == Recorder::ReadKind::StorageUpload), "the vertex regression used the wrong copied/imported path"); check(16); Require(recipe != nullptr, "the range regression did not create a draw recipe"); std::vector cleared(unit); @@ -7676,10 +7698,13 @@ void pipelineCacheTests(const Device& device, bool benchmark) { input = {}; Require(lookup(input) == first, "pipeline cache retained bytes from a larger key"); vertex.variantId = 0; - const auto unidentified = lookup(input); - Require(lookup(input) != unidentified, "pipeline cache retained an unidentified shader"); + Require(lookup(input) == first, "pipeline cache keyed on the variant id instead of the module"); + vertex.spirv[2] ^= 1u; + const auto other = lookup(input); + Require(other != first && lookup(input) == other, "pipeline cache ignored another module"); + vertex.spirv[2] ^= 1u; vertex.variantId = 1; - Require(lookup(input) == first, "an unidentified shader disturbed the retained pipeline"); + Require(lookup(input) == first, "another module disturbed the retained pipeline"); ClearCachedPipelines(context.device); Require(lookup(input) != first, "clearing the pipeline cache retained its old entry"); std::cout << "Pipeline cache key and retained lifetime tests passed\n"; @@ -8151,13 +8176,13 @@ void pipelineLibraryTests(const Device& device) { Require(StagePushOffset(88, ShaderRecompiler::ShaderStage::Fragment, true) == PipelinePushSlotBytes && StagePushOffset(88, ShaderRecompiler::ShaderStage::Fragment, false) == 88 && StagePushOffset(40, ShaderRecompiler::ShaderStage::TessellationEvaluation, true) == 40, "a stage got the wrong push offset"); const auto beforeSlots = counters().built; ShaderRecompiler::RecompileResult otherVertex = vertex; - otherVertex.variantId = 14; + otherVertex.spirv[2] ^= 1u; std::array slotted{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}; static_cast(CachedPipeline(context, state, input, resources, slotted, VK_IMAGE_LAYOUT_GENERAL)); slotted.front().program = &otherVertex; static_cast(CachedPipeline(context, state, input, resources, slotted, VK_IMAGE_LAYOUT_GENERAL)); Require(counters().built[2] == beforeSlots[2] + 1u && counters().built[1] == beforeSlots[1] + 1u, "a pixel shader in its fixed slot built its library again for another vertex shader"); - fragment.variantId = 13; + fragment.spirv[2] ^= 1u; static_cast(lookup()); Require(counters().built == std::array{2, 2, 3, 2}, "a new pixel shader rebuilt more than the fragment shader library"); state.colors.front().format = VK_FORMAT_R16G16B16A16_SFLOAT; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp index 1143e50b0d..9b59ac9b2e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace { @@ -16,8 +17,8 @@ using AgcDriver::Graphics::Require; struct ProfileInput { std::vector capabilities{spv::CapabilityShader, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; - std::array extensionStrings{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; - std::array extensions{extensionStrings[0], extensionStrings[1]}; + std::vector extensionStrings{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + std::vector extensions{extensionStrings[0], extensionStrings[1]}; VkPhysicalDeviceRobustness2FeaturesEXT robustness{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT, nullptr, VK_FALSE, VK_FALSE, VK_TRUE}; VkPhysicalDeviceDescriptorIndexingFeatures indexing{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES, &robustness}; VkPhysicalDevice8BitStorageFeatures bytes{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_8BIT_STORAGE_FEATURES, &indexing, VK_TRUE}; @@ -54,7 +55,7 @@ void Reject(TAction action, const char* expected) { Require(std::string(error.what()).find(expected) != std::string::npos, "unexpected profile validation error"); return; } - throw std::runtime_error("invalid shader device profile was accepted"); + throw std::runtime_error(std::string("invalid shader device profile was accepted: ") + expected); } void CheckProfile() { @@ -80,6 +81,7 @@ void CheckProfile() { invalid([](auto& value) { value.bda.bufferDeviceAddress = VK_FALSE; }, "bufferDeviceAddress"); invalid([](auto& value) { value.device.pNext = nullptr; }, "bufferDeviceAddress"); invalid([](auto& value) { value.core.shaderInt64 = VK_FALSE; }, "shaderInt64"); + invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityGeometry); }, "geometryShader"); invalid([](auto& value) { value.bytes.storageBuffer8BitAccess = VK_FALSE; }, "storageBuffer8BitAccess"); ProfileInput withoutNull; withoutNull.robustness.nullDescriptor = VK_FALSE; @@ -99,6 +101,8 @@ void CheckProfile() { invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityStorageImageMultisample); }, "shaderStorageImageMultisample"); invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityFragmentBarycentricKHR); }, "fragmentShaderBarycentric"); invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityMeshShadingEXT); }, "meshShader"); + invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityDrawParameters); }, "shaderDrawParameters"); + invalid([](auto& value) { value.extensions.push_back("SPV_KHR_terminate_invocation"); }, "shaderTerminateInvocation"); ProfileInput enabled; enabled.capabilities.push_back(spv::CapabilitySampledImageArrayDynamicIndexing); enabled.capabilities.push_back(spv::CapabilityStorageImageArrayNonUniformIndexing); @@ -106,6 +110,28 @@ void CheckProfile() { enabled.indexing.shaderStorageImageArrayNonUniformIndexing = VK_TRUE; const AgcDriver::ShaderDeviceProfile accepted(enabled.Target(), enabled.device, enabled.limits); Require(accepted.Target().supportedCapabilities.size() == 6u, "enabled descriptor indexing was rejected"); + ProfileInput drawParameters; + drawParameters.capabilities.push_back(spv::CapabilityDrawParameters); + VkPhysicalDeviceShaderDrawParametersFeatures drawFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DRAW_PARAMETERS_FEATURES, &drawParameters.bda, VK_TRUE}; + drawParameters.device.pNext = &drawFeatures; + const AgcDriver::ShaderDeviceProfile drawParametersAccepted(drawParameters.Target(), drawParameters.device, drawParameters.limits); + Require(std::ranges::find(drawParametersAccepted.Target().supportedCapabilities, spv::CapabilityDrawParameters) != drawParametersAccepted.Target().supportedCapabilities.end(), "enabled shaderDrawParameters was rejected"); + ProfileInput terminating; + VkPhysicalDeviceShaderTerminateInvocationFeaturesKHR terminate{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_TERMINATE_INVOCATION_FEATURES_KHR, &terminating.bda, VK_TRUE}; + terminating.device.pNext = &terminate; + terminating.extensions.push_back("SPV_KHR_terminate_invocation"); + const AgcDriver::ShaderDeviceProfile terminated(terminating.Target(), terminating.device, terminating.limits); + Require(std::ranges::find(terminated.Target().supportedExtensions, "SPV_KHR_terminate_invocation") != terminated.Target().supportedExtensions.end(), "enabled shaderTerminateInvocation was rejected"); + ProfileInput disabled; + VkPhysicalDeviceShaderTerminateInvocationFeaturesKHR disabledTerminate{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_TERMINATE_INVOCATION_FEATURES_KHR, &disabled.bda, VK_FALSE}; + disabled.device.pNext = &disabledTerminate; + disabled.extensions.push_back("SPV_KHR_terminate_invocation"); + Reject([&] { AgcDriver::ShaderDeviceProfile rejected(disabled.Target(), disabled.device, disabled.limits); }, "shaderTerminateInvocation"); + ProfileInput geometryEnabled; + geometryEnabled.capabilities.push_back(spv::CapabilityGeometry); + geometryEnabled.core.geometryShader = VK_TRUE; + const AgcDriver::ShaderDeviceProfile geometryAccepted(geometryEnabled.Target(), geometryEnabled.device, geometryEnabled.limits); + Require(std::ranges::find(geometryAccepted.Target().supportedCapabilities, spv::CapabilityGeometry) != geometryAccepted.Target().supportedCapabilities.end(), "enabled geometry shaders were rejected"); } void CheckAbi() { diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 3c7743b54e..2ac7728dfd 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -75,7 +75,7 @@ void requireSameArtifact(const CompiledShaderArtifact& left, const CompiledShade require(left.runtimeAbiVersion == right.runtimeAbiVersion, prefix + "runtime ABI version differs"); require(left.memoryOffsetDword == right.memoryOffsetDword, prefix + "memory offset differs"); require(left.shaderDataDwords == right.shaderDataDwords && left.imageMetadataDword == right.imageMetadataDword && left.runtimeImageCount == right.runtimeImageCount && left.runtimeImageResources == right.runtimeImageResources, prefix + "compact data layout differs"); - require(left.hostSubgroupSize == right.hostSubgroupSize, prefix + "host subgroup size differs"); + require(left.hostSubgroupSize == right.hostSubgroupSize && left.fullSubgroups == right.fullSubgroups, prefix + "host subgroup size differs"); require(left.vertexInputs == right.vertexInputs, prefix + "vertex inputs differ"); require(left.vertexInputPatches == right.vertexInputPatches, prefix + "vertex type patches differ"); require(left.registrationModule == right.registrationModule, prefix + "registration module flags differ"); @@ -101,6 +101,7 @@ void requireSameResult(const RecompileResult& left, const RecompileResult& right require(left.specialization == right.specialization, prefix + "specialization constants differ"); require(left.workgroupMemoryDwords == right.workgroupMemoryDwords, prefix + "workgroup memory stride differs"); require(left.poisonedSrtReads == right.poisonedSrtReads, prefix + "poisoned SRT read counts differ"); + require(left.shaderDataUserWords == right.shaderDataUserWords, prefix + "shader-data user words differ"); require(left.imageTableRanges == right.imageTableRanges && left.imageTableShader == right.imageTableShader && left.imageTableFaults == right.imageTableFaults && left.imageTablePoison.size() == right.imageTablePoison.size(), prefix + "image table invocation data differs"); require(left.imageTableReadRanges == right.imageTableReadRanges, prefix + "image table read ranges differ"); for (std::size_t i = 0; i < left.imageTablePoison.size(); ++i) { @@ -169,6 +170,7 @@ RecompileResult sampleResult() { result.bdaAbiVersion = 3; result.memoryOffsetDword = 7; result.hostSubgroupSize = 32; + result.fullSubgroups = true; result.workgroupMemoryDwords = 32769u; result.vertexInputs = {{1, 4, 2}, {5, 2, 0}}; result.vertexInputPatches = {{1, 20, {7, 8, 9}}, {5, 40, {10, 11, 12}}}; @@ -189,6 +191,7 @@ RecompileResult sampleResult() { result.stencilReplaySafe = true; result.stencilReferenceDword = 23u; result.poisonedSrtReads = 3; + result.shaderDataUserWords = {2u, 0xffffffffu, 9u}; result.barycentricEmulation = {true, false, true}; result.variantId = 99; result.imageTablePoison = {{{1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u}, 8u, 2u, 6u}, {{9u, 0u, 0u, 0u, 0u, 0u, 0u, 0u}, 4u, 0xffffffffu, 19u}}; @@ -618,6 +621,7 @@ void verifyKeySensitivity() { changes("the narrow subgroup clock", [](SampleRequest& sample) { sample.request.target.narrowSubgroupClock = true; }); changes("f64 denormal preservation", [](SampleRequest& sample) { sample.request.target.float64DenormPreserve = true; }); changes("f16 denormal preservation", [](SampleRequest& sample) { sample.request.target.float16DenormPreserve = true; }); + changes("spanned waves", [](SampleRequest& sample) { sample.request.target.spannedWaves = true; }); changes("the workgroup size limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSize[2] = 128; }); changes("the invocation limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupInvocations = 512; }); changes("the shared memory limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSharedMemoryBytes = 32768; }); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index a3294a30fe..6ad15f3495 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -678,46 +678,48 @@ void verifySignedSrtComparison() { void verifyTypeZeroImageMaterialization() { using namespace ShaderRecompiler; - IrResourcePlan plan; - plan.srtPlanComplete = true; - plan.resourceTrackingComplete = true; - plan.userDataCount = 0u; - plan.materializationSources = {0u}; + for (const std::uint32_t type : {0u, 1u}) { + IrResourcePlan plan; + plan.srtPlanComplete = true; + plan.resourceTrackingComplete = true; + plan.userDataCount = 0u; + plan.materializationSources = {0u}; - const std::array words{0x12345678u, (56u << 20u) | 0x100u, 0x0000c000u, 0x00000facu, 1u, 2u, 3u, 4u}; - std::vector> values; - DescriptorSource source; - source.dwordCount = static_cast(words.size()); - for (std::uint32_t i = 0u; i < words.size(); ++i) { - auto value = std::make_unique(IrOpcode::Void, IrType::U32, i); - value->SetImmediateU32(words[i]); - source.dwords[i] = value.get(); - values.push_back(std::move(value)); - } - plan.descriptorSources.push_back(source); + const std::array words{0x12345678u, (56u << 20u) | 0x100u, 0x0000c000u, 0x00000facu | (type << 28u), 1u, 2u, 3u, 4u}; + std::vector> values; + DescriptorSource source; + source.dwordCount = static_cast(words.size()); + for (std::uint32_t i = 0u; i < words.size(); ++i) { + auto value = std::make_unique(IrOpcode::Void, IrType::U32, i); + value->SetImmediateU32(words[i]); + source.dwords[i] = value.get(); + values.push_back(std::move(value)); + } + plan.descriptorSources.push_back(source); - ImageResource image; - image.source = 0u; - image.resourceClass = ImageResourceClass::Sampled; - image.dimension = RdnaImageDimension::Dim2D; - image.read = true; - plan.info.images.push_back(image); - plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); + ImageResource image; + image.source = 0u; + image.resourceClass = ImageResourceClass::Sampled; + image.dimension = RdnaImageDimension::Dim2D; + image.read = true; + plan.info.images.push_back(image); + plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); - ResourceSnapshot snapshot; - ResourceMaterializer{}.Materialize(plan, SrtRuntime{}, snapshot); - require(snapshot.images.size() == 1u && snapshot.images[0].dwordCount == 8u && snapshot.images[0].dwords == words, - "TYPE=0 image descriptor with a nonzero base was rejected or altered during materialization"); + ResourceSnapshot snapshot; + ResourceMaterializer{}.Materialize(plan, SrtRuntime{}, snapshot); + require(snapshot.images.size() == 1u && snapshot.images[0].dwordCount == 8u && snapshot.images[0].dwords == words, + type == 0u ? "TYPE=0 image descriptor with a nonzero base was rejected or altered during materialization" : "TYPE=1 image descriptor with a nonzero base was rejected or altered during materialization"); - for (const bool atomic : {false, true}) { - image.resourceClass = ImageResourceClass::Storage; - image.read = false; - image.written = !atomic; - image.atomic = atomic; - plan.info.images[0] = image; - plan.info.runtimeImageModes[0] = ResourceMaterializer::RuntimeImageModes(image); - expectFailure([&] { ResourceMaterializer{}.Materialize(plan, SrtRuntime{}, snapshot); }, "runtime image descriptor is invalid", - atomic ? "TYPE=0 atomic image descriptor was accepted without hardware evidence" : "TYPE=0 writable image descriptor was accepted without hardware evidence"); + for (const bool atomic : {false, true}) { + image.resourceClass = ImageResourceClass::Storage; + image.read = false; + image.written = !atomic; + image.atomic = atomic; + plan.info.images[0] = image; + plan.info.runtimeImageModes[0] = ResourceMaterializer::RuntimeImageModes(image); + expectFailure([&] { ResourceMaterializer{}.Materialize(plan, SrtRuntime{}, snapshot); }, "runtime image descriptor is invalid", + atomic ? "TYPE=0 or 1 atomic image descriptor was accepted without hardware evidence" : "TYPE=0 or 1 writable image descriptor was accepted without hardware evidence"); + } } } @@ -869,7 +871,7 @@ void verifyGuardedDescriptorSources() { static Texture texture; const auto base = static_cast(reinterpret_cast(texture.bytes.data())); const std::array valid{static_cast(base >> 8u), static_cast((base >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (3u << 30u), 3u << 14u, 0xfacu | (Type2D << 28u), 0u, 0u, 0u, 0u}; - const std::array invalid{0x12345678u, 0xffffffffu, 0xffffffffu, 1u << 28u, 0u, 0u, 0u, 0u}; + const std::array invalid{0x12345678u, 0xffffffffu, 0xffffffffu, 2u << 28u, 0u, 0u, 0u, 0u}; const std::array sampler{0u, 0u, 0u, 0u}; std::array output{}; const auto outputAddress = static_cast(reinterpret_cast(output.data())); @@ -915,6 +917,14 @@ void verifyGuardedDescriptorSources() { AgcDriver::ShaderMemory memory({}); expectFailure([&] { static_cast(memory.Capture(request)); }, "runtime image descriptor is invalid", "guarded sources: an invalid T# behind a taken branch was accepted"); } + auto typeOne = valid; + typeOne[3] = 0xfacu | (1u << 28u); + fill(typeOne, 1u); + { + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(request); + require(capture->snapshot.images.size() == 1u && std::equal(typeOne.begin(), typeOne.end(), capture->snapshot.images[0].dwords.begin()), "guarded sources: a read-only TYPE 1 T# behind a taken branch was refused or altered"); + } fill(valid, 1u); { AgcDriver::ShaderMemory memory({}); @@ -2673,6 +2683,14 @@ void verifyMeshConfiguration() { other.graphics = GraphicsCompileContext{0u, {}, otherMesh, std::nullopt, {}}; RecompileCacheKey::Build(other, key); require(key != first && RecompileCacheKey::ContextHash(request) != RecompileCacheKey::ContextHash(other), "the cache keys ignore the mesh configuration"); + require(mesh.outputPrimitiveSize == 3u && replay.request.graphics->mesh->outputPrimitiveSize == 3u, "a mesh configuration does not output triangles by default"); + otherMesh = mesh; + otherMesh.outputPrimitiveSize = 2u; + other.graphics = GraphicsCompileContext{0u, {}, otherMesh, std::nullopt, {}}; + const auto lineReplay = RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(other)); + require(lineReplay.request.graphics.has_value() && lineReplay.request.graphics->mesh.has_value() && lineReplay.request.graphics->mesh->outputPrimitiveSize == 2u, "the mesh output primitive size was lost in serialization"); + RecompileCacheKey::Build(other, key); + require(key != first && RecompileCacheKey::ContextHash(request) != RecompileCacheKey::ContextHash(other), "the cache keys ignore the mesh output primitive size"); RecompileRequest hull{}; hull.shader = {ShaderStage::TessellationControl, 0x10000u, code, 0, {}}; hull.context.waveSize = 64; @@ -3049,7 +3067,7 @@ void verifyPixelRequestSerialization() { minimal.context.pixel->layerExported = true; minimal.context.pixel->ditheredAlphaToCoverage = true; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQR4AAAA=", "new requests did not use serialization version 30"); + require(requestPrefix(encoded, 8u) == "NVNQQSAAAAA=", "new requests did not use serialization version 32"); const std::array fallbacks{0x40u, 0x1ac4u}; auto withFallbacks = minimal; withFallbacks.context.linkedCallFallbacks = fallbacks; @@ -3167,7 +3185,7 @@ void verifyPixelRequestSerialization() { for (std::size_t bytes = 0; bytes < 45u; ++bytes) { expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage, stencil reference export flag, primitive class, layer export, dither or front-face flag, quad pixel mask or quad sample mask was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQR8AAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQSEAAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } static constexpr std::string_view version18Request{ @@ -3258,6 +3276,7 @@ void verifyVertexInfoVersion19() { require(offset + 4u <= plain.size() && flagged[offset] == 0xefu && flagged[offset + 1u] == 0xcdu && flagged[offset + 2u] == 0xabu && flagged[offset + 3u] == 0x89u, "PA_CL_VS_OUT_CNTL is not the four bytes the version 20 vertex info adds"); require(std::equal(plain.begin() + offset + 4u, plain.end(), flagged.begin() + offset + 4u), "PA_CL_VS_OUT_CNTL moved the fields after it"); auto version20Vertex = flagged; + version20Vertex.pop_back(); version20Vertex[4] = 20u; const auto replay20Vertex = serializer.Deserialize(encodeBase64(version20Vertex)); require(replay20Vertex.request.context.vertex.has_value() && replay20Vertex.request.context.vertex->paClVsOutCntl == 0x89abcdefu, "a main version20 request lost PA_CL_VS_OUT_CNTL"); @@ -3266,6 +3285,7 @@ void verifyVertexInfoVersion19() { const auto layered = serializer.Deserialize(serializer.Serialize(vertex)); require(layered.request.context.vertex.has_value() && layered.request.context.vertex->paClVsOutCntl == vertex.context.vertex->paClVsOutCntl, "a layered vertex info lost its clip distances or render target index"); auto legacy = plain; + legacy.pop_back(); legacy.erase(legacy.begin() + offset, legacy.begin() + offset + 4u); for (const auto version : {16u, 17u, 18u, 19u}) { legacy[4] = static_cast(version); @@ -3313,21 +3333,25 @@ void verifyVertexInfoVersion19() { const auto sampledBytes = decodeBase64(serializer.Serialize(sampledPixel)); require(std::equal(maskedBytes.begin(), maskedBytes.begin() + static_cast(maskOffset + 1u), sampledBytes.begin()) && sampledBytes[maskOffset + 1u] == 0x1u && sampledBytes[maskOffset + 5u] == 0x2u && sampledBytes[maskOffset + 9u] == 0x4u && sampledBytes[maskOffset + 13u] == 0x8u, "the quad sample masks are not the four words after the quad pixel mask"); auto version29 = sampledBytes; + version29.pop_back(); version29.erase(version29.begin() + static_cast(maskOffset + 1u + sampleMaskBytes), version29.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u)); version29[4] = 29u; const auto replay29 = serializer.Deserialize(encodeBase64(version29)); require(replay29.request.context.pixel.has_value() && replay29.request.context.pixel->quadSampleMask == std::array{0x1u, 0x2u, 0x4u, 0x8u} && replay29.request.context.pixel->reversedBlend == 0u && decodeBase64(serializer.Serialize(replay29.request)) == sampledBytes, "a version 29 request read a reversed-order blend code or misread the fields after the quad sample masks"); auto version28 = sampledBytes; + version28.pop_back(); version28.erase(version28.begin() + static_cast(maskOffset + 1u), version28.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u)); version28[4] = 28u; const auto replay28 = serializer.Deserialize(encodeBase64(version28)); require(replay28.request.context.pixel.has_value() && replay28.request.context.pixel->quadPixelMask == 0x2u && replay28.request.context.pixel->quadSampleMask == std::array{0xffffu, 0xffffu, 0xffffu, 0xffffu} && replay28.request.context.pixel->reversedBlend == 0u && decodeBase64(serializer.Serialize(replay28.request)) == maskedBytes, "a version 28 request read quad sample masks or a reversed-order blend code, or misread the fields after them"); auto version27 = maskedBytes; + version27.pop_back(); version27.erase(version27.begin() + static_cast(maskOffset), version27.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u)); version27[4] = 27u; const auto replay27 = serializer.Deserialize(encodeBase64(version27)); require(replay27.request.context.pixel.has_value() && replay27.request.context.pixel->quadPixelMask == 0xfu && replay27.request.context.pixel->reversedBlend == 0u && serializer.Serialize(replay27.request) == encodedPixel, "a version 27 request read a quad pixel mask or reversed-order blend code, or misread the fields after it"); auto legacyPixel = plainPixel; + legacyPixel.pop_back(); legacyPixel.erase(legacyPixel.begin() + static_cast(allBitsOffset), legacyPixel.begin() + static_cast(allBitsOffset + 2u + sampleMaskBytes + 4u)); legacyPixel[4] = 26u; const auto replay26 = serializer.Deserialize(encodeBase64(legacyPixel)); @@ -4290,6 +4314,16 @@ void verifyUnusedUnnormalizedSampler() { snapshot.samplers[0].dwords[2] = 0x05500180u; require(atBaseLevel(levelZero) == std::vector{false}, "unnormalized samplers: an S# with a LOD bias was flagged as exact at the base level"); snapshot.samplers[0].dwords[2] = 0x05500000u; + auto implicitUse = levelZero; + implicitUse.samplers[0].uses = SamplerUseImplicitLod; + snapshot.samplers[0].dwords[1] = 0x00fff100u; + snapshot.samplers[0].dwords[2] = 0x01500180u; + require(atBaseLevel(implicitUse) == std::vector{true}, "unnormalized samplers: an image read through an S# without a mip filter was not flagged as exact at its base level"); + require(atBaseLevel(selectedLod) == std::vector{true}, "unnormalized samplers: an image read at a selected LOD through an S# without a mip filter was not flagged as exact at its base level"); + snapshot.samplers[0].dwords[2] = 0x05500180u; + require(atBaseLevel(implicitUse) == std::vector{false}, "unnormalized samplers: an image read through an S# with a point mip filter was flagged as exact at its base level"); + snapshot.samplers[0].dwords[1] = 0x00fff000u; + snapshot.samplers[0].dwords[2] = 0x05500000u; snapshot.samplers[0].dwords[0] = 0x00000092u; const auto normalized = populate(info); require(normalized[0].imageUnnormalized == std::vector{false} && normalized[1].samplerUnnormalized == std::vector{false}, "unnormalized samplers: a normalized S# was flagged"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ComputePipelineOutsideGpuMutex.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ComputePipelineOutsideGpuMutex.cpp index 95513fdc4b..dd464ab921 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ComputePipelineOutsideGpuMutex.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ComputePipelineOutsideGpuMutex.cpp @@ -21,7 +21,7 @@ void Require(bool condition, const std::string& message) { } alignas(256) std::array ReaderCode{0xf4040100u, 0xfa000000u, 0xf4080201u, 0xfa000000u, 0xbf8cc07fu, 0xf4000182u, 0xfa000000u, 0xbf8cc07fu, 0x7e000206u, 0xe0700000u, 0x80020000u, 0xbf810000u}; -alignas(256) std::array NewReaderCode{0xbf800000u, 0xf4040100u, 0xfa000000u, 0xf4080201u, 0xfa000000u, 0xbf8cc07fu, 0xf4000182u, 0xfa000000u, 0xbf8cc07fu, 0x7e000206u, 0xe0700000u, 0x80020000u, 0xbf810000u}; +alignas(256) std::array NewReaderCode{0xf4040100u, 0xfa000000u, 0xf4080201u, 0xfa000000u, 0xbf8cc07fu, 0xf4000182u, 0xfa000000u, 0xbf8cc07fu, 0x7e000206u, 0xe0700000u, 0x80020000u, 0xe0700004u, 0x80020000u, 0xbf810000u}; struct Slot { alignas(256) std::array value{}; @@ -104,7 +104,7 @@ int main() { Require(Slots[1].done == 0, "the dispatch finished while another thread held the GPU mutex"); } WaitDone(Slots[1], "the dispatch of the new program did not finish"); - Require(Slots[1].output[0] == 0x5678u, "the dispatch of the new program stored " + std::to_string(Slots[1].output[0])); + Require(Slots[1].output[0] == 0x5678u && Slots[1].output[1] == 0x5678u, "the dispatch of the new program stored " + std::to_string(Slots[1].output[0]) + " and " + std::to_string(Slots[1].output[1])); const auto after = VulkanDevice::ComputePipelineCounts(); Require(after.underGpuMutex == before.underGpuMutex, "a compute pipeline was created under the GPU mutex"); std::cout << "compute pipeline outside GPU mutex test passed\n"; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp new file mode 100644 index 0000000000..358d6a2c99 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp @@ -0,0 +1,169 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::DriverDetail::DrawMiss; + +alignas(256) constexpr std::array VertexCode{0x7e000208u, 0x7e020280u, 0x7e040280u, 0x7e0602f2u, 0xf80008cfu, 0x03020100u, 0xbf810000u, 0xbf810000u}; +alignas(256) constexpr std::array SecondVertexCode = VertexCode; +alignas(256) constexpr std::array PixelCode{0xf4000100u, 0xfa000000u, 0x7e000204u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u}; +alignas(16) std::array Other{0.5f, 0.0f, 0.0f, 0.0f}; +alignas(256) std::array ColorMemory{}; + +struct Header { + Shader shader{}; + ShaderUserData userData{}; +}; + +alignas(8) Header VertexHeader{}; +alignas(8) Header PixelHeader{}; +alignas(8) Header SecondVertexHeader{}; + +const Shader* Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) { + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18u; + header.shader.header_size = sizeof(Header); + header.shader.shader_size = static_cast(bytes); + header.shader.code = code; + header.shader.type = type; + header.shader.user_data = &header.userData; + AgcDriverRegisterShader_nid_postfix(&header.shader); + return &header.shader; +} + +void Append(std::vector& commands, std::uint32_t opcode, std::initializer_list payload) { + commands.push_back(0xc0000000u | (static_cast(payload.size() - 1) << 16u) | (opcode << 8u)); + commands.insert(commands.end(), payload); +} + +std::vector DrawFrom(std::initializer_list sources, const std::uint32_t* vertexCode = VertexCode.data()) { + const auto color = reinterpret_cast(ColorMemory.data()); + const auto vertex = reinterpret_cast(vertexCode); + const auto pixel = reinterpret_cast(PixelCode.data()); + const std::vector> context{ + {0x2d5, 0x2000}, + {0x1b6, 0}, {0x207, 0}, {0x200, 0}, {0x203, 0x800}, + {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0}, + {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa}, + {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff}, + {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240}, + {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010}, + {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, 0x28028}, + {0x31b, 0}, {0x31d, 0}, {0x3b0, (63u << 14u) | 3u}, + {0x3b8, 0x9000000}, {0x1e0, 0}, + {0xc, 0}, {0xd, 0x40040}, + {0x81, 0x80000000}, {0x82, 0x40040}, + {0x90, 0x80000000}, {0x91, 0x40040}, + {0x94, 0x80000000}, {0x95, 0x40040}, + {0x318, static_cast(color >> 8u)}, {0x390, static_cast(color >> 40u)}, + {0x10f, std::bit_cast(32.0f)}, {0x110, std::bit_cast(32.0f)}, + {0x111, std::bit_cast(-2.0f)}, {0x112, std::bit_cast(2.0f)}, + {0x113, std::bit_cast(1.0f)}, {0x114, 0}, + {0xb4, 0}, {0xb5, std::bit_cast(1.0f)}, + }; + const std::vector> shader{ + {0xc8, static_cast(vertex >> 8u)}, {0xc9, static_cast(vertex >> 40u)}, {0x8b, 1u << 1u}, {0x8c, 0}, + {0x008, static_cast(pixel >> 8u)}, {0x009, static_cast(pixel >> 40u)}, {0x00b, 2u << 1u}, + }; + std::vector commands; + Append(commands, 0x79, {0x242, 4}); + for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value}); + for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value}); + for (const auto* source : sources) { + const auto address = reinterpret_cast(source); + Append(commands, 0x76, {0x00c, static_cast(address)}); + Append(commands, 0x76, {0x00d, static_cast(address >> 32u)}); + Append(commands, 0x2d, {3, 2}); + } + return commands; +} + +void MovedDataHitsTheCache(bool libraries) { + Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2); + Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1); + const auto mode = std::string(libraries ? "with" : "without") + " graphics pipeline libraries"; + const auto draw = [&](std::initializer_list sources) { + auto commands = DrawFrom(sources); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw submission failed"); + AgcDriverWaitIdle_nid_postfix(); + return AgcDriver::DriverDetail::Driver::Get().DrawCacheCounters(); + }; + static std::array, 8> same{}; + for (auto& entry : same) entry = {1.0f, 0.0f, 0.0f, 0.0f}; + AgcDriver::DriverDetail::DrawEntryCounters counters{}; + for (std::size_t i = 0; i < 4; ++i) counters = draw({same[i].data()}); + Require(counters.lookups == 4 && counters.hits == 0 && counters.inserts == 2, mode + ": the draws that teach the moved pointer and its contents made " + std::to_string(counters.hits) + " hits and " + std::to_string(counters.inserts) + " inserts, expected 0 and 2"); + for (std::size_t i = 4; i < same.size(); ++i) counters = draw({same[i].data()}); + Require(counters.lookups == 8 && counters.hits == 4 && counters.inserts == 2, mode + ": draws whose data moved again did not reuse the cached draw (" + std::to_string(counters.hits) + " hits in " + std::to_string(counters.lookups) + " lookups)"); + const auto other = draw({Other.data()}); + Require(other.lookups == 9 && other.hits == 4, mode + ": a draw whose moved data holds another value reused the cached draw"); + const auto again = draw({same[0].data()}); + Require(again.lookups == 10 && again.hits == 5, mode + ": a draw at an address seen before did not reuse the cached draw (" + std::to_string(again.hits) + " hits)"); +} + +void UnrelocatedAliasesStayOut(bool libraries) { + Registered(SecondVertexHeader, SecondVertexCode.data(), sizeof(SecondVertexCode), 2); + Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1); + const auto mode = std::string(libraries ? "with" : "without") + " graphics pipeline libraries"; + const auto draw = [&](const float* source) { + auto commands = DrawFrom({source}, SecondVertexCode.data()); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw submission failed"); + AgcDriverWaitIdle_nid_postfix(); + return AgcDriver::DriverDetail::Driver::Get().DrawCacheCounters(); + }; + alignas(16) static std::array repeated{1.0f, 0.0f, 0.0f, 0.0f}; + alignas(16) static std::array, 2>, 12> twice{}; + for (std::size_t i = 0; i < twice.size(); ++i) { + for (auto& entry : twice[i]) entry = {0.25f + 0.03125f * static_cast(i), 0.0f, 0.0f, 0.0f}; + } + draw(repeated.data()); + const auto start = draw(repeated.data()); + AgcDriver::DriverDetail::DrawEntryCounters counters = start; + for (const auto& pair : twice) { + for (const auto& entry : pair) draw(entry.data()); + counters = draw(repeated.data()); + } + const auto inserts = counters.inserts - start.inserts; + Require(inserts <= AgcDriver::DriverDetail::DrawRelocationTrial::ProbationAdmissions, mode + ": draws whose alias repeats but never relocates were cached (" + std::to_string(inserts) + " inserts for " + std::to_string(twice.size()) + " aliases)"); + Require(counters.hits - start.hits == twice.size(), mode + ": the exact key of a repeated draw lost its hits next to draws whose alias never relocates (" + std::to_string(counters.hits - start.hits) + " hits in " + std::to_string(twice.size()) + " draws)"); +} + +} + +int main() { + try { + bool libraries = false; + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + libraries = device->GraphicsPipelineLibraries(); + } + if (!libraries && std::getenv("APS5_NO_GPL") == nullptr) return VulkanTestSkipped; + MovedDataHitsTheCache(libraries); + UnrelocatedAliasesStayOut(libraries); + AgcDriverShutdown_nid_postfix(); + std::cout << "A draw whose data moved reused the cached draw " << (libraries ? "with" : "without") << " graphics pipeline libraries\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + AgcDriverShutdown_nid_postfix(); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F16ConversionFloatMode.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F16ConversionFloatMode.cpp new file mode 100644 index 0000000000..86aa123c60 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F16ConversionFloatMode.cpp @@ -0,0 +1,193 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Inputs = 4; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +struct alignas(256) Shader { + std::array words; +}; + +struct Vector { + std::uint32_t a; + std::uint32_t b; + std::uint32_t expected[16]; +}; + +struct Case { + const char* name; + Shader shader; + std::span rows; +}; + +constexpr Vector VCvtF16F32Rows[] = { + {0x7f800001u, 0x00000000u, {0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u}}, + {0x00000001u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000001u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00800000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x33400000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x33800000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}}, + {0x33a00000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}}, + {0x33c00000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}}, + {0x387ff000u, 0x00000000u, {0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000000u, 0x00000000u, 0x000003ffu, 0x000003ffu, 0x00000000u, 0x00000000u, 0x000003ffu, 0x000003ffu}}, + {0x38800800u, 0x00000000u, {0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u}}, + {0x38801800u, 0x00000000u, {0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u}}, + {0x7f800001u, 0x00000000u, {0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u}}, + {0x80800000u, 0x00000000u, {0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u}}, + {0xb3400000u, 0x00000000u, {0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u}}, + {0xb3800000u, 0x00000000u, {0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u}}, + {0xb3a00000u, 0x00000000u, {0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008002u, 0x00008002u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u}}, + {0xb3c00000u, 0x00000000u, {0x00008000u, 0x00008000u, 0x00008002u, 0x00008002u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008002u, 0x00008002u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u}}, + {0xb87ff000u, 0x00000000u, {0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008000u, 0x00008000u, 0x000083ffu, 0x000083ffu, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008000u, 0x00008000u, 0x000083ffu, 0x000083ffu}}, + {0xb8800800u, 0x00000000u, {0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008401u, 0x00008401u, 0x00008401u, 0x00008401u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u}}, + {0xb8801800u, 0x00000000u, {0x00008401u, 0x00008401u, 0x00008401u, 0x00008401u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008401u, 0x00008401u, 0x00008401u, 0x00008401u, 0x00008400u, 0x00008400u, 0x00008400u, 0x00008400u}}, +}; + +constexpr Vector VCvtF16F32ClampRows[] = { + {0x00800000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x33400000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x33800000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}}, + {0x33a00000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}}, + {0x33c00000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}}, + {0x387ff000u, 0x00000000u, {0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000000u, 0x00000000u, 0x000003ffu, 0x000003ffu, 0x00000000u, 0x00000000u, 0x000003ffu, 0x000003ffu}}, + {0x38800800u, 0x00000000u, {0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u}}, + {0x38801800u, 0x00000000u, {0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000401u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u, 0x00000400u}}, +}; + +constexpr Vector VCvtF16U16Rows[] = { + {0x00000801u, 0x00000000u, {0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u}}, + {0x00000803u, 0x00000000u, {0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u}}, + {0x00008001u, 0x00000000u, {0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007801u, 0x00007801u, 0x00007801u, 0x00007801u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u}}, + {0x00008011u, 0x00000000u, {0x00007801u, 0x00007801u, 0x00007801u, 0x00007801u, 0x00007801u, 0x00007801u, 0x00007801u, 0x00007801u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u, 0x00007800u}}, +}; + +constexpr Vector VCvtF16I16Rows[] = { + {0x00000801u, 0x00000000u, {0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u, 0x00006800u}}, + {0x00000803u, 0x00000000u, {0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006802u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u, 0x00006801u}}, + {0x00008001u, 0x00000000u, {0x0000f800u, 0x0000f800u, 0x0000f800u, 0x0000f800u, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f800u, 0x0000f800u, 0x0000f800u, 0x0000f800u, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu}}, + {0x00008009u, 0x00000000u, {0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f800u, 0x0000f800u, 0x0000f800u, 0x0000f800u, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu, 0x0000f7ffu}}, +}; + +constexpr Vector VLdexpF16Rows[] = { + {0x00000001u, 0x0000ffe2u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000001u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u}}, + {0x00000001u, 0x0000000au, {0x00000000u, 0x00000400u, 0x00000000u, 0x00000400u, 0x00000000u, 0x00000400u, 0x00000000u, 0x00000400u, 0x00000000u, 0x00000400u, 0x00000000u, 0x00000400u, 0x00000000u, 0x00000400u, 0x00000000u, 0x00000400u}}, + {0x00000001u, 0x000003e8u, {0x00000000u, 0x00007c00u, 0x00000000u, 0x00007c00u, 0x00000000u, 0x00007c00u, 0x00000000u, 0x00007c00u, 0x00000000u, 0x00007bffu, 0x00000000u, 0x00007bffu, 0x00000000u, 0x00007bffu, 0x00000000u, 0x00007bffu}}, + {0x00000003u, 0x0000fffeu, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000003u, 0x0000ffffu, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u}}, + {0x00000155u, 0x0000fff8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u}}, + {0x00000400u, 0x0000ffe2u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000400u, 0x0000fff6u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}}, + {0x00000400u, 0x0000001eu, {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu}}, + {0x00000401u, 0x0000fff5u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000401u, 0x0000fff6u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}}, + {0x00000555u, 0x0000fff7u, {0x00000000u, 0x00000000u, 0x00000003u, 0x00000003u, 0x00000000u, 0x00000000u, 0x00000003u, 0x00000003u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u, 0x00000000u, 0x00000000u, 0x00000002u, 0x00000002u}}, + {0x00008001u, 0x0000ffe2u, {0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u}}, + {0x00008001u, 0x00000000u, {0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u}}, + {0x00008001u, 0x0000000au, {0x00008000u, 0x00008400u, 0x00008000u, 0x00008400u, 0x00008000u, 0x00008400u, 0x00008000u, 0x00008400u, 0x00008000u, 0x00008400u, 0x00008000u, 0x00008400u, 0x00008000u, 0x00008400u, 0x00008000u, 0x00008400u}}, + {0x00008001u, 0x000003e8u, {0x00008000u, 0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00008000u, 0x0000fbffu, 0x00008000u, 0x0000fbffu, 0x00008000u, 0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00008000u, 0x0000fbffu, 0x00008000u, 0x0000fbffu}}, + {0x00008003u, 0x0000fffeu, {0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u}}, + {0x00008003u, 0x0000ffffu, {0x00008000u, 0x00008000u, 0x00008000u, 0x00008002u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008002u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u}}, + {0x00008155u, 0x0000fff8u, {0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008002u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u}}, + {0x00008400u, 0x0000ffe2u, {0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u}}, + {0x00008400u, 0x0000fff6u, {0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u}}, + {0x00008400u, 0x0000001eu, {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fbffu, 0x0000fbffu, 0x0000fbffu, 0x0000fbffu, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fbffu, 0x0000fbffu, 0x0000fbffu, 0x0000fbffu}}, + {0x00008401u, 0x0000fff5u, {0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u}}, + {0x00008401u, 0x0000fff6u, {0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u, 0x00008000u, 0x00008000u, 0x00008002u, 0x00008002u, 0x00008000u, 0x00008000u, 0x00008001u, 0x00008001u}}, + {0x00008555u, 0x0000fff7u, {0x00008000u, 0x00008000u, 0x00008003u, 0x00008003u, 0x00008000u, 0x00008000u, 0x00008002u, 0x00008002u, 0x00008000u, 0x00008000u, 0x00008003u, 0x00008003u, 0x00008000u, 0x00008000u, 0x00008002u, 0x00008002u}}, +}; + +const Case Cases[] = { + {"v_cvt_f16_f32", {{0x34020084u, 0x34040082u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e141504u, 0xe0701000u, 0x80010a02u, 0xbf810000u, 0xbf810000u, 0xbf810000u}}, VCvtF16F32Rows}, + {"v_cvt_f16_f32 clamp", {{0x34020084u, 0x34040082u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0xd58a800au, 0x00000104u, 0xe0701000u, 0x80010a02u, 0xbf810000u, 0xbf810000u}}, VCvtF16F32ClampRows}, + {"v_cvt_f16_u16", {{0x34020084u, 0x34040082u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e14a104u, 0xe0701000u, 0x80010a02u, 0xbf810000u, 0xbf810000u, 0xbf810000u}}, VCvtF16U16Rows}, + {"v_cvt_f16_i16", {{0x34020084u, 0x34040082u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e14a304u, 0xe0701000u, 0x80010a02u, 0xbf810000u, 0xbf810000u, 0xbf810000u}}, VCvtF16I16Rows}, + {"v_ldexp_f16", {{0x34020084u, 0x34040082u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x76140b04u, 0xe0701000u, 0x80010a02u, 0xbf810000u, 0xbf810000u, 0xbf810000u}}, VLdexpF16Rows}, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Run(AgcDriver::VulkanDevice& device, const Case& item, std::uint32_t round16, std::uint32_t denorm16) { + Input.fill(0u); + for (std::uint32_t row = 0; row < item.rows.size(); ++row) { + Input[row * Inputs] = item.rows[row].a; + Input[row * Inputs + 1] = item.rows[row].b; + } + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(item.shader.words); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{(round16 << 2u) | (denorm16 << 6u), true, false, false}; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const Case& item, std::uint32_t round16, std::uint32_t denorm16) { + for (std::uint32_t row = 0; row < item.rows.size(); ++row) { + const auto& vector = item.rows[row]; + const auto expected = vector.expected[round16 * 4u + denorm16]; + Require(Output[row] == expected, std::string(item.name) + " round16 " + std::to_string(round16) + " denorm16 " + std::to_string(denorm16) + " inputs " + + Hex(vector.a) + " " + Hex(vector.b) + " is " + Hex(Output[row]) + ", expected " + Hex(expected)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + for (const auto& item : Cases) { + for (std::uint32_t round16 = 0u; round16 < 4u; ++round16) { + for (std::uint32_t denorm16 = 0u; denorm16 < 4u; ++denorm16) { + Run(*device, item, round16, denorm16); + Check(item, round16, denorm16); + } + } + } + std::puts("f16 conversion float mode tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp index a178c8c162..8daddabd90 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp @@ -55,9 +55,15 @@ alignas(256) constexpr std::array LoadCode{ 0xbf810000, }; -alignas(256) constexpr std::array D16Code{0x7e3c0300, 0x7e3e0280, 0xf0201108, 0x8001141e, 0xbf810000}; +alignas(256) constexpr std::array D16Code{ + 0x34020084, 0xe0381000, 0x80001401, 0xbf8c3f70, 0x7e3c0300, 0x7e3e0280, 0xf0201f08, 0x8001141e, + 0x7e3e0281, 0xf0201508, 0x8001141e, 0xbf810000, +}; alignas(256) constexpr std::array PackedCode{0x7e3c0300, 0x7e3e0280, 0xf0281108, 0x0001141e, 0xbf810000}; -alignas(256) constexpr std::array AtomicD16Code{0x7e3c0300, 0x7e3e0280, 0xf0441108, 0x0001141e, 0xf0201108, 0x8001141e, 0xbf810000}; +alignas(256) constexpr std::array AtomicD16Code{ + 0x34020084, 0xe0381000, 0x80001401, 0xbf8c3f70, 0x7e3c0300, 0x7e3e0280, 0xf0441108, 0x0001141e, + 0xf0201108, 0x8001141e, 0xbf810000, +}; alignas(256) constexpr std::array PackedWideCode{0x34020084, 0xe0381000, 0x80001401, 0xbf8c3f70, 0x7e3c0300, 0x7e3e0280, 0xf0281308, 0x0001141e, 0xbf810000}; alignas(256) constexpr std::array AtomicCode{0x7e3c0300, 0x7e3e0280, 0xf0441108, 0x0001141e, 0xbf810000}; @@ -181,13 +187,21 @@ std::uint32_t StoredComponent(const Format& format, std::uint32_t tid, std::uint return Saturate(Value(tid, index), format.bits); } -std::vector Expected(const Format& format, std::span writes) { +std::uint32_t StoredD16Component(const Format& format, std::uint32_t tid, std::uint32_t dmask, std::uint32_t component) { + if (((dmask >> component) & 1u) == 0u) return 0u; + std::uint32_t index = 0; + for (std::uint32_t lower = 0; lower < component; ++lower) index += (dmask >> lower) & 1u; + const auto half = (Value(tid, index / 2u) >> (16u * (index & 1u))) & 0xffffu; + return Saturate(SignExtend(half, 16u), format.bits); +} + +std::vector Expected(const Format& format, std::span writes, bool d16 = false) { auto bytes = Initial(); for (std::uint32_t tid = 0; tid < Threads; ++tid) { for (const auto& write : writes) { auto* texel = bytes.data() + TexelOffset(format, tid, write.row); for (std::uint32_t component = 0; component < format.components; ++component) { - const auto value = StoredComponent(format, tid, write.dmask, component); + const auto value = d16 ? StoredD16Component(format, tid, write.dmask, component) : StoredComponent(format, tid, write.dmask, component); std::memcpy(texel + component * format.bits / 8u, &value, format.bits / 8u); } } @@ -244,8 +258,8 @@ void Store(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span(Input.size() * 4u))); } -void CheckTexels(const std::uint8_t* texels, const Format& format, std::span writes, const std::string& what) { - const auto expected = Expected(format, writes); +void CheckTexels(const std::uint8_t* texels, const Format& format, std::span writes, const std::string& what, bool d16 = false) { + const auto expected = Expected(format, writes, d16); for (std::uint32_t y = 0; y < Height; ++y) { for (std::uint32_t x = 0; x < Width; ++x) { const auto offset = TexelOffset(format, x, y); @@ -297,8 +311,10 @@ int main() { } Store(*device, texels, TitleCode, Formats[0].format, TitleSwizzle); CheckTexels(texels, Formats[0], TitleWrites, "image_store dmask:0x1 glc with DST_SEL X,1,1,1"); - RequireRefused(*device, texels, D16Code, 12u, "stores 16-bit data to an image of a SINT format", "image_store d16 of 16_SINT"); - RequireRefused(*device, texels, D16Code, 21u, "stores 16-bit data to an image of a SINT format", "image_store d16 of 32_SINT"); + for (const auto& format : Formats) { + Store(*device, texels, D16Code, format.format, SwizzleOf(format)); + CheckTexels(texels, format, StoreWrites, "image_store d16", true); + } RequireRefused(*device, texels, PackedCode, 12u, "storage image descriptor uses an unsupported format", "image_store_pck of 16_SINT"); RequireRefused(*device, texels, AtomicCode, 12u, "atomic image descriptor uses an unsupported format 12", "image_atomic_add of 16_SINT"); RequireRefused(*device, texels, StoreCode, 73u, "storage image descriptor uses an unsupported format", "image_store of 32_32_32_SINT"); @@ -306,7 +322,8 @@ int main() { Store(*device, texels, PackedWideCode, Formats[index].format, SwizzleOf(Formats[index])); CheckTexels(texels, Formats[index], PackedWideWrites, "image_store_pck dmask:0x3"); } - RequireRefused(*device, texels, AtomicD16Code, 21u, "stores 16-bit data to an image of a SINT format", "image_store d16 of an atomically updated 32_SINT image"); + Store(*device, texels, AtomicD16Code, Formats[2].format, IdentitySwizzle); + CheckTexels(texels, Formats[2], TitleWrites, "image_store d16 after image_atomic_add", true); std::puts("image store sint tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp new file mode 100644 index 0000000000..eb4f2499b6 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp @@ -0,0 +1,140 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Lanes = 64; +constexpr std::uint32_t Slots = 16; +constexpr std::uint32_t Stride = 4; +constexpr std::uint32_t Guard = 4096; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array WaterfallCode{ + 0x34240083, 0x7e040280, 0x7e060280, 0xd9340000, 0x00000212, 0xbf8cc07f, 0x16080085, 0x3608088f, + 0x34080883, 0xd6ff0002, 0x00010300, 0xd9280000, 0x00000204, 0xbf8cc07f, 0xd9d80000, 0x06000012, + 0xbf8cc07f, 0xd4e50002, 0x00020c80, 0x7e140280, 0x7e160280, 0xbe8a0380, 0xbefe0400, 0xbf138002, + 0xbf840016, 0xbe8b1402, 0xbe821c0b, 0x8f0c830b, 0x7e18020c, 0xd9d80000, 0x0e00000c, 0xbf8cc07f, + 0x7dca1c80, 0xbf86fff5, 0x7e1c050e, 0x7e1e050f, 0xbe90140e, 0x8f921081, 0x3a1c1c12, 0x3a1e1e13, + 0x4a141481, 0x4a161610, 0x800a810a, 0xbf08ff0a, 0x00001000, 0xbf850001, 0xbf82fff1, 0xbefe04c1, + 0x7e1a020a, 0xe0702000, 0x80010a00, 0xe0702004, 0x80010b00, 0xe0702008, 0x80010d00, 0xe070200c, + 0x80010600, 0xbf810000, +}; + +alignas(256) constexpr std::array MaskedOrCode{ + 0x34240083, 0x7e040280, 0x7e060280, 0xd9340000, 0x00000212, 0xbf8cc07f, 0x3608008f, 0x34080883, + 0xd6ff0002, 0x00010300, 0xbefe0400, 0xd9280000, 0x00000204, 0xbf8cc07f, 0xbefe04c1, 0xd9d80000, + 0x06000012, 0xbf8cc07f, 0xe0702000, 0x80010600, 0xe0702004, 0x80010700, 0xbf810000, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t records) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | ((Stride * 4u) << 16u), records, 0x11016facu}; +} + +void Run(AgcDriver::VulkanDevice& device, std::span code, std::uint64_t exec, bool spanned) { + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + userData[0] = static_cast(exec); + userData[1] = static_cast(exec >> 32u); + const auto output = BufferDescriptor(Output.data(), Lanes); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const auto address = reinterpret_cast(code.data()); + const std::array memory{{{address, std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, Lanes * 2u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, address, code, 0, {}}, + {Lanes, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.target.spannedWaves = spanned; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + Require(result.fullSubgroups == spanned, spanned ? "the program did not span two host subgroups" : "the program spans host subgroups without APS5_SPANNED_WAVE"); + device.Dispatch(result, 1, 1, 1, {}, address); + device.WaitIdle(); +} + +void Check(std::uint64_t exec, bool spanned) { + std::array slots{}; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) slots[(lane * 5u) & (Slots - 1u)] |= std::uint64_t{1} << lane; + std::uint32_t iterations = 0; + std::uint32_t indexSum = 0; + for (std::uint32_t slot = 0; slot < Slots; ++slot) { + for (auto mask = slots[slot]; mask != 0; mask &= mask - 1) { + ++iterations; + indexSum += static_cast(std::countr_zero(mask)); + } + } + char name[64]; + std::snprintf(name, sizeof(name), "%s, exec 0x%016llx", spanned ? "spanned" : "two-lane", static_cast(exec)); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + const bool active = ((exec >> lane) & 1u) != 0; + const std::array expected{active ? iterations : 0u, active ? indexSum : 0u, iterations, static_cast(slots[lane])}; + const std::array fields{"inner iterations", "bit index sum", "wave iterations", "own slot low word"}; + for (std::uint32_t field = 0; field < Stride; ++field) { + const auto actual = Output[lane * Stride + field]; + Require(actual != Guard + 1u || field != 2u, std::string(name) + ": the waterfall ran past " + std::to_string(Guard) + " iterations"); + Require(actual == expected[field], std::string(name) + ": lane " + std::to_string(lane) + " " + fields[field] + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[field])); + } + } +} + +void CheckMaskedOr(std::uint64_t exec, bool spanned) { + std::array slots{}; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + if (((exec >> lane) & 1u) != 0) slots[lane & (Slots - 1u)] |= std::uint64_t{1} << lane; + } + char name[64]; + std::snprintf(name, sizeof(name), "%s masked ds_or_b64, exec 0x%016llx", spanned ? "spanned" : "two-lane", static_cast(exec)); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + const std::array expected{static_cast(slots[lane]), static_cast(slots[lane] >> 32u)}; + for (std::uint32_t word = 0; word < expected.size(); ++word) { + const auto actual = Output[lane * Stride + word]; + Require(actual == expected[word], std::string(name) + ": lane " + std::to_string(lane) + " slot word " + std::to_string(word) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[word])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (device->Target().subgroupSize < 32) { + std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); + return VulkanTestSkipped; + } + const bool spannable = device->Target().subgroupSize == 32u && device->ComputeFullSubgroups(); + if (!spannable) std::puts("spanned mode skipped: the device has no full 32-lane compute subgroups"); + for (const bool spanned : {false, true}) { + if (spanned && !spannable) continue; + for (const std::uint64_t exec : {~std::uint64_t{0}, std::uint64_t{0xaaaaaaaaaaaaaaaa}, std::uint64_t{0xffff000000000000}, std::uint64_t{0x000000000000ffff}, std::uint64_t{1} << 63u, std::uint64_t{1}, std::uint64_t{0x0000000100000000}}) { + Run(*device, WaterfallCode, exec, spanned); + Check(exec, spanned); + } + for (const std::uint64_t exec : {~std::uint64_t{0}, std::uint64_t{0x00000000ffffffff}, std::uint64_t{0xffffffff00000000}, std::uint64_t{1} << 40u, std::uint64_t{0x8000000000000001}}) { + Run(*device, MaskedOrCode, exec, spanned); + CheckMaskedOr(exec, spanned); + } + } + std::printf("LDS mask waterfall tests passed (two-lane%s)\n", spannable ? " and spanned" : ""); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PipelineJournal.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PipelineJournal.cpp new file mode 100644 index 0000000000..16d3742761 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PipelineJournal.cpp @@ -0,0 +1,431 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "../NarrowConstantStoreFixture.hpp" +#include "Recompiler.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::ColorTargetLayout; +using AgcDriver::Graphics::ColorTileMode; +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 32; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t Color1555Info = 0x8044u; +constexpr std::uint32_t LinearAttrib3 = 0x09000000u; +constexpr std::uint32_t Float32Export = 9; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +alignas(256) std::array PixelProgram{}; +std::size_t pixelProgramWords = 0; + +std::span Program(const std::array& color) { + std::size_t at = 0; + for (std::uint32_t channel = 0; channel < 4; ++channel) { + PixelProgram[at++] = 0x7e0002ffu | ((4u + channel) << 17u); + PixelProgram[at++] = std::bit_cast(color[channel]); + } + PixelProgram[at++] = 0xf800180fu; + PixelProgram[at++] = 0x07060504u; + PixelProgram[at++] = 0xbf810000u; + pixelProgramWords = at; + return std::span(PixelProgram.data(), at); +} + +class Block { +public: + explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (watched) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + if (watched) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the guest block"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate the guest block"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true, true); + } + + ~Block() { + AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (watched) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } +#else + if (watched) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + std::uint64_t Address() const { return reinterpret_cast(data); } + std::size_t bytes; + bool watched; + std::uint8_t* data = nullptr; +}; + +AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) { + AgcDriver::Registers cx; + cx[0x318] = static_cast(block.Address() >> 8u); + cx[0x31b] = 0; + cx[0x31c] = Color1555Info; + cx[0x31d] = 0; + cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u); + cx[0x3b8] = LinearAttrib3; + cx[0x390] = static_cast(block.Address() >> 40u); + auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0); + Require(color.format == VK_FORMAT_A1R5G5B5_UNORM_PACK16 && color.bytes == block.bytes, "the test target did not decode as an A1R5G5B5 surface"); + return color; +} + +void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span pixelCode) { + const auto target = device.Target(); + constexpr std::uint32_t waveSize = 64; + std::vector vertexUserData(4, 0u); + const auto address = reinterpret_cast(Triangle.data()); + const std::array vertexBuffer{static_cast(address), static_cast((address >> 32u) & 0xffffu) | (16u << 16u), static_cast(Triangle.size()), 0x01016facu}; + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.colors = {color}; + state.color = color; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + VkPipelineColorBlendAttachmentState blend{}; + blend.colorWriteMask = 0xf; + state.blends = {blend}; + state.blend = blend; + state.blendConstants = {}; + + const auto codeAddress = reinterpret_cast(pixelCode.data()); + const AgcDriver::Registers context{{0x1b6, 0x8000}, {0x1b3, 0}, {0x1b4, 0}, {0x203, 0}, {0x1c5, Float32Export}}; + const std::array pixelMemory{{{codeAddress, std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, codeAddress, pixelCode, 0, {}}, + {waveSize, 0, {}, std::nullopt, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + const auto handle = ShaderRecompiler::PrepareShader(fragment); + AgcDriver::DriverDetail::ShaderSnapshot snapshot{codeAddress, 0, 1, {pixelCode.begin(), pixelCode.end()}, {}}; + snapshot.prepared->entries.push_back({0, handle}); + fragment.shader.code = snapshot.code; + fragment.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(context, AgcDriver::Graphics::ExportMappings(state)); + const ShaderRecompiler::SrtRuntime runtime{}; + const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, fragment); + const auto pixelResult = invocation.Materialize(*invocation.Capture(runtime)); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, pixelResult.get(), PixelPushOffset(vertexPush, target)} + }}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); +} + +void ExpectTexel(AgcDriver::VulkanDevice& device, const Block& block, std::uint16_t texel, const char* what) { + AgcDriver::Graphics::StorageTexture::FlushPending(block.Address(), block.bytes, nullptr, "test read-back"); + device.WaitIdle(); + std::vector stored(block.bytes / 2u); + AgcDriver::GuestMemory::Read(block.Address(), std::as_writable_bytes(std::span(stored)), 1); + const ColorTargetLayout layout(Width, Height, ColorTileMode::Linear, 2); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + const auto index = layout.Offset(x, y) / 2u; + if (stored[index] == texel) continue; + char detail[160]; + std::snprintf(detail, sizeof(detail), ": pixel (%u, %u) holds 0x%04x, expected 0x%04x", x, y, stored[index], texel); + throw std::runtime_error(std::string(what) + detail); + } + } +} + +// The constant-store compute program with its two stored literals replaced: `salt` 0 is the +// fixture's program, every other value a program of its own with its own module. +std::vector ComputeCode(std::uint32_t salt) { + std::vector code(NarrowConstantStoreFixture::Code.begin(), NarrowConstantStoreFixture::Code.end()); + if (salt != 0) { + code[3] = 0x00a10200u ^ (salt * 0x01010101u); + code[5] = 0x0000fec3u ^ (salt * 0x00010001u); + } + return code; +} + +ShaderRecompiler::RecompileResult CompileCompute(AgcDriver::VulkanDevice& device, const std::vector& code, std::span userData, std::uint32_t waveSize) { + using namespace ShaderRecompiler; + const std::span words(code); + const std::array memory{{{reinterpret_cast(words.data()), std::as_bytes(words)}}}; + const ShaderComputeStageInfo compute{{NarrowConstantStoreFixture::Threads, 1, 1}, 0, {false, false, false}, false, 1, {}}; + RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(words.data()), words, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), {0, 0, 0, 128}, std::nullopt, false, + }; + return Recompile(request); +} + +void ExpectConstantStore(const Block& block, const char* what) { + using namespace NarrowConstantStoreFixture; + for (std::size_t offset = 0; offset < Threads * LaneBytes; ++offset) { + const auto expected = ExpectedLane[offset % LaneBytes]; + Require(block.data[offset] == expected, std::string(what) + ": byte " + std::to_string(offset) + " is " + std::to_string(block.data[offset]) + ", expected " + std::to_string(expected)); + } +} + +std::array AddressWords(const Block& block) { + const auto address = block.Address(); + return {static_cast(address), static_cast(address >> 32u)}; +} + +void SetEnvironment(const char* name, const std::string& value) { +#ifdef _WIN32 + _putenv_s(name, value.c_str()); +#else + setenv(name, value.c_str(), 1); +#endif +} + +struct Totals { + AgcDriver::Graphics::PipelineJournalCounters journal; + AgcDriver::Graphics::PipelineStoreCounters store; +}; + +Totals Snapshot() { + return {AgcDriver::Graphics::PipelineJournalTotals(), AgcDriver::Graphics::PipelineStoreTotals()}; +} + +void JournalTests() { + using namespace NarrowConstantStoreFixture; + Require(AgcDriver::Graphics::PipelineJournalEnabled(), "the pipeline journal is disabled although the shader cache directory is set"); + const auto journalPath = AgcDriver::Graphics::PipelineJournalPath(); + Block compute(BlockBytes); + Block target(ColorTargetLayout(Width, Height, ColorTileMode::Linear, 2).Bytes()); + const auto computeCode = ComputeCode(0); + const std::array red{1.0f, 0.0f, 0.0f, 1.0f}; + constexpr std::uint16_t redTexel = 0x801f; + { + const auto device = OpenVulkanTestDevice(); + Require(device != nullptr, "the first device is required once the second opened"); + device->WaitForPipelineJournalReplay(); + const auto before = Snapshot(); + Require(before.journal.loadedRecipes == 0 && before.journal.replayedGraphics == 0 && before.journal.replayedCompute == 0, "an empty journal replayed pipelines"); + std::memset(compute.data, Fill, compute.bytes); + const auto words = AddressWords(compute); + const auto result = CompileCompute(*device, computeCode, words, 32); + device->Dispatch(result, 1, 1, 1, {}, reinterpret_cast(computeCode.data())); + device->WaitIdle(); + ExpectConstantStore(compute, "first run: the constant store dispatch"); + std::memset(target.data, 0, target.bytes); + Draw(*device, DecodeTarget(target), Program(red)); + ExpectTexel(*device, target, redTexel, "first run: the draw"); + Require(device->ComputePipelinesCreated() == 1, "the first run did not build exactly one compute pipeline"); + AgcDriver::Graphics::FlushPipelineJournal(); + const auto after = Snapshot(); + Require(after.store.misses == before.store.misses + 1 && after.store.hits == before.store.hits, "the first draw did not miss the pipeline store exactly once"); + Require(after.journal.recordedGraphics == 1 && after.journal.recordedCompute == 1 && after.journal.recordedModules == 3 && after.journal.writeFailures == 0, "the first run did not record one graphics and one compute pipeline with three modules"); + Draw(*device, DecodeTarget(target), Program(red)); + device->WaitIdle(); + AgcDriver::Graphics::FlushPipelineJournal(); + const auto repeated = Snapshot(); + Require(repeated.store.hits == after.store.hits + 1 && repeated.store.misses == after.store.misses, "a repeated draw did not hit the pipeline store"); + Require(repeated.journal.recordedGraphics == 1 && repeated.journal.recordedModules == 3, "a repeated draw recorded its pipeline again"); + } + Require(std::filesystem::exists(journalPath) && std::filesystem::file_size(journalPath) > 0, "the journal file was not written"); + { + const auto device = OpenVulkanTestDevice(); + Require(device != nullptr, "the second device could not be opened"); + device->WaitForPipelineJournalReplay(); + const auto replayed = Snapshot(); + Require(replayed.journal.loadedRecipes == 2 && replayed.journal.loadedModules == 3 && replayed.journal.droppedRecords == 0, "the second run did not load the two recorded pipelines and their three modules"); + Require(replayed.journal.replayedGraphics == 1 && replayed.journal.replayedCompute == 1 && replayed.journal.replayRejected == 0 && replayed.journal.replayFailed == 0 && replayed.journal.replayPresent == 0, "the replay did not build one graphics and one compute pipeline"); + Require(device->ComputePipelinesCreated() == 1, "the replay did not build the compute pipeline on the new device"); + std::memset(compute.data, Fill, compute.bytes); + const auto words = AddressWords(compute); + const auto result = CompileCompute(*device, computeCode, words, 32); + device->Dispatch(result, 1, 1, 1, {}, reinterpret_cast(computeCode.data())); + device->WaitIdle(); + ExpectConstantStore(compute, "second run: the constant store dispatch"); + Require(device->ComputePipelinesCreated() == 1, "the second run's dispatch built a compute pipeline the replay had built"); + std::memset(target.data, 0, target.bytes); + Draw(*device, DecodeTarget(target), Program(red)); + ExpectTexel(*device, target, redTexel, "second run: the draw through the replayed pipeline"); + const auto drawn = Snapshot(); + Require(drawn.store.hits == replayed.store.hits + 1 && drawn.store.misses == replayed.store.misses, "the second run's draw did not find the replayed pipeline"); + AgcDriver::Graphics::FlushPipelineJournal(); + const auto final = Snapshot(); + Require(final.journal.recordedGraphics == 1 && final.journal.recordedCompute == 1 && final.journal.recordedModules == 3, "the second run recorded pipelines the journal already held"); + } +} + +// --benchmark [count]: `count` distinct compute programs, dispatched for the first time on three +// devices in turn. The first device starts without the driver's pipeline cache file (a driver with +// an on-disk shader cache of its own, lavapipe's Mesa cache here, still serves from it unless that +// is disabled too, MESA_SHADER_CACHE_DISABLE=true, where the driver's compile dominates); it leaves +// the driver's cache and the journal warm. The second replays the journal before its dispatches. +// The third dispatches with the driver's cache warm but the journal file removed: what a warm run +// pays at each first use without the journal. Wall times only; no pass threshold. +void Benchmark(std::uint32_t count) { + using namespace NarrowConstantStoreFixture; + Require(AgcDriver::Graphics::PipelineJournalEnabled(), "the pipeline journal is disabled"); + Block compute(BlockBytes); + const auto words = AddressWords(compute); + std::vector> codes; + for (std::uint32_t salt = 1; salt <= count; ++salt) codes.push_back(ComputeCode(salt)); + struct Run { + double openMs = 0; + double dispatchMs = 0; + std::uint64_t created = 0; + AgcDriver::Graphics::PipelineJournalCounters journal; + }; + const auto run = [&](const char* what) { + Run measured; + const auto openedAt = std::chrono::steady_clock::now(); + const auto device = OpenVulkanTestDevice(); + Require(device != nullptr, std::string("no device for the benchmark's ") + what); + device->WaitForPipelineJournalReplay(); + measured.openMs = std::chrono::duration(std::chrono::steady_clock::now() - openedAt).count(); + measured.journal = AgcDriver::Graphics::PipelineJournalTotals(); + std::vector results; + for (const auto& code : codes) results.push_back(CompileCompute(*device, code, words, 32)); + const auto createdBefore = device->ComputePipelinesCreated(); + const auto started = std::chrono::steady_clock::now(); + for (std::size_t index = 0; index < results.size(); ++index) device->Dispatch(results[index], 1, 1, 1, {}, reinterpret_cast(codes[index].data())); + device->WaitIdle(); + measured.dispatchMs = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + measured.created = device->ComputePipelinesCreated() - createdBefore; + AgcDriver::Graphics::FlushPipelineJournal(); + return measured; + }; + const auto cold = run("cold run"); + const auto replayed = run("replayed run"); + std::error_code error; + std::filesystem::remove(AgcDriver::Graphics::PipelineJournalPath(), error); + Require(!error, "cannot remove the journal for the benchmark's last run"); + const auto warm = run("warm run without the journal"); + std::printf("pipeline journal benchmark: %u compute programs, %u replay threads\n", count, replayed.journal.replayThreads); + std::printf(" cold driver cache, no journal: %llu pipelines built at their dispatches, %u dispatches in %.1f ms (%.2f ms each)\n", static_cast(cold.created), count, cold.dispatchMs, cold.dispatchMs / count); + std::printf(" warm driver cache, no journal: %llu pipelines built at their dispatches, %u dispatches in %.1f ms (%.2f ms each)\n", static_cast(warm.created), count, warm.dispatchMs, warm.dispatchMs / count); + std::printf(" warm driver cache, journal: %llu pipelines built by the replay in %.1f ms of replay work (%.1f ms from the device opening), then %llu built at their dispatches, %u dispatches in %.1f ms (%.2f ms each)\n", static_cast(replayed.journal.replayedCompute), replayed.journal.replayMs, replayed.openMs, static_cast(replayed.created), count, replayed.dispatchMs, replayed.dispatchMs / count); +} + +} + +int main(int argc, char** argv) { + try { + const auto directory = std::filesystem::temp_directory_path() / ("aps5-pipeline-journal-test-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(directory); + SetEnvironment("ANYPS5_NO_SHADER_CACHE", "0"); + SetEnvironment("ANYPS5_SHADER_CACHE_DIR", directory.string()); + SetEnvironment("APS5_NO_PIPELINE_JOURNAL", "0"); + SetEnvironment("APS5_VERIFY_SET_LAYOUTS", "1"); + SetEnvironment("APS5_PIPELINE_JOURNAL_THREADS", "2"); + struct Cleanup { + std::filesystem::path directory; + ~Cleanup() { + std::error_code error; + std::filesystem::remove_all(directory, error); + } + } cleanup{directory}; + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + if (argc >= 2 && std::string(argv[1]) == "--benchmark") { + Benchmark(argc >= 3 ? static_cast(std::strtoul(argv[2], nullptr, 10)) : 24u); + return 0; + } + JournalTests(); + std::puts("pipeline journal record and replay passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDispatch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDispatch.cpp new file mode 100644 index 0000000000..8cc143e3c5 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDispatch.cpp @@ -0,0 +1,268 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/Shutdown.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::DriverDetail::DrawPipeline; + +constexpr std::uint32_t Lanes = 32; +constexpr std::uint32_t BufferBytes = Lanes * 4u; +constexpr std::uint32_t DispatchOpcode = 0x15; + +constexpr std::size_t BlockBytes = 65536; + +std::uint32_t* NewBlock() { +#ifdef _WIN32 + auto* block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + auto* block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); +#endif + if (block == nullptr) throw std::runtime_error("pipelined dispatch: cannot allocate a guest block"); + std::memset(block, 0, BlockBytes); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, false); + return block; +} + +std::uint32_t* Input = nullptr; +std::uint32_t* Output = nullptr; +std::uint32_t* Scratch = nullptr; +std::uint32_t* Table = nullptr; +std::uint32_t* TargetA = nullptr; +std::uint32_t* TargetB = nullptr; + +alignas(256) constexpr std::array Code{0x34020082u, 0xe0301000u, 0x80000201u, 0xbf8c3f70u, 0x4a060481u, 0xe0701000u, 0x80010301u, 0xbf810000u}; + +alignas(256) constexpr std::array ScalarCode{0xf4000200u, 0xfa000000u, 0xbf8cc07fu, 0x34020082u, 0x7e040208u, 0x4a040481u, 0xe0701000u, 0x80010201u, 0xbf810000u}; + +alignas(256) constexpr std::array TableCode{0xf4080200u, 0xfa000000u, 0xbf8cc07fu, 0x34020082u, 0xe0301000u, 0x80020201u, 0xbf8c3f70u, 0x4a060481u, 0xe0701000u, 0x80010301u, 0xbf810000u}; + +void check(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error("pipelined dispatch: " + what); +} + +template +void WaitUntil(Ready ready, const std::string& what) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!ready()) { + if (std::chrono::steady_clock::now() >= deadline) throw std::runtime_error("pipelined dispatch: timed out waiting for " + what); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} + +std::array FillPacket(std::uint32_t value, const void* destination, std::uint32_t bytes) { + const auto address = reinterpret_cast(destination); + return {0xc0055000u, 2u << 29u, value, 0u, static_cast(address), static_cast(address >> 32u), bytes}; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::vector Round(const void* destination, std::uint32_t value, const void* code) { + const auto fill = FillPacket(value, destination, BufferBytes); + std::vector words(fill.begin(), fill.end()); + const auto codeAddress = reinterpret_cast(code); + const auto input = BufferDescriptor(Input, BufferBytes); + const auto output = BufferDescriptor(Output, BufferBytes); + const std::vector commands{ + 0xc0027600u, 0x20cu, static_cast(codeAddress >> 8u), static_cast(codeAddress >> 40u), + 0xc0037600u, 0x207u, Lanes, 1u, 1u, + 0xc0017600u, 0x213u, 8u << 1u, + 0xc0087600u, 0x240u, input[0], input[1], input[2], input[3], output[0], output[1], output[2], output[3], + 0xc0031500u, 1u, 1u, 1u, 0x8041u, + }; + words.insert(words.end(), commands.begin(), commands.end()); + return words; +} + +std::vector TableFills(const void* target, std::uint32_t value) { + const auto targetFill = FillPacket(value, target, BufferBytes); + const auto tableFill = FillPacket(static_cast(reinterpret_cast(target)), Table, 4u); + std::vector words(targetFill.begin(), targetFill.end()); + words.insert(words.end(), tableFill.begin(), tableFill.end()); + return words; +} + +std::vector TableDispatch(const void* code) { + const auto codeAddress = reinterpret_cast(code); + const auto tableAddress = reinterpret_cast(Table); + const auto output = BufferDescriptor(Output, BufferBytes); + const std::vector commands{ + 0xc0027600u, 0x20cu, static_cast(codeAddress >> 8u), static_cast(codeAddress >> 40u), + 0xc0037600u, 0x207u, Lanes, 1u, 1u, + 0xc0017600u, 0x213u, 8u << 1u, + 0xc0087600u, 0x240u, static_cast(tableAddress), static_cast(tableAddress >> 32u), 0u, 0u, output[0], output[1], output[2], output[3], + 0xc0031500u, 1u, 1u, 1u, 0x8041u, + }; + return commands; +} + +void Submit(std::vector& words) { + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&packet) == 0, "submission failed"); +} + +} + +int main() { +#ifdef _WIN32 + _putenv_s("APS5_PIPELINED_DRAWS", "1"); + _putenv_s("APS5_PIPELINE_DMA", "1"); +#else + setenv("APS5_PIPELINED_DRAWS", "1", 1); + setenv("APS5_PIPELINE_DMA", "1", 1); +#endif + try { + { + const auto probe = OpenVulkanTestDevice(); + if (!probe) return VulkanTestSkipped; + } + const char* dispatchSwitch = std::getenv("APS5_PIPELINE_DISPATCH"); + const bool pipelinedDispatch = dispatchSwitch != nullptr && dispatchSwitch[0] != '\0' && dispatchSwitch[0] != '0'; + + Shader shader{}; + shader.file_header = 0x34333231; + shader.version = 0x18; + shader.header_size = sizeof(Shader); + shader.shader_size = sizeof(Code); + shader.code = Code.data(); + AgcDriverRegisterShader_nid_postfix(&shader); + Shader scalarShader{}; + scalarShader.file_header = 0x34333231; + scalarShader.version = 0x18; + scalarShader.header_size = sizeof(Shader); + scalarShader.shader_size = sizeof(ScalarCode); + scalarShader.code = ScalarCode.data(); + AgcDriverRegisterShader_nid_postfix(&scalarShader); + Shader tableShader{}; + tableShader.file_header = 0x34333231; + tableShader.version = 0x18; + tableShader.header_size = sizeof(Shader); + tableShader.shader_size = sizeof(TableCode); + tableShader.code = TableCode.data(); + AgcDriverRegisterShader_nid_postfix(&tableShader); + + Input = NewBlock(); + Output = NewBlock(); + Scratch = NewBlock(); + Table = NewBlock(); + TargetA = NewBlock(); + TargetB = NewBlock(); + check((reinterpret_cast(TargetA) >> 32u) == (reinterpret_cast(TargetB) >> 32u), "the two targets do not share their high address dword"); + const auto initialDescriptor = BufferDescriptor(TargetA, BufferBytes); + std::copy(initialDescriptor.begin(), initialDescriptor.end(), Table); + const std::array values{0x0a0b0c0du, 0x11112222u, 0x33334444u, 0x55556666u, 0x77778888u}; + for (std::size_t round = 0; round < values.size(); ++round) { + const auto value = values[round]; + const auto drainsBefore = DrawPipeline::DrainRequestsByOpcode(DispatchOpcode); + auto words = Round(Input, value, Code.data()); + Submit(words); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output), BufferBytes); + const auto drains = DrawPipeline::DrainRequestsByOpcode(DispatchOpcode) - drainsBefore; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + check(Input[lane] == value, "the immediate fill did not land in lane " + std::to_string(lane)); + check(Output[lane] == value + 1u, "round " + std::to_string(round) + ": lane " + std::to_string(lane) + " of the output is " + std::to_string(Output[lane]) + ", expected " + std::to_string(value + 1u) + " (the dispatch did not see the fill before it)"); + } + std::printf("pipelined dispatch: input fill round %zu, fill 0x%08x, dispatch drains %llu\n", round, value, static_cast(drains)); + if (!pipelinedDispatch) check(drains >= 1, "a dispatch did not drain the pipeline, and APS5_PIPELINE_DISPATCH is off"); + if (pipelinedDispatch) check(drains <= 1, "a dispatch drained the pipeline more than once (" + std::to_string(drains) + ")"); + } + for (std::size_t round = 0; round < 3; ++round) { + const auto value = 0xc0de0000u + static_cast(round); + const auto drainsBefore = DrawPipeline::DrainRequestsByOpcode(DispatchOpcode); + auto words = Round(Scratch, value, Code.data()); + Submit(words); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Scratch), BufferBytes); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output), BufferBytes); + const auto drains = DrawPipeline::DrainRequestsByOpcode(DispatchOpcode) - drainsBefore; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + check(Scratch[lane] == value, "the scratch fill did not land in lane " + std::to_string(lane)); + check(Output[lane] == Input[lane] + 1u, "scratch round " + std::to_string(round) + ": lane " + std::to_string(lane) + " of the output is " + std::to_string(Output[lane]) + ", expected " + std::to_string(Input[lane] + 1u)); + } + std::printf("pipelined dispatch: scratch fill round %zu, fill 0x%08x, dispatch drains %llu\n", round, value, static_cast(drains)); + if (!pipelinedDispatch) check(drains >= 1, "a dispatch did not drain the pipeline, and APS5_PIPELINE_DISPATCH is off"); + if (pipelinedDispatch) check(drains == 0, "a dispatch that reads nothing pending drained the pipeline (" + std::to_string(drains) + ")"); + } + const std::array tableValues{0x6a6a0001u, 0x6b6b0002u, 0x6c6c0003u, 0x6d6d0004u}; + for (std::size_t round = 0; round < tableValues.size(); ++round) { + auto* target = round % 2 == 0 ? TargetA : TargetB; + const auto value = tableValues[round]; + std::atomic released{false}; + struct Release { + std::atomic& flag; + ~Release() { flag.store(true, std::memory_order_release); } + } release{released}; + WaitUntil([] { return DrawPipeline::QueuedItems() == 0; }, "the pipeline to empty before descriptor round " + std::to_string(round)); + const auto drainsBefore = DrawPipeline::DrainRequestsByOpcode(DispatchOpcode); + DrawPipeline::Queue0().Enqueue([&released] { + while (!released.load(std::memory_order_acquire)) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + }, {}); + auto fills = TableFills(target, value); + Submit(fills); + WaitUntil([] { return DrawPipeline::QueuedItems() >= 4; }, "the target and table fills queued behind the gate"); + auto dispatch = TableDispatch(TableCode.data()); + Submit(dispatch); + WaitUntil([&drainsBefore] { return DrawPipeline::QueuedItems() >= 5 || DrawPipeline::DrainRequestsByOpcode(DispatchOpcode) > drainsBefore; }, "the dispatch handled while the table fill is pending"); + released.store(true, std::memory_order_release); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output), BufferBytes); + const auto drains = DrawPipeline::DrainRequestsByOpcode(DispatchOpcode) - drainsBefore; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + check(Output[lane] == value + 1u, "descriptor round " + std::to_string(round) + ": lane " + std::to_string(lane) + " of the output is " + std::to_string(Output[lane]) + ", expected " + std::to_string(value + 1u) + " (the dispatch read the buffer the table named before the pending fill of the table)"); + } + std::printf("pipelined dispatch: descriptor round %zu, fill 0x%08x, dispatch drains %llu\n", round, value, static_cast(drains)); + if (!pipelinedDispatch) check(drains >= 1, "a dispatch did not drain the pipeline, and APS5_PIPELINE_DISPATCH is off"); + if (pipelinedDispatch) check(drains == 1, "a dispatch whose table entry is pending a write drained " + std::to_string(drains) + " times, expected once"); + } + const std::array scalarValues{0x13572468u, 0x2468ace0u, 0x0f1e2d3cu, 0x9abcdef0u, 0x31415926u}; + for (std::size_t round = 0; round < scalarValues.size(); ++round) { + const auto value = scalarValues[round]; + const auto drainsBefore = DrawPipeline::DrainRequestsByOpcode(DispatchOpcode); + auto words = Round(Input, value, ScalarCode.data()); + Submit(words); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output), BufferBytes); + const auto drains = DrawPipeline::DrainRequestsByOpcode(DispatchOpcode) - drainsBefore; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + check(Output[lane] == value + 1u, "scalar round " + std::to_string(round) + ": lane " + std::to_string(lane) + " of the output is " + std::to_string(Output[lane]) + ", expected " + std::to_string(value + 1u) + " (a stale input)"); + } + std::printf("pipelined dispatch: scalar read round %zu, fill 0x%08x, dispatch drains %llu\n", round, value, static_cast(drains)); + if (!pipelinedDispatch) check(drains >= 1, "a dispatch did not drain the pipeline, and APS5_PIPELINE_DISPATCH is off"); + if (pipelinedDispatch) check(drains <= 1, "a dispatch drained the pipeline more than once (" + std::to_string(drains) + ")"); + } + LibcRunShutdown_nid_postfix(); + std::puts("pipelined dispatch tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { LibcRunShutdown_nid_postfix(); } catch (...) {} + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp index 777dd733f4..e242f0a0b9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -4,7 +4,10 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp" +#include "../NarrowConstantStoreFixture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" #include "prx/libSceAgcDriver/Submit/include/Acb.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include @@ -27,6 +30,12 @@ #include #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif namespace { @@ -600,7 +609,7 @@ void RegisteredVertexWorkgroup(const ShaderRecompiler::SpirvTarget& target, bool header.specials.vgt_shader_stages_en.value = 0; ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "missing AGC user-data header"); header.shader.user_data = &header.userData; - if (target.subgroupSize < 64u && std::getenv("APS5_SYNC_SHADER_PREPARE") != nullptr) { + if (target.subgroupSize < 64u && std::getenv("APS5_SYNC_SHADER_PREPARE") != nullptr && std::getenv("APS5_LAZY_SHADER_PREPARE") == nullptr) { ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "v_readlane_b32 of lane 63 is outside"); } std::cout << "registered NGG preparation deferral passed; subgroup=" << target.subgroupSize << '\n'; @@ -612,6 +621,102 @@ void RegisteredVertexWorkgroup(const ShaderRecompiler::SpirvTarget& target, bool std::cout << "registered NGG wave64 preparation passed; subgroup=" << target.subgroupSize << '\n'; } +void LazyRegistration() { + using namespace NarrowConstantStoreFixture; + Require(std::getenv("APS5_LAZY_SHADER_PREPARE") != nullptr, "lazy preparation is not enabled"); + struct Header { + Shader shader{}; + std::array registers{}; + ShaderSpecialRegs specials{}; + } header; + const auto address = reinterpret_cast(Code.data()); + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18; + header.shader.header_size = sizeof(header); + header.shader.shader_size = sizeof(Code); + header.shader.code = Code.data(); + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = header.registers.size(); + header.shader.specials = &header.specials; + header.specials.dispatch_modifier = 0x8000; + header.registers = {{{0x20c, static_cast(address >> 8u)}, {0x20d, static_cast(address >> 40u)}, {0x207, Threads}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 4}}}; + AgcDriverRegisterShader_nid_postfix(&header.shader); + ++header.registers[0].value; + ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "entry point is outside shader code"); + --header.registers[0].value; + header.registers[2].value = 0; + ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "must be nonzero"); + header.registers[2].value = Threads; + header.shader.num_sh_registers = 255; + ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "truncated shader metadata"); + header.shader.num_sh_registers = header.registers.size(); + const auto registered = [&] { + const auto registry = AgcDriver::DriverDetail::Driver::Get().RegisteredShaders(); + Require(registry != nullptr, "lazy registration published no shader registry"); + auto snapshot = AgcDriver::DriverDetail::RegisteredHeader(*registry, &header.shader); + Require(snapshot != nullptr, "lazy registration published no snapshot"); + return snapshot; + }; + const auto lazySnapshot = registered(); + { + AgcDriver::DriverDetail::ShaderPreparationTransaction transaction; + const auto& state = transaction.Read(*lazySnapshot); + Require(state.deferred && state.entries.empty(), "lazy registration prepared the shader before its first dispatch"); + } + constexpr std::size_t outputBytes = 65536; +#ifdef _WIN32 + auto* allocation = static_cast(VirtualAlloc(nullptr, outputBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + auto* allocation = static_cast(std::aligned_alloc(outputBytes, outputBytes)); +#endif + Require(allocation != nullptr, "cannot allocate lazy shader test output"); + const auto release = [](std::uint8_t* block) { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + }; + const std::unique_ptr ownedOutput(allocation, release); + const std::span output(allocation, outputBytes); + GuestAllocations::Mutation().Add(output.data(), output.size(), true, true, true); + const auto outputAddress = reinterpret_cast(output.data()); + std::vector commands; + for (const auto reg : header.registers) commands.insert(commands.end(), {0xc0017600u, reg.offset, reg.value}); + commands.insert(commands.end(), {0xc0027600u, 0x240, static_cast(outputAddress), static_cast(outputAddress >> 32u), 0xc0031500u, 1, 1, 1, 0x8041}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + for (unsigned iteration = 0; iteration < 2; ++iteration) { + std::ranges::fill(output, Fill); + sceAgcDriverSubmitAcb(0x20, &packet); + AgcDriverWaitIdle_nid_postfix(); + AgcDriverSuspendPoint_nid_postfix(); + AgcDriverWaitIdle_nid_postfix(); + for (std::size_t offset = 0; offset < output.size(); ++offset) { + const auto expected = offset < Threads * LaneBytes ? ExpectedLane[offset % LaneBytes] : Fill; + if (output[offset] != expected) throw std::runtime_error("lazy dispatch " + std::to_string(iteration) + " byte " + std::to_string(offset) + " is " + std::to_string(output[offset]) + ", expected " + std::to_string(expected)); + } + } + { + AgcDriver::DriverDetail::ShaderPreparationTransaction transaction; + Require(!transaction.Read(*lazySnapshot).entries.empty(), "the first dispatch did not prepare the lazily registered shader"); + } + AgcDriverResolveShaderAbi_nid_postfix(&header.shader, {}, {}); + alignas(256) static const std::array invalid{0xbf9f0000u}; + const auto invalidAddress = reinterpret_cast(invalid.data()); + header.shader.code = invalid.data(); + header.shader.shader_size = sizeof(invalid); + header.registers[0].value = static_cast(invalidAddress >> 8u); + header.registers[1].value = static_cast(invalidAddress >> 40u); + AgcDriverRegisterShader_nid_postfix(&header.shader); + ExpectFailure([&] { AgcDriverResolveShaderAbi_nid_postfix(&header.shader, {}, {}); }, "s_code_end"); + commands[2] = header.registers[0].value; + commands[5] = header.registers[1].value; + sceAgcDriverSubmitAcb(0x20, &packet); + ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "s_code_end"); + ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "s_code_end"); +} + void DeferredRegistration() { alignas(256) std::array code{0xbe842104u, 0xbf810000u}; struct Header { @@ -672,7 +777,13 @@ void DeferredUndecodableRegistration() { int main(int argc, char** argv) { try { - Require(argc == 1 || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration" || std::string_view(argv[1]) == "--deferred" || std::string_view(argv[1]) == "--deferred-undecodable" || std::string_view(argv[1]) == "--registered-workgroup")), "invalid test arguments"); + Require(argc == 1 || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration" || std::string_view(argv[1]) == "--deferred" || std::string_view(argv[1]) == "--deferred-undecodable" || std::string_view(argv[1]) == "--registered-workgroup" || std::string_view(argv[1]) == "--lazy")), "invalid test arguments"); + if (argc == 2 && std::string_view(argv[1]) == "--lazy") { + if (!OpenVulkanTestDevice()) return VulkanTestSkipped; + LazyRegistration(); + std::cout << "lazy shader preparation tests passed\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--registered-workgroup") { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SampleMultisampleTarget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SampleMultisampleTarget.cpp index f3d7e2bf44..16e51a1967 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SampleMultisampleTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SampleMultisampleTarget.cpp @@ -54,6 +54,8 @@ constexpr std::array Drawn{0.25f, 0.5f, 0.125f, 1.0f}; constexpr double EdgeMargin = 1.0 / 32.0; constexpr std::uint32_t FillOnes = 0x3c003c00u; constexpr std::array Ones{1.0f, 1.0f, 1.0f, 1.0f}; +constexpr std::array Written{0.75f, 0.25f, 0.5f, 1.0f}; +constexpr std::uint32_t StoredSample = 1; alignas(256) constexpr std::array VertexCode{ 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, @@ -70,6 +72,18 @@ alignas(256) constexpr std::array LoadCode{ 0x80001301, 0xbf810000, }; +alignas(256) constexpr std::array StoreCode{ + 0x34020087, 0xe030100c, 0x80000501, 0xe0301010, 0x80000601, 0xe0301014, 0x80000701, 0xe0301018, + 0x80001001, 0xe030101c, 0x80001101, 0xe0301020, 0x80001201, 0xe0301024, 0x80001301, 0xbf8c3f70, + 0xf0201f30, 0x00011005, 0xbf810000, +}; + +struct SampleWrite { + std::uint32_t x; + std::uint32_t y; + std::uint32_t sample; +}; + constexpr std::array, 3> Triangle{{ {-1.0f, -1.0f, 0.5f, 1.0f}, {0.83f, -1.0f, 0.5f, 1.0f}, {-1.0f, 0.71f, 0.5f, 1.0f} }}; @@ -199,13 +213,12 @@ std::array TargetDescriptor(const Block& block, std::uint32_t }; } -ShaderRecompiler::RecompileResult CompileLoad(AgcDriver::VulkanDevice& device, const std::array& texture, std::vector& userData) { +ShaderRecompiler::RecompileResult CompileProgram(AgcDriver::VulkanDevice& device, std::span code, const std::array& texture, std::vector& userData) { const auto address = reinterpret_cast(Buffer.data()); const std::array buffer{static_cast(address), static_cast((address >> 32u) & 0xffffu), static_cast(Buffer.size() * 4u), 0x31016facu}; userData.assign(16, 0u); std::copy(buffer.begin(), buffer.end(), userData.begin()); std::copy(texture.begin(), texture.end(), userData.begin() + 4); - const std::span code(LoadCode); const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ @@ -218,6 +231,63 @@ ShaderRecompiler::RecompileResult CompileLoad(AgcDriver::VulkanDevice& device, c return ShaderRecompiler::Recompile(request); } +ShaderRecompiler::RecompileResult CompileLoad(AgcDriver::VulkanDevice& device, const std::array& texture, std::vector& userData) { + return CompileProgram(device, LoadCode, texture, userData); +} + +void StoreSamples(AgcDriver::VulkanDevice& device, const std::array& texture, const std::vector& writes) { + std::vector userData; + const auto program = CompileProgram(device, StoreCode, texture, userData); + for (std::size_t first = 0; first < writes.size(); first += Threads) { + Buffer.fill(0u); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto& write = writes[std::min(first + tid, writes.size() - 1)]; + const std::array words{0u, 0u, 0u, write.x, write.y, write.sample, std::bit_cast(Written[0]), std::bit_cast(Written[1]), std::bit_cast(Written[2]), std::bit_cast(Written[3])}; + std::copy(words.begin(), words.end(), &Buffer[tid * Words]); + } + device.Dispatch(program, 1, 1, 1, {}, reinterpret_cast(StoreCode.data())); + device.WaitIdle(); + } +} + +std::vector SampleWrites(bool (*selected)(std::uint32_t x, std::uint32_t y, std::uint32_t sample)) { + std::vector writes; + for (std::uint32_t y = 0; y < Side; ++y) { + for (std::uint32_t x = 0; x < Side; ++x) { + for (std::uint32_t sample = 0; sample < Samples; ++sample) { + if (selected(x, y, sample)) writes.push_back({x, y, sample}); + } + } + } + return writes; +} + +void CheckWritten(const std::vector>& before, const std::vector>& after, bool (*selected)(std::uint32_t x, std::uint32_t y, std::uint32_t sample), const std::string& what) { + std::uint32_t written = 0; + for (std::uint32_t index = 0; index < after.size(); ++index) { + const auto pixel = index / Samples; + const auto x = pixel % Side; + const auto y = pixel / Side; + const auto sample = index % Samples; + const bool chosen = selected(x, y, sample); + char where[96]; + std::snprintf(where, sizeof(where), "sample %u of pixel (%u, %u)", sample, x, y); + Require(after[index] == (chosen ? Written : before[index]), std::string(where) + (chosen ? " does not hold the stored color " : " changed although it was not stored to ") + what); + written += chosen ? 1u : 0u; + } + Require(written != 0 && written != after.size(), "the stores covered every sample or none " + what); +} + +void RequireStoreRefused(AgcDriver::VulkanDevice& device, const std::array& texture, const std::string& expected, const std::string& what) { + std::string refusal; + try { + StoreSamples(device, texture, {{0u, 0u, 0u}}); + } catch (const std::exception& error) { + refusal = error.what(); + } + Require(refusal.find(expected) != std::string::npos, what + " was not refused: " + refusal); +} + std::vector> LoadSamples(AgcDriver::VulkanDevice& device, const Block& block) { std::vector userData; const auto program = CompileLoad(device, TargetDescriptor(block, SampleShift), userData); @@ -341,6 +411,19 @@ int main() { const std::uint32_t odd = 0x3c000000u; std::memcpy(mixed.data + colorBytes - sizeof(odd), &odd, sizeof(odd)); RequireRefused(*device, TargetDescriptor(mixed, SampleShift), "does not hold one repeated 32-bit value", "loading a multisampled texture no draw rendered to whose color memory is not uniform"); + const auto oneSample = [](std::uint32_t, std::uint32_t, std::uint32_t sample) { return sample == StoredSample; }; + const auto drawnBefore = LoadSamples(*device, block); + StoreSamples(*device, TargetDescriptor(block, SampleShift), SampleWrites(oneSample)); + CheckWritten(drawnBefore, LoadSamples(*device, block), oneSample, "into one sample of each pixel of the drawn target"); + const auto leftHalf = [](std::uint32_t x, std::uint32_t, std::uint32_t sample) { return x < Side / 2u && sample != 0u; }; + Block undrawn(BlockBytes); + StoreSamples(*device, TargetDescriptor(undrawn, SampleShift), SampleWrites(leftHalf)); + CheckWritten(std::vector>(Side * Side * Samples), LoadSamples(*device, undrawn), leftHalf, "into zeroed color memory no draw rendered to"); + Block mixedStore(BlockBytes); + StoreWords(mixedStore, colorBytes, FillOnes); + std::memcpy(mixedStore.data + colorBytes - sizeof(odd), &odd, sizeof(odd)); + RequireStoreRefused(*device, TargetDescriptor(mixedStore, SampleShift), "writing the 16x16 multisampled texture", "storing to a multisampled texture no draw rendered to whose color memory is not uniform"); + std::puts("stores to one sample of each pixel of a drawn 4x target and to samples of zeroed color memory no draw rendered to reach exactly those samples, and a store to non-uniform color memory no draw rendered to is refused"); std::puts("every sample of a 4x color target loaded through its multisampled texture descriptor matches the triangle drawn into it, then the fill of its color memory; a surface no draw rendered to loads the value its color memory repeats and keeps it under a later draw"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SparseDmaBufImport.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SparseDmaBufImport.cpp index 6c271a9f48..680b8fcfa8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SparseDmaBufImport.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SparseDmaBufImport.cpp @@ -24,11 +24,20 @@ int main() { return VulkanTestSkipped; } #else +namespace AgcDriver { + +struct VulkanDeviceTestAccess { + static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); } +}; + +} + namespace { using AgcDriver::Graphics::Require; constexpr std::uint64_t ChunkBytes = std::uint64_t{64} << 20u; +constexpr std::uint64_t LargestTestedLimit = std::uint64_t{1} << 30u; std::uint8_t* mapping = nullptr; std::uint64_t mappingBytes = 0; @@ -56,6 +65,12 @@ bool Imported(VkDevice device, std::uint64_t address, std::size_t bytes) { return AgcDriver::Graphics::HostImportCovers(probe, address, bytes); } +bool DmaBufImport(const AgcDriver::VulkanDevice& device, std::uint64_t address, std::size_t bytes) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + const auto* import = AgcDriver::Graphics::HostImportFor(AgcDriver::VulkanDeviceTestAccess::ContextOf(device), address, bytes); + return import != nullptr && import->dmaBuf; +} + } int main() { @@ -66,6 +81,10 @@ int main() { return VulkanTestSkipped; } close(udmabuf); + if (UdmabufLimitBytes() > LargestTestedLimit) { + std::printf("skipped, udmabuf size_limit_mb is above %llu: a range above it is too large to map\n", static_cast(LargestTestedLimit >> 20u)); + return VulkanTestSkipped; + } auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; @@ -107,21 +126,30 @@ int main() { auto* small = mapping + bytes; constexpr std::uint64_t SmallBytes = std::uint64_t{1} << 20u; + const bool settles = AgcDriver::VulkanDeviceTestAccess::ContextOf(*device).settleQueue != VK_NULL_HANDLE; + const auto settledBefore = AgcDriver::Graphics::HostImportsSettled(); device->ImportGuestMemory(Register(small, SmallBytes), GuestAllocations::GuestAllocationsGeneration_nid_postfix(), false); + const auto settledAtMap = AgcDriver::Graphics::HostImportsSettled(); const bool smallImported = Imported(device->Device(), reinterpret_cast(small), static_cast(SmallBytes)); + const bool settlesShared = settles && smallImported && DmaBufImport(*device, reinterpret_cast(small), static_cast(SmallBytes)); GuestAllocations::Mutation().Remove(small); if (!smallImported) { std::printf("skipped, the device does not import dma-buf memory\n"); return VulkanTestSkipped; } + if (!settles) std::printf("note: the device has no settle queue, so dma-buf imports are not settled\n"); + else if (!settlesShared) std::printf("note: the device imports shared memory through host pointers, which are not settled\n"); + Require(!settlesShared || settledAtMap == settledBefore + 1, "sparse import: the 1 MiB dma-buf import made at map time was not settled on the settle queue"); const auto boundary = ChunkBytes; std::memset(mapping + boundary - 65536, 0x11, 131072); const auto ranges = Register(mapping, bytes); const auto address = reinterpret_cast(mapping); + const auto settledSmall = AgcDriver::Graphics::HostImportsSettled(); const bool atMap = device->ImportGuestMemory(ranges, GuestAllocations::GuestAllocationsGeneration_nid_postfix(), false); Require(Imported(device->Device(), address, static_cast(bytes)), "sparse import: a range larger than udmabuf's size limit was not imported"); Require(atMap, "sparse import: the import made without the GPU lock was refused"); + Require(!settlesShared || AgcDriver::Graphics::HostImportsSettled() == settledSmall + 1, "sparse import: the sparse dma-buf import made at map time was not settled on the settle queue"); bool filled = false; { diff --git a/core/libs/prx/libSceConvertKeycode/Export.cpp b/core/libs/prx/libSceConvertKeycode/Export.cpp index 0fa5a89b58..9b9e92908f 100644 --- a/core/libs/prx/libSceConvertKeycode/Export.cpp +++ b/core/libs/prx/libSceConvertKeycode/Export.cpp @@ -18,6 +18,7 @@ char16_t UsCharacter(std::uint16_t keycode) { if (keycode == 0x27) return u'0'; if (keycode == 0x2C) return u' '; if (keycode >= 0x2D && keycode <= 0x38) return kPunctuation[keycode - 0x2D]; + if (keycode == 0x64) return u'\\'; if (keycode >= 0x54 && keycode <= 0x63) return kKeypad[keycode - 0x54]; return 0; } diff --git a/core/libs/prx/libSceNet/CMakeLists.txt b/core/libs/prx/libSceNet/CMakeLists.txt index 98776d4b5e..4a1c9fdb5e 100644 --- a/core/libs/prx/libSceNet/CMakeLists.txt +++ b/core/libs/prx/libSceNet/CMakeLists.txt @@ -15,6 +15,16 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endforeach() + add_test_executable(guest_refused_connect_tests ${LIBS_INCLUDE_DIR}/tests/GuestRefusedConnect.cpp) + target_include_directories(guest_refused_connect_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_refused_connect_tests PRIVATE libSceNet libkernel libc) + configure_windows_unwind(guest_refused_connect_tests) + add_test(NAME guest_refused_connect COMMAND guest_refused_connect_tests) + set_tests_properties(guest_refused_connect PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_refused_connect PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() if(WIN32) add_test_executable(guest_socket_host_write_tests ${LIBS_INCLUDE_DIR}/tests/GuestSocketHostWrite.cpp) target_include_directories(guest_socket_host_write_tests PRIVATE ${LIBS_INCLUDE_DIR}) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index ac697a4120..284a9076f9 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -859,6 +859,10 @@ static int disconnect_datagram(const Sock& socket) { return 0; } +#ifdef _WIN32 +extern "C" void RefuseClosedLoopbackAtOnce_nid_no_patch(SOCKET socket, const sockaddr* address); +#endif + int APS5_VABI sceNetConnect(int s, const void* addr, uint32_t addrlen) { Sock socket; { @@ -875,6 +879,9 @@ int APS5_VABI sceNetConnect(int s, const void* addr, uint32_t addrlen) { sockaddr_storage native{}; NativeLength native_length = 0; if (!guest_to_native_address(addr, addrlen, native, native_length)) return fail(NET_EINVAL); +#ifdef _WIN32 + if (socket.type == NET_SOCK_STREAM) RefuseClosedLoopbackAtOnce_nid_no_patch(socket.native->value, reinterpret_cast(&native)); +#endif if (native.ss_family != (socket.family == NET_AF_INET ? AF_INET : AF_INET6)) return fail(NET_EAFNOSUPPORT); if (::connect(socket.native->value, reinterpret_cast(&native), native_length) == 0) return 0; #ifdef _WIN32 diff --git a/core/libs/prx/libSceSystemService/CMakeLists.txt b/core/libs/prx/libSceSystemService/CMakeLists.txt index ca57051164..68b75598dd 100644 --- a/core/libs/prx/libSceSystemService/CMakeLists.txt +++ b/core/libs/prx/libSceSystemService/CMakeLists.txt @@ -50,6 +50,13 @@ if(BUILD_TESTING) PASS_REGULAR_EXPRESSION "Guest relaunch completed" TIMEOUT 30) set_tests_properties(guest_nested_exit PROPERTIES PASS_REGULAR_EXPRESSION "Guest nested exit completed" TIMEOUT 30) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_test_executable(guest_system_load_exec_standard_error_tests ${LIBS_INCLUDE_DIR}/tests/GuestSystemServiceLoadExecStandardError.cpp) + target_include_directories(guest_system_load_exec_standard_error_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_system_load_exec_standard_error_tests PRIVATE libSceSystemService libkernel libc) + add_test(NAME guest_system_load_exec_standard_error COMMAND guest_system_load_exec_standard_error_tests) + set_tests_properties(guest_system_load_exec_standard_error PROPERTIES TIMEOUT 20) + endif() if(WIN32) set_tests_properties(guest_system_exit guest_requested_exit guest_system_exit_parameters guest_immediate_exit guest_return_from_main guest_quick_exit guest_system_relaunch guest_nested_exit PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") diff --git a/core/libs/prx/libSceSystemService/Export.cpp b/core/libs/prx/libSceSystemService/Export.cpp index 6b5bb3bb44..19bb3e3048 100644 --- a/core/libs/prx/libSceSystemService/Export.cpp +++ b/core/libs/prx/libSceSystemService/Export.cpp @@ -183,6 +183,9 @@ void StartHostExecutable(HostLaunch& launch) { std::vector argv; for (auto& argument : launch.arguments) argv.push_back(argument.data()); argv.push_back(nullptr); + if (const int host = HostStandardError_nid_no_patch(); host >= 0 && ::dup2(host, 2) != 2) { + throw std::runtime_error("sceSystemServiceLoadExec: cannot give the host standard error back to descriptor 2: " + std::string(std::strerror(errno))); + } ::execv(launch.executable.c_str(), argv.data()); throw std::runtime_error("sceSystemServiceLoadExec: execv failed: " + std::string(std::strerror(errno))); } diff --git a/core/libs/prx/libSceSystemService/LoadExec.cpp b/core/libs/prx/libSceSystemService/LoadExec.cpp index cc359f4b53..360e818297 100644 --- a/core/libs/prx/libSceSystemService/LoadExec.cpp +++ b/core/libs/prx/libSceSystemService/LoadExec.cpp @@ -1,4 +1,5 @@ #include "LoadExec.hpp" +#include "prx/libc/include/General.hpp" #include "prx/libc/include/Shutdown.hpp" #include #include @@ -153,6 +154,8 @@ Handle Duplicate(HANDLE source, DWORD access) { pointers.push_back(nullptr); LibcRunShutdown_nid_postfix(); std::fflush(nullptr); + if (const int host = HostStandardError_nid_no_patch(); host >= 0 && dup2(host, 2) != 2) + throw std::system_error(errno, std::generic_category(), "LoadExec: giving the host standard error back to descriptor 2"); std::filesystem::current_path(target.parent_path()); if ((descriptor > 3 && close_range(3, static_cast(descriptor - 1), 0) != 0) || close_range(static_cast(std::max(3, descriptor + 1)), ~0u, 0) != 0) diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index d4f3bc584a..50d0ee43d9 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -1,7 +1,9 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/General.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/AppArchive.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/CreationMask.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/FailureReport.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/gcc/StackGuard.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/windows/CrashReport.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/windows/CallTrace.cpp @@ -56,6 +58,8 @@ target_include_directories(libc PRIVATE ${LIBS_INCLUDE_DIR} ) +target_link_libraries(libc PRIVATE zarchive) + if(WIN32) target_link_libraries(libc PRIVATE advapi32 ws2_32 iphlpapi) endif() @@ -70,6 +74,20 @@ if(BUILD_TESTING) set_tests_properties(guest_allocating_strings PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + + add_test_executable(guest_app_archive_tests ${LIBS_INCLUDE_DIR}/tests/GuestAppArchive.cpp) + target_include_directories(guest_app_archive_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_app_archive_tests PRIVATE libc zarchive) + configure_windows_unwind(guest_app_archive_tests) + foreach(mode present absent damaged invalid) + add_test(NAME guest_app_archive_${mode} COMMAND guest_app_archive_tests ${mode}) + set_tests_properties(guest_app_archive_${mode} PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_app_archive_${mode} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() + add_test_executable(guest_array_callbacks_tests ${LIBS_INCLUDE_DIR}/tests/GuestArrayCallbacks.cpp) target_include_directories(guest_array_callbacks_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_array_callbacks_tests PRIVATE libc) @@ -758,5 +776,15 @@ if(MINGW) set_tests_properties(windows_backtrace_report PROPERTIES TIMEOUT 20 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$" PASS_REGULAR_EXPRESSION "aborting[^~]*SIGABRT. on thread[^~]*stack return address candidates:[^~]*rsp[+]0x") + add_test(NAME windows_backtrace_failure_report COMMAND windows_backtrace_report_tests report) + set_tests_properties(windows_backtrace_failure_report PROPERTIES TIMEOUT 20 + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$" + PASS_REGULAR_EXPRESSION "aborting[^~]*SIGABRT. on thread[^~]*all threads:[^~]*kept failure report ran" + FAIL_REGULAR_EXPRESSION "dropped failure report ran;kept failure report ran[^~]*kept failure report ran") + add_test(NAME windows_backtrace_failure_report_once COMMAND windows_backtrace_report_tests report-once) + set_tests_properties(windows_backtrace_failure_report_once PROPERTIES TIMEOUT 20 + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$" + PASS_REGULAR_EXPRESSION "kept failure report ran[^~]*aborting[^~]*SIGABRT. on thread[^~]*all threads:" + FAIL_REGULAR_EXPRESSION "kept failure report ran[^~]*kept failure report ran") endif() endif() diff --git a/core/libs/prx/libc/include/AppArchive.hpp b/core/libs/prx/libc/include/AppArchive.hpp new file mode 100644 index 0000000000..13d25f34c3 --- /dev/null +++ b/core/libs/prx/libc/include/AppArchive.hpp @@ -0,0 +1,48 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_APPARCHIVE_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_APPARCHIVE_HPP + +#include +#include +#include +#include +#include + +struct AppArchiveNode { + std::uint32_t Handle; + bool Directory; + std::uint64_t Size; +}; + +struct AppArchiveBuffer { + void* Base; + std::size_t Length; +}; + +struct AppArchiveDirectoryEntry { + std::string Name; + bool Directory; +}; + +extern "C" { + +bool AppArchiveLookup_nid_no_patch(const std::filesystem::path& hostPath, AppArchiveNode* node); +std::uint64_t AppArchiveRead_nid_no_patch(std::uint32_t handle, std::uint64_t offset, void* buffer, std::uint64_t length); +std::vector AppArchiveList_nid_no_patch(std::uint32_t handle); +void AppArchiveReady_nid_no_patch(); +void AppArchiveRefuseOnly_nid_no_patch(const std::filesystem::path& hostPath, const char* guestPath, const char* operation); +void AppArchiveRefuseMergedListing_nid_no_patch(const std::filesystem::path& hostPath, const char* guestPath); +void AppArchiveRefuseDescriptor_nid_no_patch(int descriptor, const char* operation); +std::filesystem::path AppArchivePath_nid_no_patch(); +bool AppArchiveOnly_nid_no_patch(const std::filesystem::path& hostPath); +bool AppArchiveHasDirectory_nid_no_patch(const std::filesystem::path& hostPath); + +void AppArchiveAttach_nid_no_patch(int descriptor, const AppArchiveNode& node); +void AppArchiveShare_nid_no_patch(int from, int to); +void AppArchiveDetach_nid_no_patch(int descriptor); +bool AppArchiveDescriptor_nid_no_patch(int descriptor, AppArchiveNode* node, std::uint64_t* position); +bool AppArchiveDescriptorRead_nid_no_patch(int descriptor, const AppArchiveBuffer* buffers, int count, const std::int64_t* offset, std::uint64_t* read); +bool AppArchiveDescriptorSeek_nid_no_patch(int descriptor, std::int64_t offset, int whence, std::int64_t* position); + +} + +#endif diff --git a/core/libs/prx/libc/include/FailureReport.hpp b/core/libs/prx/libc/include/FailureReport.hpp new file mode 100644 index 0000000000..7a05284d6e --- /dev/null +++ b/core/libs/prx/libc/include/FailureReport.hpp @@ -0,0 +1,8 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_FAILUREREPORT_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_FAILUREREPORT_HPP + +extern "C" void LibcRegisterFailureReport_nid_no_patch(void (*report)() noexcept); +extern "C" void LibcUnregisterFailureReport_nid_no_patch(void (*report)() noexcept) noexcept; +extern "C" void LibcRunFailureReports_nid_no_patch() noexcept; + +#endif diff --git a/core/libs/prx/libc/include/FilesystemError.hpp b/core/libs/prx/libc/include/FilesystemError.hpp index c24d52e44e..038b5f04f7 100644 --- a/core/libs/prx/libc/include/FilesystemError.hpp +++ b/core/libs/prx/libc/include/FilesystemError.hpp @@ -25,9 +25,9 @@ inline int FilesystemError(const std::error_code& error) { std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, bool replacesFile); -inline int ResolvePathError_nid_no_patch(const char* path, std::filesystem::path& resolved, bool module = false) { +inline int ResolvePathError_nid_no_patch(const char* path, std::filesystem::path& resolved, bool module = false, bool archived = false) { try { - resolved = module ? ResolvePathOrRelinkedModule_nid_no_patch(path) : ResolvePath_nid_no_patch(path); + resolved = archived ? ResolveArchivedPath_nid_no_patch(path, module) : module ? ResolvePathOrRelinkedModule_nid_no_patch(path) : ResolvePath_nid_no_patch(path); return 0; } catch (const std::filesystem::filesystem_error& error) { if (error.code() == std::errc::illegal_byte_sequence) return 2; diff --git a/core/libs/prx/libc/include/General.hpp b/core/libs/prx/libc/include/General.hpp index 37788afcfb..72dd4a7562 100644 --- a/core/libs/prx/libc/include/General.hpp +++ b/core/libs/prx/libc/include/General.hpp @@ -19,6 +19,8 @@ extern "C" int HostStandardError_nid_no_patch(); extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path); extern "C" std::filesystem::path ResolvePathOrRelinkedModule_nid_no_patch(const char* path); +extern "C" std::filesystem::path ResolveArchivedPath_nid_no_patch(const char* path, bool module = false); +extern "C" std::filesystem::path ResolveDiskPath_nid_no_patch(const char* path); extern "C" std::filesystem::path RelinkedModulePath_nid_no_patch(const std::filesystem::path& path); extern "C" int SetWorkingDirectory_nid_no_patch(const char* resolved); extern "C" bool FinalPathComponentIsDot_nid_no_patch(const char* path); diff --git a/core/libs/prx/libc/include/Shutdown.hpp b/core/libs/prx/libc/include/Shutdown.hpp index 4bfe737f75..af9ae548d1 100644 --- a/core/libs/prx/libc/include/Shutdown.hpp +++ b/core/libs/prx/libc/include/Shutdown.hpp @@ -14,5 +14,7 @@ extern "C" [[noreturn]] void LibcAwaitExit_nid_postfix(); extern "C" [[noreturn]] void LibcExit_nid_no_patch(int code); extern "C" [[noreturn]] void LibcExitWithoutHandlers_nid_no_patch(int code); extern "C" [[noreturn]] void LibcTerminate_nid_no_patch(int code); +extern "C" void LibcSetEndFlush_nid_no_patch(void (*flush)()); +extern "C" void LibcRunEndFlush_nid_no_patch(); #endif diff --git a/core/libs/prx/libc/include/general/LogMacros.hpp b/core/libs/prx/libc/include/general/LogMacros.hpp index 46ba77734e..00e511b24b 100644 --- a/core/libs/prx/libc/include/general/LogMacros.hpp +++ b/core/libs/prx/libc/include/general/LogMacros.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GENERAL_LOGMACROS_HPP #define CORE_LIBS_PRX_LIBC_INCLUDE_GENERAL_LOGMACROS_HPP +#include #include #include #include @@ -8,6 +9,25 @@ #include #include +using LogSinkFn = int (*)(std::FILE* stream, const char* text, std::size_t length); + +inline std::atomic& LogSink() { + static std::atomic sink{nullptr}; + return sink; +} + +inline long long LogTimestampMicros() { + static const bool timestamps = std::getenv("APS5_LOG_TIMESTAMPS") != nullptr; + if (!timestamps) return -1; + using Clock = std::chrono::steady_clock; + static const auto started = Clock::now(); + return static_cast(std::chrono::duration_cast(Clock::now() - started).count()); +} + +inline int FormatLogTimestamp(char* out, std::size_t capacity, long long micros) { + return std::snprintf(out, capacity, "[+%llu.%06llu s] ", static_cast(micros / 1000000), static_cast(micros % 1000000)); +} + #ifdef __MINGW_PRINTF_FORMAT [[gnu::format(__MINGW_PRINTF_FORMAT, 2, 3)]] #else @@ -16,14 +36,9 @@ inline int WriteLogLine(std::FILE* stream, const char* format, ...) { char small[1024]; std::size_t prefixLength = 0; - static const bool timestamps = std::getenv("APS5_LOG_TIMESTAMPS") != nullptr; - if (timestamps) { - using Clock = std::chrono::steady_clock; - static const auto started = Clock::now(); - const auto elapsed = std::chrono::duration_cast(Clock::now() - started).count(); - const auto seconds = static_cast(elapsed / 1000000); - const auto microseconds = static_cast(elapsed % 1000000); - const int written = std::snprintf(small, sizeof(small), "[+%llu.%06llu s] ", seconds, microseconds); + const long long micros = LogTimestampMicros(); + if (micros >= 0) { + const int written = FormatLogTimestamp(small, sizeof(small), micros); if (written < 0 || written >= static_cast(sizeof(small))) return -1; prefixLength = static_cast(written); } @@ -43,7 +58,9 @@ inline int WriteLogLine(std::FILE* stream, const char* format, ...) { va_end(again); if (length < 0) return length; const auto totalLength = prefixLength + static_cast(length); - std::fwrite(large.empty() ? small : large.data(), 1, totalLength, stream); + const char* text = large.empty() ? small : large.data(); + if (const auto sink = LogSink().load(std::memory_order_acquire)) return sink(stream, text, totalLength); + std::fwrite(text, 1, totalLength, stream); return std::fflush(stream); } diff --git a/core/libs/prx/libc/src/AppArchive.cpp b/core/libs/prx/libc/src/AppArchive.cpp new file mode 100644 index 0000000000..be3916a765 --- /dev/null +++ b/core/libs/prx/libc/src/AppArchive.cpp @@ -0,0 +1,239 @@ +#include "prx/libc/include/AppArchive.hpp" +#include "prx/libc/include/General.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Archive { + std::mutex mutex; + bool opened = false; + std::unique_ptr reader; + std::filesystem::path path; + std::filesystem::path app0; +}; + +Archive& State() { + static Archive archive; + return archive; +} + +bool SameComponent(std::string_view left, std::string_view right) { + if (left.size() != right.size()) return false; + const auto fold = [](unsigned char value) { return value >= 'A' && value <= 'Z' ? value + ('a' - 'A') : value; }; + for (std::size_t i = 0; i < left.size(); ++i) if (fold(left[i]) != fold(right[i])) return false; + return true; +} + +Archive& Opened() { + auto& archive = State(); + std::lock_guard lock(archive.mutex); + if (archive.opened) return archive; + const auto app0 = ResolveDiskPath_nid_no_patch("/app0").lexically_normal(); + auto candidate = app0; + candidate += ".zar"; + std::error_code error; + if (std::filesystem::is_regular_file(candidate, error)) { + archive.reader.reset(ZArchiveReader::OpenFromFile(candidate)); + if (!archive.reader) throw std::runtime_error("app0.zar is not a readable ZArchive: " + candidate.string()); + archive.path = candidate; + } + archive.app0 = app0; + archive.opened = true; + return archive; +} + +bool ArchiveRelative(const Archive& archive, const std::filesystem::path& hostPath, std::string& relative) { + const auto normal = hostPath.lexically_normal(); + auto part = normal.begin(); + for (const auto& component : archive.app0) { + if (part == normal.end() || !SameComponent(part->generic_string(), component.generic_string())) return false; + ++part; + } + relative.clear(); + for (; part != normal.end(); ++part) { + const auto name = part->generic_string(); + if (name.empty() || name == ".") continue; + if (name == "..") return false; + if (!relative.empty()) relative += '/'; + relative += name; + } + return true; +} + +struct OpenFile { + AppArchiveNode node; + std::uint64_t position = 0; + std::mutex mutex; +}; + +std::mutex g_descriptorMutex; +std::map> g_descriptors; + +std::shared_ptr Attached(int descriptor) { + std::lock_guard lock(g_descriptorMutex); + const auto found = g_descriptors.find(descriptor); + return found == g_descriptors.end() ? nullptr : found->second; +} + +ZArchiveReader& Reader(std::uint32_t handle) { + auto& archive = Opened(); + if (!archive.reader) throw std::logic_error("app0.zar handle " + std::to_string(handle) + " used without an archive"); + return *archive.reader; +} + +} + +extern "C" { + +bool AppArchiveLookup_nid_no_patch(const std::filesystem::path& hostPath, AppArchiveNode* node) { + if (node == nullptr) throw std::invalid_argument("AppArchiveLookup: null node"); + auto& archive = Opened(); + if (!archive.reader) return false; + std::string relative; + if (!ArchiveRelative(archive, hostPath, relative)) return false; + const auto handle = archive.reader->LookUp(relative); + if (handle == ZARCHIVE_INVALID_NODE) return false; + node->Handle = handle; + node->Directory = archive.reader->IsDirectory(handle); + node->Size = node->Directory ? 0 : archive.reader->GetFileSize(handle); + return true; +} + +std::uint64_t AppArchiveRead_nid_no_patch(std::uint32_t handle, std::uint64_t offset, void* buffer, std::uint64_t length) { + auto& reader = Reader(handle); + if (!reader.IsFile(handle)) throw std::invalid_argument("AppArchiveRead: handle " + std::to_string(handle) + " is not a file"); + const auto size = reader.GetFileSize(handle); + if (length == 0 || offset >= size) return 0; + if (buffer == nullptr) throw std::invalid_argument("AppArchiveRead: null buffer"); + const auto expected = std::min(length, size - offset); + const auto read = reader.ReadFromFile(handle, offset, length, buffer); + if (read != expected) throw std::runtime_error("app0.zar is damaged: a compressed block of handle " + std::to_string(handle) + " cannot be read"); + return read; +} + +std::vector AppArchiveList_nid_no_patch(std::uint32_t handle) { + auto& reader = Reader(handle); + if (!reader.IsDirectory(handle)) throw std::invalid_argument("AppArchiveList: handle " + std::to_string(handle) + " is not a directory"); + std::vector entries; + const auto count = reader.GetDirEntryCount(handle); + entries.reserve(count); + for (std::uint32_t index = 0; index < count; ++index) { + ZArchiveReader::DirEntry entry{}; + if (!reader.GetDirEntry(handle, index, entry)) throw std::runtime_error("app0.zar is damaged: directory entry " + std::to_string(index) + " cannot be read"); + entries.push_back({std::string(entry.name), entry.isDirectory}); + } + return entries; +} + +void AppArchiveReady_nid_no_patch() { + (void)Opened(); +} + +void AppArchiveRefuseOnly_nid_no_patch(const std::filesystem::path& hostPath, const char* guestPath, const char* operation) { + if (AppArchiveOnly_nid_no_patch(hostPath)) NotImplemented_nid_no_patch((std::string(operation) + " " + guestPath + " in app0.zar").c_str()); +} + +void AppArchiveRefuseMergedListing_nid_no_patch(const std::filesystem::path& hostPath, const char* guestPath) { + if (AppArchiveHasDirectory_nid_no_patch(hostPath)) NotImplemented_nid_no_patch((std::string("listing ") + guestPath + " merged with app0.zar").c_str()); +} + +void AppArchiveRefuseDescriptor_nid_no_patch(int descriptor, const char* operation) { + if (AppArchiveDescriptor_nid_no_patch(descriptor, nullptr, nullptr)) NotImplemented_nid_no_patch((std::string(operation) + " of a file read from app0.zar").c_str()); +} + +std::filesystem::path AppArchivePath_nid_no_patch() { + return Opened().path; +} + +bool AppArchiveOnly_nid_no_patch(const std::filesystem::path& hostPath) { + auto& archive = Opened(); + if (!archive.reader) return false; + std::error_code error; + const auto status = std::filesystem::symlink_status(hostPath, error); + if (std::filesystem::exists(status) || (error && error != std::errc::no_such_file_or_directory)) return false; + AppArchiveNode node{}; + return AppArchiveLookup_nid_no_patch(hostPath, &node); +} + +bool AppArchiveHasDirectory_nid_no_patch(const std::filesystem::path& hostPath) { + AppArchiveNode node{}; + return AppArchiveLookup_nid_no_patch(hostPath, &node) && node.Directory; +} + +void AppArchiveAttach_nid_no_patch(int descriptor, const AppArchiveNode& node) { + if (node.Directory) throw std::invalid_argument("AppArchiveAttach: descriptor " + std::to_string(descriptor) + " cannot hold an app0.zar directory"); + auto file = std::make_shared(); + file->node = node; + std::lock_guard lock(g_descriptorMutex); + g_descriptors[descriptor] = std::move(file); +} + +void AppArchiveShare_nid_no_patch(int from, int to) { + std::lock_guard lock(g_descriptorMutex); + const auto found = g_descriptors.find(from); + if (found == g_descriptors.end()) g_descriptors.erase(to); + else g_descriptors[to] = found->second; +} + +void AppArchiveDetach_nid_no_patch(int descriptor) { + std::lock_guard lock(g_descriptorMutex); + g_descriptors.erase(descriptor); +} + +bool AppArchiveDescriptor_nid_no_patch(int descriptor, AppArchiveNode* node, std::uint64_t* position) { + const auto file = Attached(descriptor); + if (!file) return false; + std::lock_guard lock(file->mutex); + if (node != nullptr) *node = file->node; + if (position != nullptr) *position = file->position; + return true; +} + +bool AppArchiveDescriptorRead_nid_no_patch(int descriptor, const AppArchiveBuffer* buffers, int count, const std::int64_t* offset, std::uint64_t* read) { + const auto file = Attached(descriptor); + if (!file) return false; + if (read == nullptr || count < 0 || (buffers == nullptr && count != 0)) throw std::invalid_argument("AppArchiveDescriptorRead: invalid arguments for descriptor " + std::to_string(descriptor)); + if (offset != nullptr && *offset < 0) throw std::invalid_argument("AppArchiveDescriptorRead: negative offset for descriptor " + std::to_string(descriptor)); + std::lock_guard lock(file->mutex); + auto cursor = offset != nullptr ? static_cast(*offset) : file->position; + std::uint64_t total = 0; + for (int i = 0; i < count; ++i) { + const auto done = AppArchiveRead_nid_no_patch(file->node.Handle, cursor, buffers[i].Base, buffers[i].Length); + cursor += done; + total += done; + if (done < buffers[i].Length) break; + } + if (offset == nullptr) file->position = cursor; + *read = total; + return true; +} + +bool AppArchiveDescriptorSeek_nid_no_patch(int descriptor, std::int64_t offset, int whence, std::int64_t* position) { + const auto file = Attached(descriptor); + if (!file) return false; + if (position == nullptr) throw std::invalid_argument("AppArchiveDescriptorSeek: null position"); + std::lock_guard lock(file->mutex); + constexpr auto limit = std::numeric_limits::max(); + std::int64_t base = 0; + if (whence == 1) base = static_cast(file->position); + else if (whence == 2) base = static_cast(file->node.Size); + else if (whence != 0) throw std::invalid_argument("AppArchiveDescriptorSeek: invalid whence " + std::to_string(whence)); + if ((offset > 0 && base > limit - offset) || base + offset < 0) { + *position = -1; + return true; + } + file->position = static_cast(base + offset); + *position = base + offset; + return true; +} + +} diff --git a/core/libs/prx/libc/src/Directory.cpp b/core/libs/prx/libc/src/Directory.cpp index ff24e15a7e..d831ad7976 100644 --- a/core/libs/prx/libc/src/Directory.cpp +++ b/core/libs/prx/libc/src/Directory.cpp @@ -1,6 +1,7 @@ #include "prx/libc/include/GuestDirectory.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/FilesystemError.hpp" +#include "prx/libc/include/AppArchive.hpp" #include #include #include @@ -109,6 +110,7 @@ void* APS5_VABI opendir_nid_postfix(const char* path) { try { auto directory = std::make_unique(); if (const int error = ResolvePathError_nid_no_patch(path, directory->path)) { errno = error; return nullptr; } + AppArchiveRefuseMergedListing_nid_no_patch(directory->path, path); directory->native = OpenNative(directory->path); if (!directory->native) { errno = DirectoryError(errno); return nullptr; } return directory.release(); diff --git a/core/libs/prx/libc/src/Environment.cpp b/core/libs/prx/libc/src/Environment.cpp index 03996644bf..cbb171729b 100644 --- a/core/libs/prx/libc/src/Environment.cpp +++ b/core/libs/prx/libc/src/Environment.cpp @@ -9,10 +9,6 @@ #include #include "prx/libc/include/general/VabiMacros.hpp" -#ifndef _WIN32 -extern char** environ; -#endif - namespace { struct EnvironmentEntry { std::unique_ptr owned; @@ -26,20 +22,7 @@ struct EnvironmentEntry { }; std::mutex environmentMutex; std::vector& Environment() { - // Seed once from the host. Guest mutations remain local to the guest runtime. - static std::vector entries = [] { - std::vector result; -#ifdef _WIN32 - char** source = _environ; -#else - char** source = environ; -#endif - if (source) for (; *source; ++source) { - if (**source && **source != '=' && std::strchr(*source, '=')) - result.push_back({std::make_unique(*source), nullptr}); - } - return result; - }(); + static std::vector entries; return entries; } bool ValidEnvironmentName(const char* name) { diff --git a/core/libs/prx/libc/src/FailureReport.cpp b/core/libs/prx/libc/src/FailureReport.cpp new file mode 100644 index 0000000000..9fcb5473de --- /dev/null +++ b/core/libs/prx/libc/src/FailureReport.cpp @@ -0,0 +1,39 @@ +#include "prx/libc/include/FailureReport.hpp" +#include +#include +#include + +namespace { + +using Report = void (*)() noexcept; + +std::array, 8> reports{}; +std::atomic ran{false}; + +} + +extern "C" void LibcRegisterFailureReport_nid_no_patch(Report report) { + if (report == nullptr) throw std::invalid_argument("libc: null failure report"); + for (auto& slot : reports) { + if (slot.load(std::memory_order_acquire) == report) return; + } + for (auto& slot : reports) { + Report empty = nullptr; + if (slot.compare_exchange_strong(empty, report, std::memory_order_acq_rel)) return; + } + throw std::runtime_error("libc: more than 8 failure reports registered"); +} + +extern "C" void LibcUnregisterFailureReport_nid_no_patch(Report report) noexcept { + for (auto& slot : reports) { + Report expected = report; + slot.compare_exchange_strong(expected, nullptr, std::memory_order_acq_rel); + } +} + +extern "C" void LibcRunFailureReports_nid_no_patch() noexcept { + if (ran.exchange(true, std::memory_order_acq_rel)) return; + for (auto& slot : reports) { + if (const auto report = slot.load(std::memory_order_acquire)) report(); + } +} diff --git a/core/libs/prx/libc/src/FileAndHeap.cpp b/core/libs/prx/libc/src/FileAndHeap.cpp index 16025ecbb9..bf4ae212de 100644 --- a/core/libs/prx/libc/src/FileAndHeap.cpp +++ b/core/libs/prx/libc/src/FileAndHeap.cpp @@ -26,6 +26,7 @@ #include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/FilesystemError.hpp" +#include "prx/libc/include/AppArchive.hpp" #include "prx/libc/include/GuestArena.hpp" #ifdef __linux__ @@ -237,6 +238,7 @@ extern "C" { FileStream* APS5_VABI fdopen_nid_postfix(int descriptor, const char* mode) { if (descriptor < 0) { errno = 9; return nullptr; } if (!mode) { errno = 22; return nullptr; } + AppArchiveRefuseDescriptor_nid_no_patch(descriptor, "fdopen"); const auto parsed = ParseFileMode(mode); if (!parsed) { errno = 22; return nullptr; } try { diff --git a/core/libs/prx/libc/src/General.cpp b/core/libs/prx/libc/src/General.cpp index 2bbfbb1f5e..27caa6d483 100644 --- a/core/libs/prx/libc/src/General.cpp +++ b/core/libs/prx/libc/src/General.cpp @@ -31,6 +31,7 @@ #include #endif #include "prx/libc/include/General.hpp" +#include "prx/libc/include/AppArchive.hpp" #include "prx/libc/include/ApplicationHeap.hpp" namespace { @@ -192,7 +193,7 @@ void RequireHostDirectory(WorkingDirectory& state, const std::filesystem::path& #endif throw std::filesystem::filesystem_error("Guest path component is not a directory", host, error); } -std::filesystem::path Resolve(WorkingDirectory& state, const char* path, bool module = false) { +std::filesystem::path Resolve(WorkingDirectory& state, const char* path, bool module = false, bool archived = false) { std::string text(path); for (auto& character : text) if (character == '\\') character = '/'; std::filesystem::path input(text); @@ -206,7 +207,7 @@ std::filesystem::path Resolve(WorkingDirectory& state, const char* path, bool mo for (auto part = relative.begin(); part != relative.end(); ++part) { if (*part == "..") { const auto parent = guest.lexically_normal(); - if (parent.has_relative_path()) RequireHostDirectory(state, parent); + if (parent.has_relative_path() && !(archived && AppArchiveHasDirectory_nid_no_patch(HostPath(state, parent)))) RequireHostDirectory(state, parent); } guest /= *part; } @@ -424,18 +425,32 @@ extern "C" void BlockPathAlias_nid_no_patch(const char* guestPrefix) { aliases.blocked.push_back(prefix); } -extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path) { +static std::filesystem::path ResolveLocked(const char* path, bool module, bool archived) { if (!path) { APS5_INVALID_ARG_EX; } + if (archived) AppArchiveReady_nid_no_patch(); auto& state = Directories(); std::lock_guard lock(state.mutex); - return Resolve(state, path); + return Resolve(state, path, module, archived); +} + +extern "C" std::filesystem::path ResolveDiskPath_nid_no_patch(const char* path) { + return ResolveLocked(path, false, false); +} + +extern "C" std::filesystem::path ResolveArchivedPath_nid_no_patch(const char* path, bool module) { + return ResolveLocked(path, module, true); +} + +extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path) { + auto resolved = ResolveLocked(path, false, true); + AppArchiveRefuseOnly_nid_no_patch(resolved, path, "reading"); + return resolved; } extern "C" std::filesystem::path ResolvePathOrRelinkedModule_nid_no_patch(const char* path) { - if (!path) { APS5_INVALID_ARG_EX; } - auto& state = Directories(); - std::lock_guard lock(state.mutex); - return Resolve(state, path, true); + auto resolved = ResolveLocked(path, true, true); + AppArchiveRefuseOnly_nid_no_patch(resolved, path, "reading"); + return resolved; } extern "C" std::filesystem::path RelinkedModulePath_nid_no_patch(const std::filesystem::path& path) { @@ -472,9 +487,11 @@ extern "C" int APS5_VABI chdir_nid_postfix(const char* path) { if (!path) { errno = 14; return -1; } if (!*path) { errno = 2; return -1; } try { + AppArchiveReady_nid_no_patch(); auto& state = Directories(); std::lock_guard lock(state.mutex); - const auto resolved = Resolve(state, path); + const auto resolved = Resolve(state, path, false, true); + AppArchiveRefuseOnly_nid_no_patch(resolved, path, "changing the working directory to"); const int error = SetWorkingDirectory(state, resolved); if (error) { errno = error; return -1; } return 0; diff --git a/core/libs/prx/libc/src/Process.cpp b/core/libs/prx/libc/src/Process.cpp index 78b4e6195e..08aab26c27 100644 --- a/core/libs/prx/libc/src/Process.cpp +++ b/core/libs/prx/libc/src/Process.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -98,6 +99,38 @@ extern "C" void LibcRunShutdown_nid_postfix() { namespace { +std::atomic endFlush{nullptr}; +void (*previousAbortHandler)(int) = SIG_DFL; + +void FlushOnAbort(int number) { + LibcRunEndFlush_nid_no_patch(); + if (previousAbortHandler == SIG_IGN) return; + if (previousAbortHandler != SIG_DFL && previousAbortHandler != SIG_ERR) { + previousAbortHandler(number); + return; + } +#ifndef _WIN32 + std::signal(number, SIG_DFL); + std::raise(number); +#endif +} + +} + +extern "C" void LibcSetEndFlush_nid_no_patch(void (*flush)()) { + if (flush == nullptr) throw std::runtime_error("libc: invalid end flush registration"); + void (*registered)() = nullptr; + if (!endFlush.compare_exchange_strong(registered, flush) && registered != flush) throw std::runtime_error("libc: end flush already registered"); + static std::once_flag installed; + std::call_once(installed, [] { previousAbortHandler = std::signal(SIGABRT, FlushOnAbort); }); +} + +extern "C" void LibcRunEndFlush_nid_no_patch() { + if (const auto flush = endFlush.load(std::memory_order_acquire)) flush(); +} + +namespace { + void StartExit() { exitRequested.store(true); if (exitStarted.exchange(true)) { @@ -122,6 +155,7 @@ extern "C" { } [[noreturn]] void LibcTerminate_nid_no_patch(int code) { + LibcRunEndFlush_nid_no_patch(); #ifdef _WIN32 TerminateProcess(GetCurrentProcess(), static_cast(code)); #endif @@ -159,6 +193,7 @@ void APS5_VABI exit_nid_postfix(int code) { ) { (void)arg0; (void)arg1; (void)arg2; (void)arg3; (void)arg4; (void)arg5; + LibcRunEndFlush_nid_no_patch(); std::fprintf(stderr, "[libc] abort() called from %p\n", __builtin_return_address(0)); std::fflush(nullptr); std::abort(); diff --git a/core/libs/prx/libc/src/exception/Exports.cpp b/core/libs/prx/libc/src/exception/Exports.cpp index dbc0c7acbc..4f6575b340 100644 --- a/core/libs/prx/libc/src/exception/Exports.cpp +++ b/core/libs/prx/libc/src/exception/Exports.cpp @@ -18,7 +18,6 @@ GCC_GLOBAL_ALIAS(__cxa_throw_nid_no_patch_cut, __cxa_throw_nid_postfix); GCC_GLOBAL_ALIAS(__cxa_begin_catch_nid_no_patch_cut, __cxa_begin_catch_nid_postfix); GCC_GLOBAL_ALIAS(__cxa_end_catch_nid_no_patch_cut, __cxa_end_catch_nid_postfix); GCC_GLOBAL_ALIAS(__cxa_rethrow_nid_no_patch_cut, __cxa_rethrow_nid_postfix); -GCC_GLOBAL_ALIAS(__gxx_personality_v0_nid_no_patch_cut, __gxx_personality_v0_nid_postfix); GCC_GLOBAL_ALIAS(_Unwind_Resume_nid_no_patch_cut, _Unwind_Resume_nid_postfix); GCC_GLOBAL_ALIAS(__cxa_free_exception, __cxa_free_exception_nid_postfix); GCC_GLOBAL_ALIAS(_ZSt9terminatev, _ZSt9terminatev_nid_postfix); diff --git a/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp b/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp index 6491da364c..19691b7964 100644 --- a/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp +++ b/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp @@ -2,7 +2,9 @@ #include #include #include "prx/libc/include/CallTrace.hpp" +#include "prx/libc/include/FailureReport.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/Shutdown.hpp" #include #include #include @@ -304,6 +306,7 @@ LONG WINAPI ReportCrash(EXCEPTION_POINTERS* info) { // Another thread is already reporting; let it finish before this fault ends the process. Sleep(INFINITE); } + LibcRunEndFlush_nid_no_patch(); const auto* context = info->ContextRecord; char line[MAX_PATH + 64]; char threadName[128] = ""; @@ -394,6 +397,7 @@ LONG WINAPI ReportCrash(EXCEPTION_POINTERS* info) { ReportCallTrace(); std::fflush(stderr); ReportAllThreads(); + LibcRunFailureReports_nid_no_patch(); std::fflush(stderr); return EXCEPTION_CONTINUE_SEARCH; } @@ -546,10 +550,12 @@ void ReportBacktrace(const char* what, bool reportThreads) { void AbortSignalHandler(int) { ReportBacktrace("abort() (SIGABRT)", true); + LibcRunFailureReports_nid_no_patch(); } void InvalidParameterHandler(const wchar_t*, const wchar_t*, const wchar_t*, unsigned, std::uintptr_t) { ReportBacktrace("UCRT invalid parameter", true); + LibcRunFailureReports_nid_no_patch(); } // Static destruction runs on the thread that ends the process (ExitProcess -> DLL_PROCESS_DETACH), diff --git a/core/libs/prx/libkernel/Aio/src/Aio.cpp b/core/libs/prx/libkernel/Aio/src/Aio.cpp index c16167088f..8f7367c074 100644 --- a/core/libs/prx/libkernel/Aio/src/Aio.cpp +++ b/core/libs/prx/libkernel/Aio/src/Aio.cpp @@ -10,6 +10,7 @@ #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestArena.hpp" #include "prx/libc/include/PreciseWait.hpp" +#include "prx/libkernel/File/include/ArchivedFile.hpp" #include "prx/libkernel/File/include/File.hpp" #include "prx/libkernel/File/include/SharedMemory.hpp" #include "prx/libkernel/KernelErrors.hpp" @@ -62,6 +63,11 @@ std::int64_t NativePread(std::int32_t fd, void* buf, std::size_t nbyte, std::int errno = EFAULT; return -1; } + if (const auto archived = File::ReadArchivedFile(fd, buf, nbyte, &offset)) { + if (*archived >= 0) return *archived; + errno = static_cast(-*archived); + return -1; + } #ifdef _WIN32 if (offset < 0) { errno = EINVAL; diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 21cf02d628..52472fd9ad 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Apr/src/Apr.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Equeue/Equeue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Equeue/src/EventAccessors.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/File/src/ArchivedFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/File/src/DescriptorControl.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Equeue/src/FileEvent.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Equeue/src/Kqueue.cpp @@ -47,6 +48,7 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Socket/src/Udp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Socket/src/SocketOptions.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Socket/src/AddressConversion.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Socket/src/SocketConnect.cpp ${CMAKE_CURRENT_SOURCE_DIR}/EventFlag/src/EventFlag.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Module/src/Module.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Module/src/DynamicLoader.cpp @@ -81,6 +83,7 @@ target_link_libraries(libkernel PRIVATE ${CMAKE_DL_LIBS}) target_link_libraries(libkernel PRIVATE libc) if(WIN32) target_link_libraries(libkernel PRIVATE ws2_32 shell32 advapi32 wbemuuid ole32 oleaut32) + target_link_libraries(libkernel PRIVATE iphlpapi) target_compile_definitions(libkernel PRIVATE _WIN32_WINNT=0x0602) endif() configure_windows_unwind(libkernel) @@ -224,7 +227,7 @@ if(BUILD_TESTING) add_dependencies(guest_dynamic_loader_fixed_base_tests guest_module_fixture) set_target_properties(guest_dynamic_loader_fixed_base_tests PROPERTIES ENABLE_EXPORTS ON POSITION_INDEPENDENT_CODE OFF) target_compile_options(guest_dynamic_loader_fixed_base_tests PRIVATE -fno-pie) - target_link_options(guest_dynamic_loader_fixed_base_tests PRIVATE -no-pie) + target_link_options(guest_dynamic_loader_fixed_base_tests PRIVATE -no-pie -fuse-ld=bfd) add_test(NAME guest_dynamic_loader_fixed_base COMMAND guest_dynamic_loader_fixed_base_tests $) set_tests_properties(guest_dynamic_loader_fixed_base PROPERTIES WORKING_DIRECTORY "$") @@ -1169,6 +1172,18 @@ if(BUILD_TESTING) set_tests_properties(guest_socket_pair PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_kernel_app_archive_tests ${LIBS_INCLUDE_DIR}/tests/GuestKernelAppArchive.cpp) + target_include_directories(guest_kernel_app_archive_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_kernel_app_archive_tests PRIVATE libkernel libc zarchive) + configure_windows_unwind(guest_kernel_app_archive_tests) + foreach(mode archive plain mmap) + add_test(NAME guest_kernel_app_archive_${mode} COMMAND guest_kernel_app_archive_tests ${mode}) + set_tests_properties(guest_kernel_app_archive_${mode} PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_kernel_app_archive_${mode} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() add_test_executable(guest_dup_tests ${LIBS_INCLUDE_DIR}/tests/GuestDup.cpp) target_include_directories(guest_dup_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_dup_tests PRIVATE libkernel libc) @@ -1399,8 +1414,14 @@ if(BUILD_TESTING) configure_windows_unwind(guest_kernel_fcntl_tests) add_test(NAME guest_kernel_fcntl COMMAND guest_kernel_fcntl_tests) set_tests_properties(guest_kernel_fcntl PROPERTIES TIMEOUT 15) + add_test_executable(guest_append_writes_tests ${LIBS_INCLUDE_DIR}/tests/GuestAppendWrites.cpp) + target_include_directories(guest_append_writes_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_append_writes_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_append_writes_tests) + add_test(NAME guest_append_writes COMMAND guest_append_writes_tests) + set_tests_properties(guest_append_writes PROPERTIES TIMEOUT 20) if(WIN32) - set_tests_properties(guest_kernel_fcntl PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_kernel_fcntl guest_append_writes PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") add_test_executable(guest_file_lock_tests ${LIBS_INCLUDE_DIR}/tests/GuestFileLock.cpp) target_include_directories(guest_file_lock_tests PRIVATE ${LIBS_INCLUDE_DIR}) diff --git a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp index 6f2c99760c..250d252b47 100644 --- a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp @@ -3,6 +3,7 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libkernel/File/include/ArchivedFile.hpp" #include "prx/libkernel/File/include/SharedMemory.hpp" #include #include @@ -134,6 +135,7 @@ void* APS5_VABI mmap_nid_postfix(void* address, std::size_t length, int protecti (mappingFlags & ~(GuestShared | GuestPrivate | GuestMapFixed | GuestMapNoOverwrite | GuestMapNoCoalesce)) != 0) return failed(GuestNotSupported); if (descriptor < 0) return failed(GuestBadDescriptor); + File::RefuseArchivedFile(descriptor, "mmap"); if (offset < 0 || (static_cast(offset) & (PS5_PAGE_SIZE - 1)) != 0 || rounded > static_cast(std::numeric_limits::max()) || static_cast(offset) > static_cast(std::numeric_limits::max()) - rounded) diff --git a/core/libs/prx/libkernel/File/include/ArchivedFile.hpp b/core/libs/prx/libkernel/File/include/ArchivedFile.hpp new file mode 100644 index 0000000000..73f84c1cca --- /dev/null +++ b/core/libs/prx/libkernel/File/include/ArchivedFile.hpp @@ -0,0 +1,28 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_FILE_ARCHIVEDFILE_HPP +#define CORE_LIBS_PRX_LIBKERNEL_FILE_ARCHIVEDFILE_HPP + +#include +#include +#include +#include + +#include "SceTypes.hpp" + +namespace File { + +std::optional OpenArchivedFile(const std::filesystem::path& native, int flags); +void RefuseMergedDirectory(const std::filesystem::path& native); +bool IsArchivedFile(int fd); +void RefuseArchivedFile(int fd, const char* operation); +std::optional ReadArchivedFile(int fd, void* buf, std::size_t nbytes, const std::int64_t* offset); +std::optional ReadArchivedIovecs(int fd, const void* iov, int iovcnt, const std::int64_t* offset); +std::optional SeekArchivedFile(int fd, std::int64_t offset, int whence); +std::optional ArchivedFileRemaining(int fd); +bool FillArchivedStat(const std::filesystem::path& native, FileStat* sb); +bool FillArchivedStatFromDescriptor(int fd, FileStat* sb); +void ShareArchivedFile(int from, int to); +void ForgetArchivedFile(int fd); + +} + +#endif diff --git a/core/libs/prx/libkernel/File/include/DescriptorControl.hpp b/core/libs/prx/libkernel/File/include/DescriptorControl.hpp index cb327dff54..d7345faec2 100644 --- a/core/libs/prx/libkernel/File/include/DescriptorControl.hpp +++ b/core/libs/prx/libkernel/File/include/DescriptorControl.hpp @@ -14,6 +14,7 @@ int DuplicateFrom(int fd, int start); void RememberDescriptor(int fd, int openFlags); void ShareDescriptor(int from, int to); void ForgetDescriptor(int fd); +bool Appends(int fd); } diff --git a/core/libs/prx/libkernel/File/include/File.hpp b/core/libs/prx/libkernel/File/include/File.hpp index 4a14063e0e..d759d1e3f9 100644 --- a/core/libs/prx/libkernel/File/include/File.hpp +++ b/core/libs/prx/libkernel/File/include/File.hpp @@ -30,6 +30,9 @@ namespace File { int HostStandardError(); [[nodiscard]] std::unique_lock LockPositionedTransfer(int descriptor); void SharePositionedTransfer(int from, int to); +#ifdef _WIN32 +std::int64_t WriteAtEnd(int descriptor, const void* buf, std::size_t nbytes); +#endif } #endif diff --git a/core/libs/prx/libkernel/File/src/ArchivedFile.cpp b/core/libs/prx/libkernel/File/src/ArchivedFile.cpp new file mode 100644 index 0000000000..df3a8548b4 --- /dev/null +++ b/core/libs/prx/libkernel/File/src/ArchivedFile.cpp @@ -0,0 +1,121 @@ +#include "prx/libkernel/File/include/ArchivedFile.hpp" +#include "prx/libkernel/File/include/FileFlags.hpp" +#include "prx/libkernel/File/include/NativeStat.hpp" +#include "prx/libc/include/AppArchive.hpp" +#include "prx/libc/include/General.hpp" + +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#include +#endif + +namespace { + +constexpr std::uint16_t GuestRegularMode = 0100444; +constexpr std::uint16_t GuestDirectoryMode = 0040555; + +int OpenPlaceholder() { +#ifdef _WIN32 + return ::_open("NUL", _O_RDONLY | _O_BINARY); +#else + return ::open("/dev/null", O_RDONLY | O_CLOEXEC); +#endif +} + +void FillNode(const AppArchiveNode& node, FileStat* sb) { + File::FillFileStat(AppArchivePath_nid_no_patch(), sb); + sb->st_mode = node.Directory ? GuestDirectoryMode : GuestRegularMode; + sb->st_ino = node.Handle + 1; + sb->st_nlink = node.Directory ? 2 : 1; + sb->st_size = static_cast(node.Size); + sb->st_blocks = static_cast((node.Size + 511) / 512); +} + +} + +namespace File { + +std::optional OpenArchivedFile(const std::filesystem::path& native, int flags) { + if (!AppArchiveOnly_nid_no_patch(native)) return std::nullopt; + AppArchiveNode node{}; + if (!AppArchiveLookup_nid_no_patch(native, &node)) throw std::logic_error("OpenArchivedFile: " + native.string() + " left app0.zar while opening"); + if ((flags & (SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL)) == (SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL)) return -EEXIST; + if ((flags & SCE_KERNEL_O_ACCMODE) != SCE_KERNEL_O_RDONLY || (flags & SCE_KERNEL_O_TRUNC) != 0) return -EROFS; + if (node.Directory) NotImplemented_nid_no_patch(("opening the app0.zar directory " + native.string()).c_str()); + if ((flags & SCE_KERNEL_O_DIRECTORY) != 0) return -ENOTDIR; + const int fd = OpenPlaceholder(); + if (fd < 0) return -errno; + AppArchiveAttach_nid_no_patch(fd, node); + return fd; +} + +void RefuseMergedDirectory(const std::filesystem::path& native) { + if (!AppArchiveHasDirectory_nid_no_patch(native)) return; + std::error_code error; + if (std::filesystem::is_directory(native, error)) NotImplemented_nid_no_patch(("opening " + native.string() + " merged with app0.zar").c_str()); +} + +bool IsArchivedFile(int fd) { + return AppArchiveDescriptor_nid_no_patch(fd, nullptr, nullptr); +} + +void RefuseArchivedFile(int fd, const char* operation) { + if (IsArchivedFile(fd)) NotImplemented_nid_no_patch((std::string(operation) + " on a file read from app0.zar, fd=" + std::to_string(fd)).c_str()); +} + +std::optional ReadArchivedFile(int fd, void* buf, std::size_t nbytes, const std::int64_t* offset) { + const AppArchiveBuffer buffer{buf, nbytes}; + return ReadArchivedIovecs(fd, &buffer, 1, offset); +} + +std::optional ReadArchivedIovecs(int fd, const void* iov, int iovcnt, const std::int64_t* offset) { + if (!IsArchivedFile(fd)) return std::nullopt; + if (offset != nullptr && *offset < 0) return -EINVAL; + std::uint64_t read = 0; + if (!AppArchiveDescriptorRead_nid_no_patch(fd, static_cast(iov), iovcnt, offset, &read)) return std::nullopt; + return static_cast(read); +} + +std::optional SeekArchivedFile(int fd, std::int64_t offset, int whence) { + std::int64_t position = 0; + if (!AppArchiveDescriptorSeek_nid_no_patch(fd, offset, whence, &position)) return std::nullopt; + return position < 0 ? -EINVAL : position; +} + +std::optional ArchivedFileRemaining(int fd) { + AppArchiveNode node{}; + std::uint64_t position = 0; + if (!AppArchiveDescriptor_nid_no_patch(fd, &node, &position)) return std::nullopt; + return position >= node.Size ? 0 : static_cast(node.Size - position); +} + +bool FillArchivedStat(const std::filesystem::path& native, FileStat* sb) { + if (!AppArchiveOnly_nid_no_patch(native)) return false; + AppArchiveNode node{}; + if (!AppArchiveLookup_nid_no_patch(native, &node)) throw std::logic_error("FillArchivedStat: " + native.string() + " left app0.zar while reading its status"); + FillNode(node, sb); + return true; +} + +bool FillArchivedStatFromDescriptor(int fd, FileStat* sb) { + AppArchiveNode node{}; + if (!AppArchiveDescriptor_nid_no_patch(fd, &node, nullptr)) return false; + FillNode(node, sb); + return true; +} + +void ShareArchivedFile(int from, int to) { + AppArchiveShare_nid_no_patch(from, to); +} + +void ForgetArchivedFile(int fd) { + AppArchiveDetach_nid_no_patch(fd); +} + +} diff --git a/core/libs/prx/libkernel/File/src/DescriptorControl.cpp b/core/libs/prx/libkernel/File/src/DescriptorControl.cpp index 3e7f3c0644..791af5ad09 100644 --- a/core/libs/prx/libkernel/File/src/DescriptorControl.cpp +++ b/core/libs/prx/libkernel/File/src/DescriptorControl.cpp @@ -2,10 +2,12 @@ #include "prx/libkernel/File/include/DirectoryDescriptor.hpp" #include "prx/libkernel/File/include/File.hpp" #include "prx/libkernel/File/include/RandomDevice.hpp" +#include "prx/libkernel/File/include/ArchivedFile.hpp" #include "prx/libc/include/GuestArena.hpp" #include "prx/libc/include/FileLock.hpp" #include "prx/libc/include/general/VabiMacros.hpp" +#include #include #include #include @@ -105,7 +107,7 @@ static_assert(sizeof(GuestLock) == 32); static_assert(offsetof(GuestLock, type) == 20); constexpr std::size_t OldGuestLockSize = 24; -enum class Kind { File, Directory, Pipe, Device, RandomDevice }; +enum class Kind { File, Directory, Pipe, Device, RandomDevice, ArchivedFile }; struct Entry { bool closeOnExec = false; @@ -127,6 +129,7 @@ const char* KindName(Kind kind) { case Kind::Pipe: return "pipe"; case Kind::Device: return "character device"; case Kind::RandomDevice: return "random device"; + case Kind::ArchivedFile: return "file read from app0.zar"; } return "descriptor"; } @@ -140,6 +143,7 @@ std::optional Classify(int fd, const char* function) { if (fd < 0 || !DescriptorIsOpen_nid_no_patch(fd)) return std::nullopt; if (File::DirectoryDescriptorPath(fd)) return Kind::Directory; if (File::IsRandomDevice(fd)) return Kind::RandomDevice; + if (File::IsArchivedFile(fd)) return Kind::ArchivedFile; switch (::GetFileType(WindowsDescriptorHandle_nid_no_patch(fd))) { case FILE_TYPE_DISK: return Kind::File; case FILE_TYPE_PIPE: return Kind::Pipe; @@ -259,6 +263,7 @@ std::optional Classify(int fd, const char* function) { throw std::runtime_error(std::string(function) + ": fstat failed, fd=" + std::to_string(fd) + ", errno=" + std::to_string(errno)); } if (File::IsRandomDevice(fd)) return Kind::RandomDevice; + if (File::IsArchivedFile(fd)) return Kind::ArchivedFile; if (S_ISREG(status.st_mode)) return Kind::File; if (S_ISDIR(status.st_mode)) return Kind::Directory; if (S_ISFIFO(status.st_mode)) return Kind::Pipe; @@ -409,7 +414,7 @@ int RecordLock(int fd, int command, std::intptr_t argument) { int ReadAheadControl(int fd, int command, std::intptr_t argument) { const auto kind = Classify(fd, "fcntl"); if (!kind || *kind == Kind::Pipe) return Fail(GuestEbadf); - if (*kind != Kind::File) return Fail(GuestEnotty); + if (*kind != Kind::File && *kind != Kind::ArchivedFile) return Fail(GuestEnotty); std::lock_guard lock(g_mutex); auto& status = *Materialize(fd).status; if (command == DarwinReadAhead || argument >= 0) status |= ReadAheadFlag; @@ -521,8 +526,9 @@ int Ioctl(int fd, std::uint64_t request, void* argument) { } if (command == ReadableRequest) { if (*kind == Kind::RandomDevice) return Fail(GuestEnotty); - if (*kind != Kind::File && *kind != Kind::Pipe) Unsupported("ioctl", "FIONREAD", *kind, fd); - const int available = Readable(fd, *kind, "ioctl"); + if (*kind != Kind::File && *kind != Kind::Pipe && *kind != Kind::ArchivedFile) Unsupported("ioctl", "FIONREAD", *kind, fd); + const auto archived = File::ArchivedFileRemaining(fd); + const int available = archived ? static_cast(std::min(*archived, std::numeric_limits::max())) : Readable(fd, *kind, "ioctl"); if (argument == nullptr) return Fail(GuestEfault); const GuestArena::HostWrite destination(argument, sizeof(available)); if (!destination.Open()) return Fail(GuestEfault); @@ -544,6 +550,7 @@ void RememberDescriptor(int fd, int openFlags) { } void ShareDescriptor(int from, int to) { + File::ShareArchivedFile(from, to); std::lock_guard lock(g_mutex); auto status = Materialize(from).status; g_entries[to] = Entry{false, std::move(status)}; @@ -554,8 +561,15 @@ void ShareDescriptor(int from, int to) { } void ForgetDescriptor(int fd) { + File::ForgetArchivedFile(fd); std::lock_guard lock(g_mutex); g_entries.erase(fd); } +bool Appends(int fd) { + std::lock_guard lock(g_mutex); + const auto found = g_entries.find(fd); + return found != g_entries.end() && found->second.status && (*found->second.status & Append) != 0; +} + } diff --git a/core/libs/prx/libkernel/File/src/NativeStat.cpp b/core/libs/prx/libkernel/File/src/NativeStat.cpp index 9d097eec22..b110e980df 100644 --- a/core/libs/prx/libkernel/File/src/NativeStat.cpp +++ b/core/libs/prx/libkernel/File/src/NativeStat.cpp @@ -1,5 +1,6 @@ #include "prx/libkernel/File/include/NativeStat.hpp" #include "prx/libkernel/File/include/DirectoryDescriptor.hpp" +#include "prx/libkernel/File/include/ArchivedFile.hpp" #include "prx/libc/include/General.hpp" #include @@ -156,6 +157,7 @@ void FillFileStat(int nativeDescriptor, FileStat* sb) { } bool FillFileStatFromDescriptor(int fd, FileStat* sb) { + if (FillArchivedStatFromDescriptor(fd, sb)) return true; NativeStat st{}; if (DoFstat(fd, &st) != 0) return false; CopyNativeStat(st, sb); diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp index 9760bbe879..c68f36535c 100644 --- a/core/libs/prx/libkernel/File/src/Open.cpp +++ b/core/libs/prx/libkernel/File/src/Open.cpp @@ -12,6 +12,7 @@ #include "prx/libkernel/Socket/include/SocketRuntime.hpp" #include "prx/libkernel/File/include/RandomDevice.hpp" #include "prx/libkernel/File/include/HostStreams.hpp" +#include "prx/libkernel/File/include/ArchivedFile.hpp" #include "prx/libkernel/KernelErrors.hpp" #include "prx/libkernel/Equeue/Equeue.hpp" #include "SceTypes.hpp" @@ -73,6 +74,10 @@ static int NativeWrite(int fd, const void* buf, std::size_t n) { const auto positioned = File::LockPositionedTransfer(fd); const auto shared = File::WriteSharedMemory_nid_no_patch(fd, buf, n); if (shared != -2) return static_cast(shared); + if (File::Appends(fd)) { + const auto appended = File::WriteAtEnd(fd, buf, n); + if (appended != -2) return static_cast(appended); + } const int result = ::_write(fd, buf, static_cast(n)); if (result < 0 && errno == ENOSPC && _doserrno == 0 && ::GetFileType(reinterpret_cast(::_get_osfhandle(fd))) == FILE_TYPE_PIPE) errno = EAGAIN; return result; @@ -174,6 +179,12 @@ static int SceIoErrorFromErrno(int error) { } static int OpenNative(const std::filesystem::path& native, int flags, std::uint16_t mode) { + if (const auto archived = File::OpenArchivedFile(native, flags)) { + if (*archived < 0) return SceErrorFromErrno(-*archived); + File::RememberDescriptor(*archived, flags); + return *archived; + } + File::RefuseMergedDirectory(native); #ifdef _WIN32 if ((flags & (SCE_KERNEL_O_DIRECTORY | SCE_KERNEL_O_CREAT)) == SCE_KERNEL_O_DIRECTORY) { std::error_code error; @@ -222,7 +233,7 @@ int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) { } std::filesystem::path native; const bool module = (flags & (SCE_KERNEL_O_ACCMODE | SCE_KERNEL_O_CREAT)) == SCE_KERNEL_O_RDONLY; - if (const int error = ResolvePathError_nid_no_patch(path, native, module)) return SceKernelError(error); + if (const int error = ResolvePathError_nid_no_patch(path, native, module, true)) return SceKernelError(error); return OpenNative(native, flags, mode); } @@ -262,6 +273,7 @@ std::int64_t APS5_VABI sceKernelRead(int d, void* buf, std::size_t nbytes) { const GuestArena::HostWrite destination(buf, nbytes); if (!destination.Open()) errno = EFAULT; else if (File::ReadRandomDevice(d, buf, nbytes)) return static_cast(nbytes); + else if (const auto archived = File::ReadArchivedFile(d, buf, nbytes, nullptr)) return *archived; auto n = destination.Open() ? NativeRead(d, buf, nbytes) : -1; if (n < 0) return SceIoErrorFromErrno(errno); return static_cast(n); @@ -285,6 +297,7 @@ std::int64_t APS5_VABI sceKernelLseek(int d, std::int64_t offset, int whence) { if (whence < 0 || whence > 2) { throw std::invalid_argument(std::string(__func__) + ": invalid whence=" + std::to_string(whence)); } + if (const auto archived = File::SeekArchivedFile(d, offset, whence)) return *archived < 0 ? SceIoErrorFromErrno(-*archived) : *archived; #ifdef _WIN32 if (const auto directory = File::SeekDirectoryDescriptor(d, offset, whence)) { if (*directory < 0) return SceIoErrorFromErrno(errno); @@ -302,10 +315,17 @@ int APS5_VABI sceKernelStat(const char* path, FileStat* sb) { } if (*path == '\0') return SceErrorFromErrno(2); std::filesystem::path native; - if (const int error = ResolvePathError_nid_no_patch(path, native, true)) return SceKernelError(error); + if (const int error = ResolvePathError_nid_no_patch(path, native, true, true)) return SceKernelError(error); std::error_code error; constexpr int GuestEnotdir = 20; constexpr int GuestEfault = 14; + FileStat archived{}; + if (File::FillArchivedStat(native, &archived)) { + if (std::string_view(path).back() == '/' && (archived.st_mode & 0170000) != 0040000) return SceErrorFromErrno(GuestEnotdir); + if (sb == nullptr) return SceErrorFromErrno(GuestEfault); + *sb = archived; + return 0; + } const auto status = std::filesystem::status(native, error); if (error == std::errc::not_a_directory) return SceErrorFromErrno(GuestEnotdir); if (!std::filesystem::exists(status)) { diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 5ded00a7ba..afbe5aa517 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -17,6 +17,8 @@ #include "prx/libkernel/File/include/FileLock.hpp" #include "prx/libkernel/File/include/RandomDevice.hpp" #include "prx/libkernel/File/include/HostStreams.hpp" +#include "prx/libkernel/File/include/ArchivedFile.hpp" +#include "prx/libc/include/AppArchive.hpp" #include "prx/libkernel/KernelErrors.hpp" #include "prx/libkernel/Equeue/Equeue.hpp" #include "prx/libkernel/Socket/include/SocketRuntime.hpp" @@ -41,6 +43,7 @@ struct KernelIovec { std::size_t length; }; static_assert(sizeof(KernelIovec) == sizeof(GuestSockets::Iovec)); +static_assert(sizeof(KernelIovec) == sizeof(AppArchiveBuffer) && offsetof(KernelIovec, length) == offsetof(AppArchiveBuffer, Length)); static constexpr int KERNEL_IOV_MAX = 1024; @@ -237,6 +240,27 @@ std::int64_t NativePositioned_nid_no_patch(int descriptor, void* buf, std::size_ const auto positioned = File::LockPositionedTransfer(descriptor); return NativePositionedTransfer(descriptor, buf, nbytes, offset, write); } +std::int64_t File::WriteAtEnd(int descriptor, const void* buf, std::size_t nbytes) { + if (nbytes > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("WriteAtEnd: nbytes exceeds platform limit"); + } + const auto handle = reinterpret_cast(WithoutParameterHandler([descriptor] { return ::_get_osfhandle(descriptor); })); + if (handle == INVALID_HANDLE_VALUE) { + errno = EBADF; + return -1; + } + if (::GetFileType(handle) != FILE_TYPE_DISK) return -2; + OVERLAPPED end{}; + end.Offset = 0xFFFFFFFF; + end.OffsetHigh = 0xFFFFFFFF; + DWORD done = 0; + if (!::WriteFile(handle, buf, static_cast(nbytes), &done, &end)) { + const DWORD error = ::GetLastError(); + errno = error == ERROR_ACCESS_DENIED ? EBADF : error == ERROR_DISK_FULL || error == ERROR_HANDLE_DISK_FULL ? ENOSPC : EIO; + return -1; + } + return done; +} static std::int64_t NativeTransferBuffer(int descriptor, void* buf, std::size_t nbytes, bool write) { if (nbytes > static_cast(std::numeric_limits::max())) { throw std::runtime_error("NativeTransfer: nbytes exceeds platform limit"); @@ -244,6 +268,10 @@ static std::int64_t NativeTransferBuffer(int descriptor, void* buf, std::size_t if (write) { const auto result = File::WriteSharedMemory_nid_no_patch(descriptor, buf, nbytes); if (result != -2) return result; + if (File::Appends(descriptor)) { + const auto appended = File::WriteAtEnd(descriptor, buf, nbytes); + if (appended != -2) return appended; + } } return write ? ::_write(descriptor, buf, static_cast(nbytes)) : ::_read(descriptor, buf, static_cast(nbytes)); } @@ -650,6 +678,7 @@ int APS5_VABI dup2_nid_postfix(int from, int to) { if (GuestSockets::DuplicateTo(from, to) < 0) return -1; if (from == to) return to; File::ForgetRandomDevice(to); + File::ForgetArchivedFile(to); #ifdef _WIN32 File::ForgetDirectoryDescriptor(to); File::ForgetFileLock(to); @@ -696,6 +725,7 @@ int APS5_VABI dup2_nid_postfix(int from, int to) { int APS5_VABI flock_nid_postfix(int d, int operation) { if (GuestSockets::IsOpen(d)) return PosixFailure(GUEST_EOPNOTSUPP); + File::RefuseArchivedFile(d, "flock"); const int type = operation & 8 ? 8 : operation & 2 ? 2 : operation & 1 ? 1 : 0; if (type == 0) return PosixFailure(GUEST_EBADF); if (NativeFlock(d, type | (operation & 4)) != 0) { @@ -715,6 +745,7 @@ int APS5_VABI flock_nid_postfix(int d, int operation) { int APS5_VABI _fstatfs_nid_postfix(int d, KernelStatfs* buf) { if (GuestSockets::IsOpen(d)) return PosixFailure(GUEST_EINVAL); if (d < 0) return PosixFailure(GUEST_EBADF); + File::RefuseArchivedFile(d, "fstatfs"); KernelStatfs result{}; result.f_version = KERNEL_STATFS_VERSION; if (const int error = NativeFstatfs(d, &result)) return PosixFailure(error); @@ -741,6 +772,7 @@ int APS5_VABI ftruncate_nid_postfix(int d, int64_t length) { if (length < 0 || GuestSockets::IsOpen(d)) { return PosixFailure(GUEST_EINVAL); } + File::RefuseArchivedFile(d, "ftruncate"); #ifdef _WIN32 int error = NativeFtruncate(d, length); if (error != 0) { @@ -811,6 +843,7 @@ int64_t APS5_VABI pread_nid_postfix(int d, void* buf, size_t nbytes, int64_t off const GuestArena::HostWrite destination(buf, nbytes); if (!destination.Open()) return PosixFailure(GUEST_EFAULT); if (File::ReadRandomDevice(d, buf, nbytes)) return static_cast(nbytes); + if (const auto archived = File::ReadArchivedFile(d, buf, nbytes, &offset)) return *archived; char empty = 0; const auto result = NativePread(d, buf == nullptr ? &empty : buf, nbytes, offset); return result < 0 ? PosixFailure(SceErrorFromErrno(errno) & 0xffff) : result; @@ -881,7 +914,8 @@ int APS5_VABI sceKernelCheckReachability(const char* path) { if (*path == '\0') return SCE_KERNEL_ERROR_ENOENT; if (std::strlen(path) > 255) return SCE_KERNEL_ERROR_ENAMETOOLONG; std::filesystem::path native; - if (const int error = ResolvePathError_nid_no_patch(path, native, true)) return SceKernelError(error); + if (const int error = ResolvePathError_nid_no_patch(path, native, true, true)) return SceKernelError(error); + if (AppArchiveOnly_nid_no_patch(native)) return 0; std::error_code error; return std::filesystem::exists(native, error) ? 0 : SCE_KERNEL_ERROR_ENOENT; } @@ -898,6 +932,7 @@ int APS5_VABI sceKernelFstat(int d, FileStat* sb) { int APS5_VABI sceKernelFsync(int fd) { if (GuestSockets::IsOpen(fd)) return SceErrorFromErrno(GUEST_EINVAL); + File::RefuseArchivedFile(fd, "fsync"); #ifdef _WIN32 if (NativeSync(fd) != 0) return SceErrorFromErrno(errno); #else @@ -908,6 +943,7 @@ int APS5_VABI sceKernelFsync(int fd) { int APS5_VABI sceKernelFdatasync(int fd) { if (GuestSockets::IsOpen(fd)) return SceErrorFromErrno(GUEST_EINVAL); + File::RefuseArchivedFile(fd, "fdatasync"); #ifdef _WIN32 if (NativeSync(fd) != 0) return SceErrorFromErrno(errno); #else @@ -929,6 +965,7 @@ int APS5_VABI sceKernelWriteThrottlingStatus(std::uint64_t* status) { int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* basep) { if (GuestSockets::IsOpen(fd)) return SceErrorFromErrno(GUEST_EINVAL); + File::RefuseArchivedFile(fd, "getdirentries"); if (buf == nullptr) return SceErrorFromErrno(GUEST_EFAULT); if (nbytes <= 0) return SceErrorFromErrno(GUEST_EINVAL); if (basep != nullptr) *basep = 0; @@ -943,6 +980,7 @@ int APS5_VABI sceKernelGetdents(int fd, char* buf, int nbytes) { int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* basep) { if (GuestSockets::IsOpen(fd)) return SceErrorFromErrno(GUEST_EINVAL); + File::RefuseArchivedFile(fd, "getdirentries"); constexpr std::size_t GuestHeaderBytes = 8; constexpr std::size_t GuestMaxName = 255; if (buf == nullptr) return SceErrorFromErrno(GUEST_EFAULT); @@ -1044,6 +1082,7 @@ int64_t APS5_VABI sceKernelPread(int d, void* buf, size_t nbytes, int64_t offset const GuestArena::HostWrite destination(buf, nbytes); if (!destination.Open()) return SceErrorFromErrno(GUEST_EFAULT); if (File::ReadRandomDevice(d, buf, nbytes)) return static_cast(nbytes); + if (const auto archived = File::ReadArchivedFile(d, buf, nbytes, &offset)) return *archived; char empty = 0; const auto result = NativePread(d, nbytes == 0 && buf == nullptr ? &empty : buf, nbytes, offset); return result < 0 ? SceErrorFromErrno(errno) : result; @@ -1079,6 +1118,9 @@ static std::int64_t TransferIovecs(int d, const KernelIovec* iov, int iovcnt, co return result < 0 ? SceKernelError(*__error_nid_postfix()) : result; } if (!write && File::IsRandomDevice(d)) return ReadRandomIovecs(d, iov, iovcnt); + if (!write) { + if (const auto archived = File::ReadArchivedIovecs(d, iov, iovcnt, offset)) return *archived; + } const auto positioned = File::LockPositionedTransfer(d); if (total == 0) { char none = 0; @@ -1086,6 +1128,16 @@ static std::int64_t TransferIovecs(int d, const KernelIovec* iov, int iovcnt, co return result < 0 ? SceErrorFromErrno(errno) : 0; } if (!write && offset == nullptr && File::DirectoryDescriptorPath(d)) return SceErrorFromErrno(GUEST_EISDIR); + if (write && offset == nullptr && iovcnt > 1 && total <= static_cast(std::numeric_limits::max()) && File::Appends(d)) { + std::vector gathered; + gathered.reserve(total); + for (int i = 0; i < iovcnt; ++i) { + const auto* base = static_cast(iov[i].base); + gathered.insert(gathered.end(), base, base + iov[i].length); + } + const auto result = NativeTransferBuffer(d, gathered.data(), gathered.size(), true); + return result < 0 ? SceErrorFromErrno(errno) : result; + } const bool whole = write || offset != nullptr || NativeIsDisk(d); std::int64_t done = 0; for (int i = 0; i < iovcnt; ++i) { @@ -1127,6 +1179,7 @@ int64_t APS5_VABI sceKernelPread(int d, void* buf, size_t nbytes, int64_t offset const GuestArena::HostWrite destination(buf, nbytes); if (!destination.Open()) return SceErrorFromErrno(GUEST_EFAULT); if (File::ReadRandomDevice(d, buf, nbytes)) return static_cast(nbytes); + if (const auto archived = File::ReadArchivedFile(d, buf, nbytes, &offset)) return *archived; const auto result = NativePread(d, buf, nbytes, offset); return result < 0 ? SceErrorFromErrno(errno) : result; } @@ -1148,6 +1201,7 @@ int64_t APS5_VABI sceKernelReadv(int d, const KernelIovec* iov, int iovcnt) { return result < 0 ? SceKernelError(*__error_nid_postfix()) : result; } if (File::IsRandomDevice(d)) return ReadRandomIovecs(d, iov, iovcnt); + if (const auto archived = File::ReadArchivedIovecs(d, iov, iovcnt, nullptr)) return *archived; const auto result = static_cast(::readv(d, NativeIovecs(iov), iovcnt)); return result < 0 ? SceErrorFromErrno(errno) : result; } @@ -1171,6 +1225,7 @@ int64_t APS5_VABI sceKernelPreadv(int d, const KernelIovec* iov, int iovcnt, int std::deque destinations; if (!OpenIovecs(iov, iovcnt, destinations)) return SceErrorFromErrno(GUEST_EFAULT); if (File::IsRandomDevice(d)) return ReadRandomIovecs(d, iov, iovcnt); + if (const auto archived = File::ReadArchivedIovecs(d, iov, iovcnt, &offset)) return *archived; const auto result = static_cast(::preadv(d, NativeIovecs(iov), iovcnt, static_cast(offset))); return result < 0 ? SceErrorFromErrno(errno) : result; } @@ -1257,6 +1312,7 @@ int APS5_VABI sceKernelChmod_nid_postfix(const char* path, std::uint16_t mode) { int APS5_VABI sceKernelFchmod(int d, std::uint16_t mode) { if (GuestSockets::IsOpen(d)) return SceErrorFromErrno(GUEST_EINVAL); + File::RefuseArchivedFile(d, "fchmod"); if (NativeFchmod(d, mode & 07777) != 0) return SceErrorFromErrno(errno); return 0; } @@ -1319,6 +1375,7 @@ int APS5_VABI lutimes_nid_postfix(const char* path, const KernelTimeval* times) int APS5_VABI sceKernelFutimes(int d, const KernelTimeval* times) { if (GuestSockets::IsOpen(d)) return SceErrorFromErrno(GUEST_EINVAL); + File::RefuseArchivedFile(d, "futimes"); if (times != nullptr) { for (int i = 0; i < 2; ++i) { if (times[i].tv_usec < 0 || times[i].tv_usec >= 1000000) return SceErrorFromErrno(GUEST_EINVAL); diff --git a/core/libs/prx/libkernel/Socket/include/SocketConnect.hpp b/core/libs/prx/libkernel/Socket/include/SocketConnect.hpp new file mode 100644 index 0000000000..51ebf73520 --- /dev/null +++ b/core/libs/prx/libkernel/Socket/include/SocketConnect.hpp @@ -0,0 +1,6 @@ +#pragma once +#ifdef _WIN32 +#include + +extern "C" void RefuseClosedLoopbackAtOnce_nid_no_patch(SOCKET socket, const sockaddr* address); +#endif diff --git a/core/libs/prx/libkernel/Socket/src/SocketConnect.cpp b/core/libs/prx/libkernel/Socket/src/SocketConnect.cpp new file mode 100644 index 0000000000..245fd5e527 --- /dev/null +++ b/core/libs/prx/libkernel/Socket/src/SocketConnect.cpp @@ -0,0 +1,47 @@ +#ifdef _WIN32 +#include "prx/libkernel/Socket/include/SocketConnect.hpp" +#include +#include +#include +#include + +namespace { + +bool IsLoopback(const sockaddr* address) { + if (address->sa_family == AF_INET) return (ntohl(reinterpret_cast(address)->sin_addr.s_addr) >> 24) == 127; + if (address->sa_family == AF_INET6) return IN6_IS_ADDR_LOOPBACK(&reinterpret_cast(address)->sin6_addr); + return false; +} + +u_short Port(const sockaddr* address) { + return address->sa_family == AF_INET ? reinterpret_cast(address)->sin_port : reinterpret_cast(address)->sin6_port; +} + +template +bool ListenerOnPort(ULONG family, u_short port) { + std::vector table; + ULONG size = 0; + DWORD result = ERROR_INSUFFICIENT_BUFFER; + while (result == ERROR_INSUFFICIENT_BUFFER) { + table.resize(size); + result = GetExtendedTcpTable(table.data(), &size, FALSE, family, TCP_TABLE_OWNER_PID_LISTENER, 0); + } + if (result != NO_ERROR) return true; + const auto* rows = reinterpret_cast(table.data()); + for (DWORD i = 0; i < rows->dwNumEntries; ++i) { + if (static_cast(rows->table[i].dwLocalPort) == port) return true; + } + return false; +} + +} + +extern "C" void RefuseClosedLoopbackAtOnce_nid_no_patch(SOCKET socket, const sockaddr* address) { + if (!IsLoopback(address)) return; + const u_short port = Port(address); + if (ListenerOnPort(AF_INET, port) || ListenerOnPort(AF_INET6, port)) return; + TCP_INITIAL_RTO_PARAMETERS parameters{TCP_INITIAL_RTO_UNSPECIFIED_RTT, TCP_INITIAL_RTO_NO_SYN_RETRANSMISSIONS}; + DWORD returned = 0; + WSAIoctl(socket, SIO_TCP_INITIAL_RTO, ¶meters, sizeof(parameters), nullptr, 0, &returned, nullptr, nullptr); +} +#endif diff --git a/core/libs/prx/libkernel/Socket/src/Udp.cpp b/core/libs/prx/libkernel/Socket/src/Udp.cpp index 7588023bb7..f45edc6aa5 100644 --- a/core/libs/prx/libkernel/Socket/src/Udp.cpp +++ b/core/libs/prx/libkernel/Socket/src/Udp.cpp @@ -28,6 +28,7 @@ #include "prx/libkernel/File/include/File.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" #include "prx/libkernel/Socket/include/SocketPoll.hpp" +#include "prx/libkernel/Socket/include/SocketConnect.hpp" #include #include #include @@ -956,6 +957,9 @@ int APS5_VABI connect_nid_postfix(int descriptor, const void* address, std::uint sockaddr_storage native{}; socklen_t size; if (!Address(address, length, native, size)) return -1; +#ifdef _WIN32 + if (socket->type == 1) RefuseClosedLoopbackAtOnce_nid_no_patch(socket->value, reinterpret_cast(&native)); +#endif if (native.ss_family != (socket->family == 2 ? AF_INET : AF_INET6)) return Fail(47); { std::lock_guard lock(socket->modeMutex); diff --git a/core/libs/tests/AppArchiveFixture.hpp b/core/libs/tests/AppArchiveFixture.hpp new file mode 100644 index 0000000000..b6c5e4529c --- /dev/null +++ b/core/libs/tests/AppArchiveFixture.hpp @@ -0,0 +1,53 @@ +#ifndef CORE_LIBS_TESTS_APPARCHIVEFIXTURE_HPP +#define CORE_LIBS_TESTS_APPARCHIVEFIXTURE_HPP + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace AppArchiveFixture { + +inline std::vector Pattern(std::size_t size) { + std::vector data(size); + for (std::size_t i = 0; i < size; ++i) data[i] = static_cast((i * 7 + i / 4096) & 0xff); + return data; +} + +inline std::filesystem::path EnterFreshRoot(const char* name) { + const auto root = std::filesystem::temp_directory_path() / + (std::string(name) + "-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + std::filesystem::create_directories(root / "app0" / "sce_sys"); + std::filesystem::current_path(root); + return std::filesystem::canonical(root); +} + +inline void WriteArchive(const std::filesystem::path& path, const std::map>& files, const std::vector& directories = {}) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output) std::abort(); + ZArchiveWriter writer( + [](std::int32_t, void*) {}, + [](const void* data, std::size_t length, void* context) { + static_cast(context)->write(static_cast(data), static_cast(length)); + }, + &output); + for (const auto& directory : directories) if (!writer.MakeDir(directory.c_str(), true)) std::abort(); + for (const auto& [name, data] : files) { + const auto slash = name.find_last_of('/'); + if (slash != std::string::npos) writer.MakeDir(name.substr(0, slash).c_str(), true); + if (!writer.StartNewFile(name.c_str())) std::abort(); + if (!data.empty()) writer.AppendData(data.data(), data.size()); + } + writer.Finalize(); + output.close(); + if (!output) std::abort(); +} + +} + +#endif diff --git a/core/libs/tests/GuestAppArchive.cpp b/core/libs/tests/GuestAppArchive.cpp new file mode 100644 index 0000000000..1f865808a5 --- /dev/null +++ b/core/libs/tests/GuestAppArchive.cpp @@ -0,0 +1,178 @@ +#include "AppArchiveFixture.hpp" +#include "prx/libc/include/AppArchive.hpp" +#include "prx/libc/include/General.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) { + std::cerr << message << '\n'; + std::exit(1); + } +} + +bool Lookup(const char* guestPath, AppArchiveNode& node) { + return AppArchiveLookup_nid_no_patch(ResolveArchivedPath_nid_no_patch(guestPath), &node); +} + +bool GuardRefuses(const char* guestPath) { + try { + (void)ResolvePath_nid_no_patch(guestPath); + } catch (const std::runtime_error& error) { + return std::strstr(error.what(), "not implemented") != nullptr && std::strstr(error.what(), guestPath) != nullptr; + } + return false; +} + +constexpr std::size_t BigSize = 200 * 1024 + 123; + +std::map> Contents() { + return { + {"Data/Big.bin", AppArchiveFixture::Pattern(BigSize)}, + {"Data/Empty.txt", {}}, + {"Data/Sub/Deep.txt", {'d', 'e', 'e', 'p'}}, + {"Readme.TXT", {'r', 'e', 'a', 'd'}}, + }; +} + +void Present(const std::filesystem::path& root) { + AppArchiveFixture::WriteArchive(root / "app0.zar", Contents(), {"Data/Unused"}); + { std::ofstream eboot(root / "app0" / "eboot.bin"); eboot << "elf"; } + Require(AppArchivePath_nid_no_patch() == root / "app0.zar", "the archive beside app0 was not opened"); + + AppArchiveNode node{}; + Require(Lookup("/app0/data/BIG.BIN", node) && !node.Directory && node.Size == BigSize, "a file was not found case-insensitively with its size"); + const auto big = node.Handle; + Require(Lookup("/APP0/Data/Empty.txt", node) && !node.Directory && node.Size == 0, "an empty file was not found"); + Require(Lookup("/app0/Data", node) && node.Directory, "a directory was not found"); + const auto data = node.Handle; + Require(Lookup("/app0/", node) && node.Directory, "the archive root was not app0"); + Require(Lookup("/app0/Data/Sub/../Sub/deep.txt", node) && node.Size == 4, "a normalised path was not found"); + Require(!Lookup("/app0/Data/Missing.bin", node), "a missing file was found"); + Require(!Lookup("/app0/Readme.TXT/x", node), "a path through a file was found"); + Require(!Lookup("/app0x/Readme.TXT", node), "a sibling of app0 was looked up in the archive"); + Require(!Lookup("/Readme.TXT", node), "a path outside app0 was looked up in the archive"); + + Require(GuardRefuses("/app0/Data/Big.bin"), "an archive-only file resolved through an unaware call"); + Require(GuardRefuses("/app0/Data"), "an archive-only directory resolved through an unaware call"); + Require(ResolvePath_nid_no_patch("/app0/eboot.bin") == root / "app0" / "eboot.bin", "a file on disk was refused"); + Require(ResolvePath_nid_no_patch("/app0/Data/Missing.bin") == root / "app0" / "Data" / "Missing.bin", "a path missing everywhere was refused"); + Require(Lookup("/app0/sce_sys", node) == false && ResolvePath_nid_no_patch("/app0/sce_sys") == root / "app0" / "sce_sys", "a directory only on disk was not left to the disk"); + + bool rejected = false; + const auto expected = AppArchiveFixture::Pattern(BigSize); + std::vector buffer(BigSize + 64, 0); + Require(AppArchiveRead_nid_no_patch(big, 0, buffer.data(), BigSize) == BigSize && std::equal(expected.begin(), expected.end(), buffer.begin()), "the whole file did not read back"); + for (const std::uint64_t offset : {std::uint64_t{1}, std::uint64_t{65535}, std::uint64_t{65536}, std::uint64_t{131071}, std::uint64_t{BigSize - 5}}) { + std::fill(buffer.begin(), buffer.end(), 0); + const auto read = AppArchiveRead_nid_no_patch(big, offset, buffer.data(), 70000); + const auto wanted = std::min(70000, BigSize - offset); + Require(read == wanted && std::equal(expected.begin() + static_cast(offset), expected.begin() + static_cast(offset + wanted), buffer.begin()), "a read across block boundaries returned wrong bytes"); + } + Require(AppArchiveRead_nid_no_patch(big, BigSize, buffer.data(), 10) == 0, "a read at the end returned bytes"); + Require(AppArchiveRead_nid_no_patch(big, 10, buffer.data(), 0) == 0, "a zero-length read returned bytes"); + + AppArchiveNode bigNode{}; + Require(Lookup("/app0/Data/Big.bin", bigNode), "the big file vanished"); + constexpr int Descriptor = 1000003; + constexpr int Shared = 1000004; + Require(!AppArchiveDescriptor_nid_no_patch(Descriptor, nullptr, nullptr), "a descriptor was attached before attaching"); + AppArchiveAttach_nid_no_patch(Descriptor, bigNode); + AppArchiveShare_nid_no_patch(Descriptor, Shared); + std::uint64_t done = 0; + std::vector first(65530), second(20); + const AppArchiveBuffer split[] = {{first.data(), first.size()}, {second.data(), second.size()}}; + Require(AppArchiveDescriptorRead_nid_no_patch(Descriptor, split, 2, nullptr, &done) && done == first.size() + second.size(), "a vector read returned the wrong length"); + Require(std::equal(first.begin(), first.end(), expected.begin()) && std::equal(second.begin(), second.end(), expected.begin() + 65530), "a vector read returned wrong bytes"); + std::uint64_t position = 0; + Require(AppArchiveDescriptor_nid_no_patch(Shared, nullptr, &position) && position == 65550, "a shared descriptor did not share the position"); + const std::int64_t at = 131000; + const AppArchiveBuffer single[] = {{second.data(), second.size()}}; + Require(AppArchiveDescriptorRead_nid_no_patch(Shared, single, 1, &at, &done) && done == 20 && std::equal(second.begin(), second.end(), expected.begin() + at), "a positioned read returned wrong bytes"); + Require(AppArchiveDescriptor_nid_no_patch(Descriptor, nullptr, &position) && position == 65550, "a positioned read moved the position"); + std::int64_t sought = 0; + Require(AppArchiveDescriptorSeek_nid_no_patch(Shared, -3, 2, &sought) && sought == static_cast(BigSize - 3), "seeking from the end failed"); + Require(AppArchiveDescriptorRead_nid_no_patch(Descriptor, single, 1, nullptr, &done) && done == 3, "a read at the end was not short"); + Require(AppArchiveDescriptorRead_nid_no_patch(Descriptor, single, 1, nullptr, &done) && done == 0, "a read past the end returned bytes"); + Require(AppArchiveDescriptorSeek_nid_no_patch(Descriptor, -1, 0, &sought) && sought == -1, "a negative seek was accepted"); + Require(AppArchiveDescriptorSeek_nid_no_patch(Descriptor, 10, 1, &sought) && sought == static_cast(BigSize + 10), "seeking past the end failed"); + AppArchiveDetach_nid_no_patch(Descriptor); + Require(!AppArchiveDescriptor_nid_no_patch(Descriptor, nullptr, nullptr) && AppArchiveDescriptor_nid_no_patch(Shared, nullptr, nullptr), "detaching one descriptor detached its duplicate"); + AppArchiveShare_nid_no_patch(Descriptor, Shared); + Require(!AppArchiveDescriptor_nid_no_patch(Shared, nullptr, nullptr), "duplicating a plain descriptor over an archived one kept the archive"); + rejected = false; + try { AppArchiveAttach_nid_no_patch(Descriptor, AppArchiveNode{0, true, 0}); } catch (const std::invalid_argument&) { rejected = true; } + Require(rejected, "a directory was attached to a descriptor"); + + std::set> names; + for (const auto& entry : AppArchiveList_nid_no_patch(data)) names.insert({entry.Name, entry.Directory}); + Require(names == std::set>{{"Big.bin", false}, {"Empty.txt", false}, {"Sub", true}, {"Unused", true}}, "the directory listing is wrong"); + + rejected = false; + try { (void)AppArchiveList_nid_no_patch(big); } catch (const std::invalid_argument&) { rejected = true; } + Require(rejected, "a file was listed as a directory"); + rejected = false; + try { (void)AppArchiveRead_nid_no_patch(data, 0, buffer.data(), 1); } catch (const std::invalid_argument&) { rejected = true; } + Require(rejected, "a directory was read as a file"); +} + +void Absent() { + AppArchiveNode node{}; + Require(AppArchivePath_nid_no_patch().empty(), "an archive was reported without app0.zar"); + Require(!Lookup("/app0/Readme.TXT", node), "a path was found without an archive"); + Require(ResolvePath_nid_no_patch("/app0/Readme.TXT").filename() == "Readme.TXT", "resolution changed without an archive"); +} + +void Damaged(const std::filesystem::path& root) { + AppArchiveFixture::WriteArchive(root / "app0.zar", Contents()); + { + std::fstream archive(root / "app0.zar", std::ios::binary | std::ios::in | std::ios::out); + const char garbage[8] = {0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x00}; + archive.seekp(0); + archive.write(garbage, sizeof(garbage)); + } + AppArchiveNode node{}; + Require(Lookup("/app0/Data/Big.bin", node), "the damaged archive's index was not read"); + std::vector buffer(16); + bool thrown = false; + try { (void)AppArchiveRead_nid_no_patch(node.Handle, 0, buffer.data(), buffer.size()); } catch (const std::runtime_error& error) { thrown = std::strstr(error.what(), "damaged") != nullptr; } + Require(thrown, "a damaged block did not fail the read"); +} + +void Invalid(const std::filesystem::path& root) { + { std::ofstream archive(root / "app0.zar", std::ios::binary); archive << "not an archive"; } + AppArchiveNode node{}; + bool thrown = false; + try { (void)Lookup("/app0/Readme.TXT", node); } catch (const std::runtime_error& error) { thrown = std::strstr(error.what(), "app0.zar") != nullptr; } + Require(thrown, "an invalid app0.zar was not reported"); +} + +} + +int main(int argc, char** argv) { + if (argc != 2) { + std::cerr << "usage: guest_app_archive_tests present|absent|damaged|invalid\n"; + return 2; + } + const std::string mode = argv[1]; + const auto root = AppArchiveFixture::EnterFreshRoot("anyps5-app-archive"); + if (mode == "present") Present(root); + else if (mode == "absent") Absent(); + else if (mode == "damaged") Damaged(root); + else if (mode == "invalid") Invalid(root); + else return 2; + std::error_code error; + std::filesystem::current_path(std::filesystem::temp_directory_path(), error); + std::filesystem::remove_all(root, error); + return 0; +} diff --git a/core/libs/tests/GuestAppendWrites.cpp b/core/libs/tests/GuestAppendWrites.cpp new file mode 100644 index 0000000000..0780e303eb --- /dev/null +++ b/core/libs/tests/GuestAppendWrites.cpp @@ -0,0 +1,110 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct GuestIovec { + const void* base; + std::size_t length; +}; + +extern "C" { +int APS5_VABI sceKernelOpen(const char*, int, std::uint16_t); +int APS5_VABI sceKernelClose(int); +std::int64_t APS5_VABI sceKernelWrite(int, const void*, std::size_t); +std::int64_t APS5_VABI sceKernelLseek(int, std::int64_t, int); +std::int64_t APS5_VABI write_nid_postfix(int, const char*, std::int64_t); +std::int64_t APS5_VABI writev_nid_postfix(int, const GuestIovec*, int); +int APS5_VABI dup_nid_postfix(int); +} + +static void Check(bool condition, int line) { + if (!condition) { + std::fprintf(stderr, "append check failed at line %d\n", line); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) + +static constexpr const char* Path = "guest_append_writes.txt"; +static constexpr int CreateWriteAppend = 0x0209; +static constexpr int Truncate = 0x0400; +static constexpr int FromStart = 0; +static constexpr int FromCurrent = 1; +static constexpr int Writers = 4; +static constexpr int LinesPerWriter = 1500; +static constexpr std::size_t LineLength = 32; + +static std::string Line(int writer, int index) { + std::string line = "w" + std::to_string(writer) + " " + std::to_string(index) + " "; + line.resize(LineLength - 1, '.'); + return line + "\n"; +} + +static void WriteLines(int fd, int writer) { + for (int index = 0; index < LinesPerWriter; ++index) { + const std::string line = Line(writer, index); + std::int64_t written = -1; + if (writer == 0 || writer == 2) { + written = sceKernelWrite(fd, line.data(), line.size()); + } else if (writer == 1) { + written = write_nid_postfix(fd, line.data(), static_cast(line.size())); + } else { + const GuestIovec parts[2]{{line.data(), 7}, {line.data() + 7, line.size() - 7}}; + written = writev_nid_postfix(fd, parts, 2); + } + Require(written == static_cast(LineLength)); + } +} + +static std::string Contents() { + std::ifstream file(Path, std::ios::binary); + return std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); +} + +int main() { + const int first = sceKernelOpen(Path, CreateWriteAppend | Truncate, 0644); + Require(first >= 0); + const int second = sceKernelOpen(Path, CreateWriteAppend, 0644); + const int duplicate = dup_nid_postfix(first); + const int vectored = sceKernelOpen(Path, CreateWriteAppend, 0644); + Require(second >= 0 && duplicate >= 0 && vectored >= 0); + + const int descriptors[Writers]{first, second, duplicate, vectored}; + std::vector threads; + for (int writer = 0; writer < Writers; ++writer) threads.emplace_back(WriteLines, descriptors[writer], writer); + for (auto& thread : threads) thread.join(); + + const std::string contents = Contents(); + Require(contents.size() == static_cast(Writers) * LinesPerWriter * LineLength); + std::set> seen; + for (std::size_t offset = 0; offset < contents.size(); offset += LineLength) { + const std::string line = contents.substr(offset, LineLength); + int writer = -1; + int index = -1; + Require(std::sscanf(line.c_str(), "w%d %d ", &writer, &index) == 2); + Require(writer >= 0 && writer < Writers && index >= 0 && index < LinesPerWriter); + Require(line == Line(writer, index)); + Require(seen.emplace(writer, index).second); + } + Require(seen.size() == static_cast(Writers) * LinesPerWriter); + + const auto size = static_cast(contents.size()); + Require(sceKernelLseek(first, 0, FromStart) == 0); + const std::string tail = Line(9, 0); + Require(sceKernelWrite(first, tail.data(), tail.size()) == static_cast(LineLength)); + Require(sceKernelLseek(first, 0, FromCurrent) == size + static_cast(LineLength)); + Require(Contents().substr(static_cast(size)) == tail); + + for (const int fd : descriptors) Require(sceKernelClose(fd) == 0); + std::remove(Path); +} diff --git a/core/libs/tests/GuestConvertKeycode.cpp b/core/libs/tests/GuestConvertKeycode.cpp index ec8968222b..7e4b8c1cc4 100644 --- a/core/libs/tests/GuestConvertKeycode.cpp +++ b/core/libs/tests/GuestConvertKeycode.cpp @@ -36,6 +36,10 @@ int main() { Require(character[0] == u'1'); Require(sceConvertKeycodeGetCharacter(0x3A, state, 0, character) == 0); Require(character[0] == 0); + Require(sceConvertKeycodeGetCharacter(0x64, state, 0, character) == 0); + Require(character[0] == u'\\'); + Require(sceConvertKeycodeGetCharacter(0x32, state, 0, character) == 0); + Require(character[0] == 0); } return 0; } diff --git a/core/libs/tests/GuestEnvironment.cpp b/core/libs/tests/GuestEnvironment.cpp index 7bd0c7fb88..46e2b2b11b 100644 --- a/core/libs/tests/GuestEnvironment.cpp +++ b/core/libs/tests/GuestEnvironment.cpp @@ -13,18 +13,25 @@ static void Require(bool value) { if (!value) std::abort(); } int main() { const char* key = "ANYPS5_GUEST_ENV_TEST_4C27"; #ifdef _WIN32 - Require(_putenv_s(key, "inherited") == 0); + Require(_putenv_s(key, "host") == 0); #else - Require(::setenv(key, "inherited", 1) == 0); + Require(::setenv(key, "host", 1) == 0); #endif - Require(std::strcmp(getenv_nid_postfix(key), "inherited") == 0); + Require(getenv_nid_postfix(key) == nullptr); +#ifdef _WIN32 + Require(_putenv_s("HOME", "C:\\Users\\host") == 0); +#else + Require(::setenv("HOME", "/home/host", 1) == 0); +#endif + for (const char* host : {"HOME", "PATH"}) Require(std::getenv(host) != nullptr && getenv_nid_postfix(host) == nullptr); + Require(setenv_nid_postfix(key, "inherited", 0) == 0); Require(setenv_nid_postfix(key, "kept out", 0) == 0); Require(std::strcmp(getenv_nid_postfix(key), "inherited") == 0); char value[] = "copied"; Require(setenv_nid_postfix(key, value, 1) == 0); value[0] = 'X'; Require(std::strcmp(getenv_nid_postfix(key), "copied") == 0); - Require(std::strcmp(std::getenv(key), "inherited") == 0); + Require(std::strcmp(std::getenv(key), "host") == 0); Require(setenv_nid_postfix(key, "", 1) == 0); Require(getenv_nid_postfix(key) && *getenv_nid_postfix(key) == 0); char borrowed[] = "ANYPS5_GUEST_ENV_TEST_4C27=one"; diff --git a/core/libs/tests/GuestKernelAppArchive.cpp b/core/libs/tests/GuestKernelAppArchive.cpp new file mode 100644 index 0000000000..2eb8aae01f --- /dev/null +++ b/core/libs/tests/GuestKernelAppArchive.cpp @@ -0,0 +1,257 @@ +#include "AppArchiveFixture.hpp" +#include "SceTypes.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libkernel/File/include/FileFlags.hpp" +#include "prx/libkernel/KernelErrors.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct KernelIovec { + void* base; + std::size_t length; +}; + +extern "C" { +int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode); +int APS5_VABI sceKernelClose(int d); +std::int64_t APS5_VABI sceKernelRead(int d, void* buf, std::size_t nbytes); +std::int64_t APS5_VABI sceKernelWrite(int d, const void* buf, std::size_t nbytes); +std::int64_t APS5_VABI sceKernelPread(int d, void* buf, std::size_t nbytes, std::int64_t offset); +std::int64_t APS5_VABI sceKernelReadv(int d, const KernelIovec* iov, int iovcnt); +std::int64_t APS5_VABI sceKernelPreadv(int d, const KernelIovec* iov, int iovcnt, std::int64_t offset); +std::int64_t APS5_VABI sceKernelLseek(int d, std::int64_t offset, int whence); +int APS5_VABI sceKernelStat(const char* path, FileStat* sb); +int APS5_VABI sceKernelFstat(int d, FileStat* sb); +int APS5_VABI sceKernelCheckReachability(const char* path); +int APS5_VABI dup_nid_postfix(int d); +int APS5_VABI dup2_nid_postfix(int from, int to); +int APS5_VABI close_nid_postfix(int d); +std::int64_t APS5_VABI pread_nid_postfix(int d, void* buf, std::size_t nbytes, std::int64_t offset); +std::int64_t APS5_VABI lseek_nid_postfix(int d, std::int64_t offset, int whence); +int APS5_VABI fcntl_nid_postfix(int d, int command, ...); +int APS5_VABI ioctl_nid_postfix(int d, std::uint64_t request, void* argument); +void* APS5_VABI opendir_nid_postfix(const char* path); +void* APS5_VABI fdopen_nid_postfix(int d, const char* mode); +int APS5_VABI sceKernelAioSubmitReadCommands(KernelAioRwRequest* req, std::int32_t size, std::int32_t prio, std::int32_t* id); +int APS5_VABI sceKernelAioWaitRequest(std::int32_t id, std::int32_t* state, std::uint32_t* usec); +int APS5_VABI sceKernelAioDeleteRequest(std::int32_t id, std::int32_t* ret); +int APS5_VABI flock_nid_postfix(int d, int operation); +int APS5_VABI _fstatfs_nid_postfix(int d, void* buf); +int APS5_VABI ftruncate_nid_postfix(int d, std::int64_t length); +int APS5_VABI sceKernelFsync(int d); +int APS5_VABI sceKernelFdatasync(int d); +int APS5_VABI sceKernelGetdents(int d, char* buf, int nbytes); +int APS5_VABI sceKernelFchmod(int d, std::uint16_t mode); +int APS5_VABI sceKernelFutimes(int d, const void* times); +void* APS5_VABI mmap_nid_postfix(void* address, std::size_t length, int protection, int flags, int descriptor, std::int64_t offset) noexcept; +void* APS5_VABI __cxa_current_primary_exception_nid_postfix(); +std::type_info* APS5_VABI __cxa_current_exception_type_nid_postfix(); +using TerminateHandler = void (*)(); +TerminateHandler APS5_VABI _ZSt13set_terminatePFvvE_nid_postfix(TerminateHandler); +} + +namespace { + +void Require(bool value, int line) { + if (!value) { + std::fprintf(stderr, "app0.zar kernel check failed at line %d\n", line); + std::exit(1); + } +} +#define Check(value) Require((value), __LINE__) + +void OnMmapRefused() { + const auto* type = __cxa_current_exception_type_nid_postfix(); + const auto* error = static_cast(__cxa_current_primary_exception_nid_postfix()); + const bool refused = type != nullptr && error != nullptr && std::strcmp(type->name(), typeid(std::runtime_error).name()) == 0 && + std::strstr(error->what(), "mmap on a file read from app0.zar") != nullptr && std::strstr(error->what(), "not implemented") != nullptr; + std::_Exit(refused ? 0 : 1); +} + +template bool NotImplemented(TCall call) { + try { + call(); + } catch (const std::runtime_error& error) { + return std::strstr(error.what(), "not implemented") != nullptr; + } + return false; +} + +constexpr std::size_t BigSize = 3 * 65536 + 77; +constexpr std::uint16_t TypeMask = 0170000; +constexpr std::uint16_t RegularType = 0100000; +constexpr std::uint16_t DirectoryType = 0040000; +constexpr int DuplicateFrom = 0; +constexpr std::uint64_t ReadableRequest = 0x4004667f; + +void Archive(const std::filesystem::path& root) { + AppArchiveFixture::WriteArchive(root / "app0.zar", { + {"Data/Big.bin", AppArchiveFixture::Pattern(BigSize)}, + {"Data/Small.txt", {'z', 'a', 'r'}}, + {"sce_sys/Packed.json", {'{', '}'}}, + {"eboot.bin", {'o', 'l', 'd'}}, + }); + std::ofstream eboot(root / "app0" / "eboot.bin", std::ios::binary); + eboot << "disk"; +} + +void Reads() { + const auto expected = AppArchiveFixture::Pattern(BigSize); + const int fd = sceKernelOpen("/app0/data/big.BIN", SCE_KERNEL_O_RDONLY, 0); + Check(fd >= 0); + std::vector buffer(BigSize + 16); + Check(sceKernelRead(fd, buffer.data(), 70000) == 70000); + Check(std::equal(buffer.begin(), buffer.begin() + 70000, expected.begin())); + Check(sceKernelLseek(fd, 0, 1) == 70000); + Check(lseek_nid_postfix(fd, 0, 1) == 70000); + + std::vector left(10), right(65536); + const KernelIovec vector[] = {{left.data(), left.size()}, {right.data(), right.size()}}; + Check(sceKernelReadv(fd, vector, 2) == 65546); + Check(std::equal(left.begin(), left.end(), expected.begin() + 70000) && std::equal(right.begin(), right.end(), expected.begin() + 70010)); + Check(sceKernelPreadv(fd, vector, 2, 5) == 65546); + Check(std::equal(left.begin(), left.end(), expected.begin() + 5) && std::equal(right.begin(), right.end(), expected.begin() + 15)); + Check(sceKernelLseek(fd, 0, 1) == 135546); + + Check(sceKernelPread(fd, buffer.data(), 100, 65500) == 100 && std::equal(buffer.begin(), buffer.begin() + 100, expected.begin() + 65500)); + Check(pread_nid_postfix(fd, buffer.data(), 100, BigSize - 40) == 40 && std::equal(buffer.begin(), buffer.begin() + 40, expected.end() - 40)); + Check(sceKernelPread(fd, buffer.data(), 1, -1) == SCE_KERNEL_ERROR_EINVAL); + Check(sceKernelLseek(fd, 0, 1) == 135546); + + int available = 0; + Check(ioctl_nid_postfix(fd, ReadableRequest, &available) == 0 && available == static_cast(BigSize - 135546)); + + Check(sceKernelLseek(fd, -7, 2) == static_cast(BigSize - 7)); + Check(sceKernelRead(fd, buffer.data(), 100) == 7 && std::equal(buffer.begin(), buffer.begin() + 7, expected.end() - 7)); + Check(sceKernelRead(fd, buffer.data(), 100) == 0); + Check(sceKernelLseek(fd, -1, 0) == SCE_KERNEL_ERROR_EINVAL); + Check(sceKernelLseek(fd, 0, 1) == static_cast(BigSize)); + Check(sceKernelWrite(fd, "x", 1) == SCE_KERNEL_ERROR_EBADF); + + FileStat status{}; + Check(sceKernelFstat(fd, &status) == 0); + Check((status.st_mode & TypeMask) == RegularType && status.st_size == static_cast(BigSize) && status.st_nlink == 1); + + KernelAioResult result{-1, 0}; + KernelAioRwRequest request{65530, 20, buffer.data(), &result, fd}; + std::int32_t id = 0; + std::int32_t state = 0; + Check(sceKernelAioSubmitReadCommands(&request, 1, 0, &id) == 0); + Check(sceKernelAioWaitRequest(id, &state, nullptr) == 0); + Check(result.return_value == 20 && std::equal(buffer.begin(), buffer.begin() + 20, expected.begin() + 65530)); + std::int32_t removed = 0; + Check(sceKernelAioDeleteRequest(id, &removed) == 0); + + Check(sceKernelLseek(fd, 1000, 0) == 1000); + const int duplicate = dup_nid_postfix(fd); + Check(duplicate >= 0 && sceKernelLseek(duplicate, 0, 1) == 1000); + Check(sceKernelRead(duplicate, buffer.data(), 24) == 24 && sceKernelLseek(fd, 0, 1) == 1024); + const int above = fcntl_nid_postfix(fd, DuplicateFrom, 200); + Check(above >= 200 && sceKernelLseek(above, 0, 1) == 1024); + Check(sceKernelClose(fd) == 0); + Check(sceKernelRead(duplicate, buffer.data(), 4) == 4 && std::equal(buffer.begin(), buffer.begin() + 4, expected.begin() + 1024)); + Check(sceKernelLseek(above, 0, 1) == 1028); + Check(close_nid_postfix(above) == 0); + + const int plain = sceKernelOpen("/app0/eboot.bin", SCE_KERNEL_O_RDONLY, 0); + Check(plain >= 0); + Check(dup2_nid_postfix(plain, duplicate) == duplicate); + Check(sceKernelPread(duplicate, buffer.data(), 8, 0) == 4 && std::memcmp(buffer.data(), "disk", 4) == 0); + Check(sceKernelClose(plain) == 0); + Check(sceKernelClose(duplicate) == 0); +} + +void Status() { + FileStat status{}; + Check(sceKernelStat("/app0/Data/Small.txt", &status) == 0); + Check((status.st_mode & TypeMask) == RegularType && (status.st_mode & 0222) == 0 && status.st_size == 3 && status.st_blocks == 1); + Check(sceKernelStat("/app0/data", &status) == 0 && (status.st_mode & TypeMask) == DirectoryType); + Check(sceKernelStat("/app0/Data/", &status) == 0); + Check(sceKernelStat("/app0/Data/Small.txt/", &status) == SCE_KERNEL_ERROR_ENOTDIR); + Check(sceKernelStat("/app0/Data/Missing.txt", &status) == SCE_KERNEL_ERROR_ENOENT); + Check(sceKernelStat("/app0/eboot.bin", &status) == 0 && status.st_size == 4); + Check(sceKernelCheckReachability("/app0/Data/Small.txt") == 0); + Check(sceKernelCheckReachability("/app0/Data/Missing.txt") == SCE_KERNEL_ERROR_ENOENT); +} + +void Refusals() { + Check(sceKernelOpen("/app0/Data/Small.txt", SCE_KERNEL_O_RDWR, 0) == SCE_KERNEL_ERROR_EROFS); + Check(sceKernelOpen("/app0/Data/Small.txt", SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_TRUNC, 0) == SCE_KERNEL_ERROR_EROFS); + Check(sceKernelOpen("/app0/Data/Small.txt", SCE_KERNEL_O_WRONLY | SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL, 0666) == SCE_KERNEL_ERROR_EEXIST); + Check(sceKernelOpen("/app0/Data/Small.txt", SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0) == SCE_KERNEL_ERROR_ENOTDIR); + Check(sceKernelOpen("/app0/Data/Missing.txt", SCE_KERNEL_O_RDONLY, 0) == SCE_KERNEL_ERROR_ENOENT); + Check(NotImplemented([] { (void)sceKernelOpen("/app0/Data", SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0); })); + Check(NotImplemented([] { (void)sceKernelOpen("/app0/sce_sys", SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0); })); + Check(NotImplemented([] { (void)opendir_nid_postfix("/app0/sce_sys"); })); + Check(NotImplemented([] { (void)opendir_nid_postfix("/app0/Data"); })); + const int fd = sceKernelOpen("/app0/Data/Small.txt", SCE_KERNEL_O_RDONLY, 0); + Check(fd >= 0); + Check(NotImplemented([fd] { (void)fdopen_nid_postfix(fd, "r"); })); + Check(NotImplemented([fd] { (void)flock_nid_postfix(fd, 1); })); + Check(NotImplemented([fd] { alignas(8) char statfs[512]; (void)_fstatfs_nid_postfix(fd, statfs); })); + Check(NotImplemented([fd] { (void)ftruncate_nid_postfix(fd, 0); })); + Check(NotImplemented([fd] { (void)sceKernelFsync(fd); })); + Check(NotImplemented([fd] { (void)sceKernelFdatasync(fd); })); + Check(NotImplemented([fd] { char entries[512]; (void)sceKernelGetdents(fd, entries, sizeof(entries)); })); + Check(NotImplemented([fd] { (void)sceKernelFchmod(fd, 0444); })); + Check(NotImplemented([fd] { (void)sceKernelFutimes(fd, nullptr); })); + FileStat status{}; + Check(sceKernelFstat(fd, &status) == 0 && status.st_size == 3); + Check(sceKernelClose(fd) == 0); +} + +void WithoutArchive() { + { std::ofstream file(std::filesystem::current_path() / "app0" / "Plain.txt", std::ios::binary); file << "plain"; } + const int fd = sceKernelOpen("/app0/Plain.txt", SCE_KERNEL_O_RDONLY, 0); + Check(fd >= 0); + char buffer[8] = {}; + Check(sceKernelRead(fd, buffer, sizeof(buffer)) == 5 && std::memcmp(buffer, "plain", 5) == 0); + Check(sceKernelClose(fd) == 0); + Check(sceKernelOpen("/app0/Data/Small.txt", SCE_KERNEL_O_RDONLY, 0) == SCE_KERNEL_ERROR_ENOENT); + FileStat status{}; + Check(sceKernelStat("/app0/Data/Small.txt", &status) == SCE_KERNEL_ERROR_ENOENT); + const int directory = sceKernelOpen("/app0/sce_sys", SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0); + Check(directory >= 0 && sceKernelClose(directory) == 0); +} + +} + +int main(int argc, char** argv) { + if (argc != 2) { + std::fprintf(stderr, "usage: guest_kernel_app_archive_tests archive|plain\n"); + return 2; + } + const std::string mode = argv[1]; + const auto root = AppArchiveFixture::EnterFreshRoot("anyps5-kernel-app-archive"); + if (mode == "archive") { + Archive(root); + Reads(); + Status(); + Refusals(); + } else if (mode == "mmap") { + Archive(root); + const int fd = sceKernelOpen("/app0/Data/Small.txt", SCE_KERNEL_O_RDONLY, 0); + Check(fd >= 0); + _ZSt13set_terminatePFvvE_nid_postfix(OnMmapRefused); + (void)mmap_nid_postfix(nullptr, 16384, 1, 2, fd, 0); + return 1; + } else if (mode == "plain") { + WithoutArchive(); + } else { + return 2; + } + std::error_code error; + std::filesystem::current_path(std::filesystem::temp_directory_path(), error); + std::filesystem::remove_all(root, error); + return 0; +} diff --git a/core/libs/tests/GuestRandom.cpp b/core/libs/tests/GuestRandom.cpp index f611927f88..df1d5f83c9 100644 --- a/core/libs/tests/GuestRandom.cpp +++ b/core/libs/tests/GuestRandom.cpp @@ -11,11 +11,13 @@ int main() { constexpr int invalid = static_cast(0x817C0016); unsigned char buffer[80]; Require(sceRandomGetRandomNumber(nullptr, 16) == invalid); + Require(sceRandomGetRandomNumber(nullptr, 0) == invalid); std::memset(buffer, 0xAA, sizeof(buffer)); Require(sceRandomGetRandomNumber(buffer, 65) == invalid); Require(buffer[0] == 0xAA); Require(sceRandomGetRandomNumber(buffer, 0) == 0); Require(buffer[0] == 0xAA); + Require(sceRandomGetRandomNumber(buffer, 1) == 0); Require(sceRandomGetRandomNumber(buffer, 7) == 0); Require(buffer[7] == 0xAA); unsigned char first[64] = {}; diff --git a/core/libs/tests/GuestRefusedConnect.cpp b/core/libs/tests/GuestRefusedConnect.cpp new file mode 100644 index 0000000000..409f8ad97c --- /dev/null +++ b/core/libs/tests/GuestRefusedConnect.cpp @@ -0,0 +1,108 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI socket_nid_postfix(int, int, int); +int APS5_VABI bind_nid_postfix(int, const void*, std::uint32_t); +int APS5_VABI connect_nid_postfix(int, const void*, std::uint32_t); +int APS5_VABI listen_nid_postfix(int, int); +int APS5_VABI accept_nid_postfix(int, void*, std::uint32_t*); +int APS5_VABI getsockname_nid_postfix(int, void*, std::uint32_t*); +int APS5_VABI close_nid_postfix(int); +int* APS5_VABI __error_nid_postfix(); +int APS5_VABI sceNetInit_nid_postfix(); +int APS5_VABI sceNetSocket(const char*, int, int, int); +int APS5_VABI sceNetConnect(int, const void*, std::uint32_t); +int APS5_VABI sceNetSocketClose(int); +int* APS5_VABI sceNetErrnoLoc(); +} + +namespace { + +constexpr int GuestInet = 2; +constexpr int GuestStream = 1; +constexpr int GuestConnectionRefused = 61; +constexpr auto RefusalLimit = std::chrono::milliseconds(1000); + +void Require(bool value) { if (!value) std::abort(); } + +std::array ClosedLoopbackPort() { + std::array address{16, 2, 0, 0, 127, 0, 0, 1}; + const int descriptor = socket_nid_postfix(GuestInet, GuestStream, 0); + Require(descriptor >= 0); + Require(bind_nid_postfix(descriptor, address.data(), address.size()) == 0); + std::uint32_t length = address.size(); + Require(getsockname_nid_postfix(descriptor, address.data(), &length) == 0); + Require(close_nid_postfix(descriptor) == 0); + Require(address[2] != 0 || address[3] != 0); + return address; +} + +template +bool ConnectsOnceBacklogDrains(TConnect connect) { + std::array address{16, 2, 0, 0, 127, 0, 0, 1}; + const int listener = socket_nid_postfix(GuestInet, GuestStream, 0); + Require(listener >= 0); + Require(bind_nid_postfix(listener, address.data(), address.size()) == 0); + std::uint32_t length = address.size(); + Require(getsockname_nid_postfix(listener, address.data(), &length) == 0); + Require(listen_nid_postfix(listener, 1) == 0); + const int queued = socket_nid_postfix(GuestInet, GuestStream, 0); + Require(queued >= 0 && connect_nid_postfix(queued, address.data(), address.size()) == 0); + std::thread acceptor([listener] { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + const int accepted = accept_nid_postfix(listener, nullptr, nullptr); + Require(accepted >= 0 && close_nid_postfix(accepted) == 0); + }); + const bool connected = connect(address); + acceptor.join(); + Require(close_nid_postfix(queued) == 0 && close_nid_postfix(listener) == 0); + return connected; +} + +template +bool RefusedAtOnce(TConnect connect) { + const auto start = std::chrono::steady_clock::now(); + const bool refused = connect(); + return refused && std::chrono::steady_clock::now() - start < RefusalLimit; +} + +} + +int main() { + Require(sceNetInit_nid_postfix() == 0); + const auto address = ClosedLoopbackPort(); + Require(RefusedAtOnce([&] { + const int descriptor = socket_nid_postfix(GuestInet, GuestStream, 0); + Require(descriptor >= 0); + const bool refused = connect_nid_postfix(descriptor, address.data(), address.size()) == -1 && *__error_nid_postfix() == GuestConnectionRefused; + Require(close_nid_postfix(descriptor) == 0); + return refused; + })); + Require(RefusedAtOnce([&] { + const int descriptor = sceNetSocket("refused", GuestInet, GuestStream, 0); + Require(descriptor >= 0); + const bool refused = sceNetConnect(descriptor, address.data(), address.size()) == static_cast(0x80410100u | GuestConnectionRefused) + && *sceNetErrnoLoc() == GuestConnectionRefused; + Require(sceNetSocketClose(descriptor) == 0); + return refused; + })); + Require(ConnectsOnceBacklogDrains([](const std::array& listening) { + const int descriptor = socket_nid_postfix(GuestInet, GuestStream, 0); + Require(descriptor >= 0); + const bool connected = connect_nid_postfix(descriptor, listening.data(), listening.size()) == 0; + Require(close_nid_postfix(descriptor) == 0); + return connected; + })); + Require(ConnectsOnceBacklogDrains([](const std::array& listening) { + const int descriptor = sceNetSocket("backlog", GuestInet, GuestStream, 0); + Require(descriptor >= 0); + const bool connected = sceNetConnect(descriptor, listening.data(), listening.size()) == 0; + Require(sceNetSocketClose(descriptor) == 0); + return connected; + })); +} diff --git a/core/libs/tests/GuestSystemServiceLoadExecStandardError.cpp b/core/libs/tests/GuestSystemServiceLoadExecStandardError.cpp new file mode 100644 index 0000000000..ec83ac1c10 --- /dev/null +++ b/core/libs/tests/GuestSystemServiceLoadExecStandardError.cpp @@ -0,0 +1,74 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceSystemServiceLoadExec(const char*, const char* const*); +int APS5_VABI dup2_nid_postfix(int, int); +} + +namespace { + +constexpr const char* Marker = "APS5_LOAD_EXEC_STANDARD_ERROR"; +constexpr const char* AfterLoadExec = "--after-load-exec"; + +void Require(bool value) { + if (!value) std::abort(); +} + +std::string Drain(int descriptor) { + std::string text; + char buffer[4096]; + for (ssize_t count; (count = ::read(descriptor, buffer, sizeof(buffer))) > 0;) text.append(buffer, static_cast(count)); + ::close(descriptor); + return text; +} + +void Run(const char* mode, const char* const* arguments) { + int host[2] = {-1, -1}; + int guest[2] = {-1, -1}; + Require(::pipe(host) == 0); + Require(::pipe(guest) == 0); + const pid_t child = ::fork(); + Require(child >= 0); + if (child == 0) { + ::close(host[0]); + ::close(guest[0]); + if (::dup2(host[1], 2) != 2) ::_exit(70); + ::close(host[1]); + if (dup2_nid_postfix(guest[1], 2) != 2) ::_exit(71); + ::close(guest[1]); + if (::setenv(Marker, mode, 1) != 0) ::_exit(72); + sceSystemServiceLoadExec("/app0/eboot.bin", arguments); + ::_exit(73); + } + ::close(host[1]); + ::close(guest[1]); + const std::string hostText = Drain(host[0]); + const std::string guestText = Drain(guest[0]); + int status = 0; + Require(::waitpid(child, &status, 0) == child); + Require(WIFEXITED(status) && WEXITSTATUS(status) == 0); + Require(hostText == std::string("started ") + mode + "\n"); + Require(guestText.empty()); +} + +} + +int main(int argc, char** argv) { + if (const char* mode = std::getenv(Marker); mode != nullptr) { + if (std::strcmp(mode, "list") == 0) Require(argc == 1 && std::strcmp(argv[0], AfterLoadExec) == 0); + else Require(std::strcmp(mode, "none") == 0 && argc == 1); + std::fprintf(stderr, "started %s\n", mode); + return 0; + } + const char* const arguments[] = {AfterLoadExec, nullptr}; + Run("list", arguments); + Run("none", nullptr); + std::puts("LoadExec standard error tests passed"); + return 0; +} diff --git a/core/libs/tests/WindowsBacktraceReport.cpp b/core/libs/tests/WindowsBacktraceReport.cpp index a5d93edc43..32871b4f63 100644 --- a/core/libs/tests/WindowsBacktraceReport.cpp +++ b/core/libs/tests/WindowsBacktraceReport.cpp @@ -1,11 +1,35 @@ #include #include +#include #include "../prx/libc/include/general/VabiMacros.hpp" +#include "../prx/libc/include/FailureReport.hpp" extern "C" int* APS5_VABI __error_nid_postfix(); -int main() { +namespace { + +void KeptReport() noexcept { + std::fputs("kept failure report ran\n", stderr); +} + +void DroppedReport() noexcept { + std::fputs("dropped failure report ran\n", stderr); +} + +} + +int main(int argc, char** argv) { if (__error_nid_postfix() == nullptr) return 1; + if (argc == 2 && std::strcmp(argv[1], "report") == 0) { + LibcRegisterFailureReport_nid_no_patch(DroppedReport); + LibcRegisterFailureReport_nid_no_patch(KeptReport); + LibcRegisterFailureReport_nid_no_patch(KeptReport); + LibcUnregisterFailureReport_nid_no_patch(DroppedReport); + } + if (argc == 2 && std::strcmp(argv[1], "report-once") == 0) { + LibcRegisterFailureReport_nid_no_patch(KeptReport); + LibcRunFailureReports_nid_no_patch(); + } _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); std::fputs("aborting\n", stderr); std::abort(); diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index d227fc017d..9710a16bf7 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -524,6 +524,9 @@ if(BUILD_TESTING) add_test(NAME guest_export_lookup COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_export_lookup.py $) set_tests_properties(guest_export_lookup PROPERTIES TIMEOUT 30) + add_test(NAME guest_module_fini COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_module_fini.py + $ $) add_test(NAME guest_needed_nested_modules COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_needed_nested_modules.py $) set(windows_stdio_host_arguments) diff --git a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp index 76598e35ae..b64e126fec 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp @@ -141,18 +141,18 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma bytes.insert(bytes.end(), stub.begin(), stub.end()); return start; }; - const auto callThrough = [&](std::uint64_t target) { + const auto finalizer = [&](std::uint64_t target) { if (target == 0) return std::uint64_t{}; const auto start = address(); - const auto distance = static_cast(target) - static_cast(start + 5); + const auto distance = static_cast(target) - static_cast(start + 11); if (distance < std::numeric_limits::min() || distance > std::numeric_limits::max()) throw Domain::RelinkerException("Guest finaliser exceeds relative branch range"); - bytes.push_back(0xe9); + bytes.insert(bytes.end(), {0x31, 0xff, 0x31, 0xf6, 0x31, 0xd2, 0xe9}); Io::AppendU32(bytes, static_cast(distance)); return start; }; const auto init = lifecycle(image.Init); - const auto fini = callThrough(image.Fini); + const auto fini = finalizer(image.Fini); Io::AlignBuffer(bytes, 8); const auto dynamicOffset = bytes.size(); const auto dynamicAddress = address(); diff --git a/core/relinker/elfpatcher/tests/GuestModuleArgsHost.cpp b/core/relinker/elfpatcher/tests/GuestModuleArgsHost.cpp index 708d39dc7b..b99a198578 100644 --- a/core/relinker/elfpatcher/tests/GuestModuleArgsHost.cpp +++ b/core/relinker/elfpatcher/tests/GuestModuleArgsHost.cpp @@ -45,6 +45,13 @@ extern "C" void SetPendingArgs(std::size_t args, const void* argp) { pending.argp = argp; } +extern "C" void APS5_VABI RecordFini(std::size_t args, const void* argp, const void* callback) { + char line[112]; + std::snprintf(line, sizeof(line), "fini=%016zx argp=%016zx callback=%016zx\n", args, + reinterpret_cast(argp), reinterpret_cast(callback)); + Write(line); +} + extern "C" const void* __aps5_get_pending_module_args_nid_no_patch() { return &pending; } diff --git a/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp b/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp index 8e082aa3e9..cf7f7a80ae 100644 --- a/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp +++ b/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp @@ -19,7 +19,9 @@ using namespace Elfpatcher; namespace { +constexpr std::uint16_t kEtNone = 0; constexpr std::uint16_t kEtRel = 1; +constexpr std::uint16_t kEtCore = 4; } @@ -55,8 +57,13 @@ std::string RelinkerPipeline::relocationTypeName(std::uint32_t type) { } RelinkResult RelinkerPipeline::Relink() { - if (elfReader->ReadHeader().Type == kEtRel) + const auto inputType = elfReader->ReadHeader().Type; + if (inputType == kEtRel) throw RelinkerException("ET_REL relocatable inputs are not supported"); + if (inputType == kEtNone) + throw RelinkerException("ET_NONE inputs do not specify an ELF object type"); + if (inputType == kEtCore) + throw RelinkerException("ET_CORE core dumps are not executable images"); auto programHeaders = elfReader->ReadProgramHeaders(); diff --git a/core/relinker/relinker/tests/test_elf_header.py b/core/relinker/relinker/tests/test_elf_header.py index 9b42382448..d345aa292d 100644 --- a/core/relinker/relinker/tests/test_elf_header.py +++ b/core/relinker/relinker/tests/test_elf_header.py @@ -73,10 +73,50 @@ def structural_cases(relinker, directory): assert not output.exists() and not registry.exists(), (case.name, "output written") +def object_type_cases(relinker, directory): + cases = ((0, "ET_NONE inputs do not specify an ELF object type"), + (1, "ET_REL relocatable inputs are not supported"), + (4, "ET_CORE core dumps are not executable images"), + (2, None), (3, None), (0xFE10, None), (0xFE18, None)) + for mode, flags in enumerate(MODES): + for filter_level in (0, 1, 2): + for object_type, error in cases: + if object_type == 2 and ("--windows" in flags or "--macos" in flags): + continue + case = Path(directory) / f"type-{mode}-{filter_level}-{object_type}" + case.mkdir() + source = case / "input.elf" + output = case / "output.out" + registry = case / "output.registry.json" + data = main_fixture() + if filter_level == 2: + struct.pack_into("mesh ? &*request.graphics->mesh : nullptr; append(key, mesh != nullptr); if (mesh == nullptr) return; - for (const auto value : {mesh->inputPrimitive, mesh->primitivesPerGroup, mesh->verticesPerGroup, mesh->maxVertices, mesh->maxPrimitives, mesh->threadsPerGroup, mesh->ldsSizeDwords, mesh->provokingVertex, mesh->esgsItemSize}) append(key, value); + for (const auto value : {mesh->inputPrimitive, mesh->primitivesPerGroup, mesh->verticesPerGroup, mesh->maxVertices, mesh->maxPrimitives, mesh->threadsPerGroup, mesh->ldsSizeDwords, mesh->provokingVertex, mesh->esgsItemSize, mesh->outputPrimitiveSize}) append(key, value); } static void appendTessellation(std::vector& key, const RecompileRequest& request) { @@ -261,6 +261,7 @@ private: append(key, value.registrationWordLimit); append(key, value.vertexSubgroups); append(key, value.float16DenormPreserve); + append(key, value.spannedWaves); } }; diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index 20c4e29f51..b43d75c3c9 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -518,6 +518,7 @@ void writeMeshConfiguration(Writer& writer, const MeshConfiguration& configurati writer.WriteU32(configuration.ldsSizeDwords); writer.WriteU32(configuration.provokingVertex); writer.WriteU32(configuration.esgsItemSize); + writer.WriteU32(configuration.outputPrimitiveSize); } MeshConfiguration readMeshConfiguration(Reader& reader, std::uint32_t version) { @@ -531,6 +532,7 @@ MeshConfiguration readMeshConfiguration(Reader& reader, std::uint32_t version) { configuration.ldsSizeDwords = reader.ReadU32(); configuration.provokingVertex = reader.ReadU32(); configuration.esgsItemSize = version >= 4u ? reader.ReadU32() : 4u; + if (version >= 31u) configuration.outputPrimitiveSize = reader.ReadU32(); return configuration; } @@ -776,7 +778,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(30u); + writer.WriteU32(32u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -801,6 +803,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const writer.WriteBool(request.target.vertexSubgroups); writer.WriteBool(request.target.float16DenormPreserve); writer.WriteU32Span(request.context.linkedCallFallbacks); + writer.WriteBool(request.target.spannedWaves); return base64Encode(buffer); } @@ -809,7 +812,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 30u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 32u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -845,6 +848,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const result.linkedCallFallbacks = reader.ReadU32Vector(); result.request.context.linkedCallFallbacks = result.linkedCallFallbacks; } + if (version >= 32u) result.request.target.spannedWaves = reader.ReadBool(); if (version >= 21u && reader.Remaining() != 0u) throw std::runtime_error("RequestSerializer: unexpected trailing data"); return result; } diff --git a/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp b/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp index 4b3755e656..7396e8e96d 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp @@ -19,6 +19,7 @@ struct BindingAllocationResult : CompiledBindingLayout { std::vector specialization; std::vector bindings; std::vector pushConstants; + std::vector shaderDataUserWords; }; class BindingAllocator { diff --git a/core/shader/recompiler/Optimization/include/Optimization/ConstantFolder.hpp b/core/shader/recompiler/Optimization/include/Optimization/ConstantFolder.hpp index d2df4ed3cc..6cbf62c460 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ConstantFolder.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ConstantFolder.hpp @@ -14,6 +14,8 @@ public: void Fold(IrProgram& program, std::span blocks) const; + [[nodiscard]] bool FoldVertexLaneBounds(IrProgram& program, std::uint32_t hostSubgroupSize, std::uint32_t spirvVersion) const; + [[nodiscard]] bool ComposePackedAncillary(IrProgram& program) const; private: diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index 77328b8672..f83da69525 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -38,6 +38,7 @@ struct ResourceCapture { // The walk's read addresses when the plan has pure flat slots (the leaf of each pure slot, // and every other read sorted and deduplicated); empty otherwise. SrtReadTrace readTrace; + SrtUserDataUse userDataUse; // APS5_PROFILE_DRAW: what CaptureResources spent resolving the source (the stage inputs, the // key over the code, the plan) before the walk. std::uint64_t sourceNanoseconds = 0; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp index 72fe2675ee..2ba98418bd 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp @@ -92,6 +92,7 @@ struct ShaderMeshInputInfo: ShaderWorkgroupInputInfo { std::uint32_t maxPrimitives = 0; std::uint32_t provokingVertex = 0; std::uint32_t esgsItemSize = 0; + std::uint32_t outputPrimitiveSize = 3; [[nodiscard]] std::uint32_t InputPrimitiveSize() const { return inputPrimitive == 1u ? 1u : inputPrimitive == 2u ? 2u : 3u; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp index 46dc84574f..8d345285de 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp @@ -2,6 +2,7 @@ #define CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_SRTWALKER_HPP #include "IntermediateRepresentation/IrProgram.hpp" +#include "UserDataUse.hpp" #include #include @@ -35,6 +36,7 @@ struct SrtRuntime { SrtMemoryProbe accessible = nullptr; SrtMemoryProbe pendingWrite = nullptr; SrtMemoryBulkReader readWords = nullptr; + SrtUserDataUse* userDataUse = nullptr; }; enum class RuntimeValueType { diff --git a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp index 3505f640f4..52643c8990 100644 --- a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp +++ b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp @@ -846,6 +846,22 @@ void ConstantFolder::Fold(IrProgram& program) const { Fold(program, program.BlockOrder()); } +bool ConstantFolder::FoldVertexLaneBounds(IrProgram& program, std::uint32_t hostSubgroupSize, std::uint32_t spirvVersion) const { + if (program.Resources().stage != IrShaderStage::Vertex || hostSubgroupSize != 32u || spirvVersion < 0x00010000u || spirvVersion >= 0x00010600u) return false; + bool changed = false; + for (IrBlock* block : program.BlockOrder()) { + for (IrValue* value : block->Instructions()) { + if (value->Opcode() != IrOpcode::SelectU32 || value->ArgumentCount() != 3u) continue; + const auto& half = resolveArg(*value, 0); + auto& low = resolveArg(*value, 1); + if (half.Opcode() != IrOpcode::ULessThan32 || half.ArgumentCount() != 2u || !isLaneIndex(resolveArg(half, 0)) || !isImmediateU32(resolveArg(half, 1), 32u) || !isImmediate(low, IrType::U32) || !isImmediate(resolveArg(*value, 2), IrType::U32)) continue; + replaceWith(*value, low); + changed = true; + } + } + return changed; +} + bool ConstantFolder::ComposePackedAncillary(IrProgram& program) const { std::vector words; for (IrBlock* block : program.BlockOrder()) { diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index d988925a91..fb6e05b76f 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -229,7 +229,7 @@ UnnormalizedProof ProveUnnormalized(const ShaderInfo& info, const ResourceSnapsh if (!exact) break; if (pair.image != resource) continue; const auto& words = snapshot.samplers.at(pair.sampler).dwords; - exact = proof.samplers[pair.sampler] && info.samplers.at(pair.sampler).uses == SamplerUseExplicitLod && (words[1] & 0xfffu) == 0u && (words[2] & 0x3fffu) == 0u; + exact = proof.samplers[pair.sampler] && (((words[2] >> 26u) & 3u) == 0u || (info.samplers.at(pair.sampler).uses == SamplerUseExplicitLod && (words[1] & 0xfffu) == 0u && (words[2] & 0x3fffu) == 0u)); } proof.imagesAtBaseLevel[resource] = exact; } @@ -525,6 +525,10 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con const auto& layout = compiled.layout; const bool needsShaderData = layout.UsesPushData() || plan.storageData; auto shaderData = needsShaderData ? ShaderDataDwordsFor(layout, userDataBase, snapshot, partialThreads, plan.imageMetadata, plan.runtimeState) : std::vector{}; + allocation.shaderDataUserWords.clear(); + if (needsShaderData) { + for (const auto reg : layout.userDataRegisters) allocation.shaderDataUserWords.push_back(reg - userDataBase); + } std::vector bindings; bindings.reserve(plan.bindings.size()); std::size_t writtenHere = 0; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 6823ae7a2a..ff8686cb2f 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -78,7 +78,7 @@ ImageType rawImageType(const DescriptorValue& descriptor) { bool nullImageDescriptor(const ImageResource& image, const DescriptorValue& descriptor) { if (nullImageDescriptor(descriptor)) return true; - return rawImageType(descriptor) == static_cast(0u) && image.table == NoTable && image.read && !image.written && !image.atomic; + return static_cast(rawImageType(descriptor)) <= 1u && image.table == NoTable && image.read && !image.written && !image.atomic; } IrBufferFormat rawImageFormat(const DescriptorValue& descriptor) { @@ -260,7 +260,6 @@ ImageDecode inspectImageDescriptor(const DescriptorValue& descriptor, const Imag decoded.numericClass = IrTextureNumericClass::Uint; } if ((rawSintStorage || (base.atomic && format == IrBufferFormat::Format32SInt)) && !base.packed) { - if (!base.depthBitsCompatible) return rejectImage(PoisonReason::InvalidFormat, "stores 16-bit data to an image of a SINT format"); decoded.conversionFormat = format; } } else if (decoded.numericClass == IrTextureNumericClass::Unsupported || (base.depthCompare && decoded.numericClass != IrTextureNumericClass::Float)) { @@ -1318,7 +1317,7 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe } } } - if (storage && image.depthBitsCompatible && !image.packed && !image.atomic64 && ((image.written && !image.read) || image.atomic)) { + if (storage && !image.packed && !image.atomic64 && ((image.written && !image.read) || image.atomic)) { constexpr std::array formats{IrBufferFormat::Format32SInt, IrBufferFormat::Format32_32SInt, IrBufferFormat::Format32_32_32_32SInt, IrBufferFormat::Format16SInt, IrBufferFormat::Format8_8SInt, IrBufferFormat::Format16_16SInt, IrBufferFormat::Format8_8_8_8SInt, IrBufferFormat::Format16_16_16_16SInt}; for (const auto format : formats) { if (image.atomic && format != IrBufferFormat::Format32SInt) continue; diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp index 190e6af479..d9a5fbccf7 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp @@ -194,7 +194,18 @@ bool BasicEvaluator::EvaluateRawRead(const TNode& inst, std::uint64_t& re std::uint64_t low = 0; std::uint64_t high = 0; std::uint64_t offset = 0; - if (!Arg(handle, 0, low) || !Arg(handle, 1, high) || !Arg(inst, 1, offset)) { + std::int32_t origin = -1; + const auto baseWord = [&] { + const TNode source = handle.Argument(0); + if (_runtime.userDataUse == nullptr || source.HasImmediate() || source.Opcode() != IrOpcode::GetUserData) return Arg(handle, 0, low); + const auto reg = RegIndex(static_cast(source.Argument(0).RegisterIndex())); + if (reg < _program.userDataBase || reg - _program.userDataBase >= _runtime.userData.size() || reg - _program.userDataBase >= SrtUserDataUse::TrackedWords) return Arg(handle, 0, low); + origin = static_cast(reg - _program.userDataBase); + low = _runtime.userData[reg - _program.userDataBase]; + _runtime.userDataUse->asBase |= std::uint64_t {1} << origin; + return true; + }; + if (!baseWord() || !Arg(handle, 1, high) || !Arg(inst, 1, offset)) { return false; } const auto base = ((high << 32u) | static_cast(low)) & AddressMask; @@ -229,6 +240,7 @@ bool BasicEvaluator::EvaluateRawRead(const TNode& inst, std::uint64_t& re result = 0; return true; } + if (_runtime.userDataUse != nullptr) _runtime.userDataUse->reads.emplace_back(address, origin); if (_inaccessible != nullptr && _runtime.accessible != nullptr && !_runtime.accessible(_runtime.userContext, address, sizeof(std::uint32_t)) && LoadedFromMemory(*handle.Value())) { *_inaccessible = {inst.Value(), address}; return false; @@ -407,6 +419,7 @@ bool BasicEvaluator::EvaluateInst(const TNode& inst, std::uint64_t& resul if (reg < _program.userDataBase || reg - _program.userDataBase >= _runtime.userData.size()) { return false; } + if (_runtime.userDataUse != nullptr) _runtime.userDataUse->asValue |= reg - _program.userDataBase < SrtUserDataUse::TrackedWords ? std::uint64_t {1} << (reg - _program.userDataBase) : ~std::uint64_t {0}; result = _runtime.userData[reg - _program.userDataBase]; return true; } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 32dcb23ba2..9bf473cf58 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -494,6 +494,12 @@ void prepareInterface(const RecompileRequest& request, IrProgram& program) { const bool nativeSampleOffsets = request.target.nonConstantImageOffsets && std::find(request.target.supportedCapabilities.begin(), request.target.supportedCapabilities.end(), spv::CapabilityImageGatherExtended) != request.target.supportedCapabilities.end(); resourceMaterializer.ApplyStaticInterface(program, nativeSampleOffsets); + constexpr ConstantFolder constantFolder; + if (request.shader.stage == ShaderStage::Vertex && constantFolder.FoldVertexLaneBounds(program, HostSubgroupSize(request), request.target.spirvVersion)) { + constantFolder.Fold(program); + ResolveControlFlowIdentities(program); + } + deadCodeEliminator.RemoveIdentities(program); constexpr RedundantValueEliminator redundantValueEliminator; static_cast(redundantValueEliminator.Eliminate(program)); @@ -524,11 +530,19 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) targetOptions.registrationModules = registration; targetOptions.fragmentDescriptorSet = request.target.registrationModules; targetOptions.float16DenormPreserve = request.target.float16DenormPreserve; + targetOptions.spannedWaves = request.target.spannedWaves; constexpr SpirvEmitter spirvEmitter; CompiledShaderArtifact result; result.variantId = nextVariantId(); - result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions); + SpirvEmitInfo emitted; + try { + result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions, &emitted); + } catch (const SpannedWaveUnsupported& error) { + std::fprintf(stderr, "[spanned-wave] 0x%llx falls back to two lanes per invocation: %s\n", static_cast(request.shader.codeAddress), error.what()); + targetOptions.spannedWaves = false; + result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions); + } if (registration && result.spirv.size() > request.target.registrationWordLimit) { registration = false; program.Metadata().bindingLayoutComplete = false; @@ -549,6 +563,7 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) result.imageMetadataDword = bindings.layout.ImageMetadataDword(); result.runtimeImageCount = bindings.layout.runtimeImageCount; result.hostSubgroupSize = HostSubgroupSize(request); + result.fullSubgroups = emitted.fullSubgroups; result.vertexOffsetSgpr = program.Info().vertexOffsetSgpr; result.instanceOffsetSgpr = program.Info().instanceOffsetSgpr; result.vertexOffsetShared = program.Info().vertexOffsetShared; @@ -865,6 +880,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil .vertexOffsetConflict = artifact.vertexOffsetConflict, .instanceOffsetConflict = artifact.instanceOffsetConflict, .hostSubgroupSize = artifact.hostSubgroupSize, + .fullSubgroups = artifact.fullSubgroups, .parameterExports = artifact.parameterExports, .fragmentParameters = artifact.fragmentParameters, .barycentricEmulation = artifact.barycentricEmulation, @@ -992,6 +1008,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil std::copy_n(runtimeSelectors.begin(), layout.runtimeStateDwords, data->guestDescriptor.begin() + first); } } + result.shaderDataUserWords = std::move(bindings.shaderDataUserWords); result.poisonedSrtReads = static_cast(snapshot.srtPoison.size()) + snapshot.nullRootReads; if (!tables.map.empty()) { result.imageTablePoison.reserve(tables.poison.size()); @@ -1326,11 +1343,12 @@ namespace { // traced into the capture (ResourceCapture::readTrace). void materializeCapture(ResourceCapture& capture, const SrtRuntime& runtime) { const auto& plan = *capture.plan; + SrtRuntime traced = runtime; + traced.userDataUse = &capture.userDataUse; if (std::none_of(plan.pureFlatSlots.begin(), plan.pureFlatSlots.end(), [](std::uint8_t pure) { return pure != 0u; })) { - ResourceMaterializer{}.Materialize(plan, runtime, capture.snapshot); + ResourceMaterializer{}.Materialize(plan, traced, capture.snapshot); return; } - SrtRuntime traced = runtime; traced.readTrace = &capture.readTrace; ResourceMaterializer{}.Materialize(plan, traced, capture.snapshot); auto& other = capture.readTrace.otherReads; diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 157ab11feb..123c488f40 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -5,6 +5,7 @@ #include "PipelineSpecialization.hpp" #include #include +#include #include #include #include @@ -266,6 +267,7 @@ struct SpirvTarget { std::uint32_t registrationWordLimit = 200000u; bool vertexSubgroups = true; bool float16DenormPreserve = false; + bool spannedWaves = false; }; struct BindingLayout { @@ -302,6 +304,7 @@ struct MeshConfiguration { std::uint32_t ldsSizeDwords; std::uint32_t provokingVertex; std::uint32_t esgsItemSize = 0; + std::uint32_t outputPrimitiveSize = 3; }; struct TessellationConfiguration { @@ -468,25 +471,64 @@ struct BarycentricEmulationLayout { [[nodiscard]] BarycentricEmulationLayout LayoutBarycentricEmulation(std::span parameters, const BarycentricEmulation& emulation); +// A 128-bit hash of a module's words (SharedSpirv::ContentHash), zero for an empty module. The +// mixing is pure integer arithmetic over the words in order, so the value is the same on every run, +// platform and compiler: a pipeline journal names modules by it across runs. +struct SpirvContentHash { + std::uint64_t low = 0; + std::uint64_t high = 0; + bool operator==(const SpirvContentHash&) const = default; +}; + +inline std::uint64_t HashSpirvWords(std::span words, std::uint64_t seed) { + constexpr std::uint64_t multiplier = 0x9e3779b97f4a7c15ull; + std::uint64_t hash = seed ^ (static_cast(words.size()) * multiplier); + const auto mix = [&](std::uint64_t chunk) { + chunk *= 0xff51afd7ed558ccdull; + chunk ^= chunk >> 32u; + hash = (hash ^ chunk) * multiplier; + hash = (hash << 27u) | (hash >> 37u); + }; + std::size_t index = 0; + for (; index + 2 <= words.size(); index += 2) mix(words[index] | (static_cast(words[index + 1]) << 32u)); + if (index < words.size()) mix(words[index] ^ 0x8000000000000000ull); + hash ^= hash >> 33u; + hash *= 0xff51afd7ed558ccdull; + hash ^= hash >> 33u; + hash *= 0xc4ceb9fe1a85ec53ull; + hash ^= hash >> 33u; + return hash; +} + // Compiled SPIR-V shared between a cached variant and every result materialized from it: results // are copied per dispatch and draw, so the words are reference counted and only duplicated when a // holder writes to them (tests and tools patch modules in place). Reads look like a vector. // The non-const data(), operator[], begin() and end() count as writes: on a shared holder (every // cached result) they clone the module, so a consumer that only reads takes the result by const // reference, or the per-dispatch copy this class removes comes back without a compiler hint. +// ContentHash is computed once per shared holder, on first use, and dropped by a write. class SharedSpirv { public: SharedSpirv() = default; - SharedSpirv(std::vector words) : words(std::make_shared>(std::move(words))) {} - SharedSpirv(std::shared_ptr owner, std::span view) : owner(std::move(owner)), view(view) {} + SharedSpirv(std::vector words) : holder(std::make_shared(std::move(words), nullptr, std::span{})) {} + SharedSpirv(std::shared_ptr owner, std::span view) : holder(std::make_shared(std::vector{}, std::move(owner), view)) {} SharedSpirv& operator=(std::vector other) { - words = std::make_shared>(std::move(other)); - owner.reset(); - view = {}; + holder = std::make_shared(std::move(other), nullptr, std::span{}); return *this; } - [[nodiscard]] std::span Words() const { return words ? std::span(*words) : view; } + [[nodiscard]] std::span Words() const { return holder ? holder->Words() : std::span{}; } + [[nodiscard]] SpirvContentHash ContentHash() const { + if (holder == nullptr) return {}; + if (holder->hashed.load(std::memory_order_acquire)) return {holder->hashLow.load(std::memory_order_relaxed), holder->hashHigh.load(std::memory_order_relaxed)}; + const auto words = holder->Words(); + if (words.empty()) return {}; + const SpirvContentHash hash{HashSpirvWords(words, 0x5eed5a1d00000001ull), HashSpirvWords(words, 0x5eed5a1d00000002ull)}; + holder->hashLow.store(hash.low, std::memory_order_relaxed); + holder->hashHigh.store(hash.high, std::memory_order_relaxed); + holder->hashed.store(true, std::memory_order_release); + return hash; + } [[nodiscard]] std::vector Copy() const { return {Words().begin(), Words().end()}; } [[nodiscard]] std::size_t size() const { return Words().size(); } [[nodiscard]] bool empty() const { return Words().empty(); } @@ -503,18 +545,27 @@ public: friend bool operator==(const SharedSpirv& left, const SharedSpirv& right) { return std::ranges::equal(left.Words(), right.Words()); } private: + struct Holder { + Holder(std::vector words, std::shared_ptr owner, std::span view) : words(std::move(words)), owner(std::move(owner)), view(view) {} + [[nodiscard]] std::span Words() const { return owner ? view : std::span(words); } + std::vector words; + std::shared_ptr owner; + std::span view; + mutable std::atomic hashLow{0}; + mutable std::atomic hashHigh{0}; + mutable std::atomic hashed{false}; + }; + std::vector& Mutable() { - if (words == nullptr || words.use_count() != 1) { - words = std::make_shared>(Copy()); - owner.reset(); - view = {}; + if (holder == nullptr || holder.use_count() != 1 || holder->owner != nullptr) { + holder = std::make_shared(Copy(), nullptr, std::span{}); + } else { + holder->hashed.store(false, std::memory_order_relaxed); } - return *words; + return holder->words; } - std::shared_ptr> words; - std::shared_ptr owner; - std::span view; + std::shared_ptr holder; }; struct VertexInput { @@ -553,6 +604,7 @@ struct CompiledShaderArtifact { bool vertexOffsetConflict = false; bool instanceOffsetConflict = false; std::uint32_t hostSubgroupSize = 0; + bool fullSubgroups = false; std::vector parameterExports; std::vector fragmentParameters; BarycentricEmulation barycentricEmulation; @@ -576,10 +628,43 @@ struct ShaderInvocation { std::uint32_t imageTableFaults = 0; }; +// What a Vulkan pipeline built from a result depends on besides the fixed-function state and the +// descriptor set layouts: the module words, the specialization constants handed to the driver, the +// subgroup width the module was laid out for, the workgroup memory slice and the bindings' shape. +// Two results with equal identities build equal pipelines, in this run or another: unlike the +// process-lifetime variant ids, the identity is the same on every run, so the pipeline stores key on +// it and a pipeline journal can name a stage across runs. Empty for a result without a module. +struct PipelineIdentity { + std::uint64_t low = 0; + std::uint64_t high = 0; + [[nodiscard]] bool Empty() const { return low == 0 && high == 0; } + bool operator==(const PipelineIdentity&) const = default; + auto operator<=>(const PipelineIdentity&) const = default; +}; + struct RecompileResult : CompiledShaderArtifact, ShaderInvocation { bool cacheHit = false; std::uint32_t workgroupMemoryDwords = 0; + std::vector shaderDataUserWords; [[nodiscard]] std::uint64_t PipelineVariantId() const { return specializationId != 0 ? specializationId : variantId; } + [[nodiscard]] PipelineIdentity Identity() const { + const auto hash = spirv.ContentHash(); + if (hash == SpirvContentHash{}) return {}; + PipelineIdentity identity{hash.low, hash.high}; + const auto mix = [&](std::uint64_t value) { + identity.low = (identity.low ^ value) * 0x9e3779b97f4a7c15ull; + identity.low ^= identity.low >> 29u; + identity.high = (identity.high ^ (value + 0x7f4a7c159e3779b9ull)) * 0xc2b2ae3d27d4eb4full; + identity.high ^= identity.high >> 31u; + }; + mix(specialization.size()); + for (const auto& constant : specialization) mix(constant.id | (static_cast(constant.value) << 32u)); + mix(hostSubgroupSize | (fullSubgroups ? 0x10000u : 0u) | (static_cast(workgroupMemoryDwords) << 32u)); + mix(bindings.size()); + for (const auto& binding : bindings) mix(binding.descriptorSet | (static_cast(binding.binding) << 16u) | (static_cast(binding.kind) << 40u) | (static_cast(binding.count) << 48u)); + if (identity.Empty()) identity.low = 1; + return identity; + } }; [[nodiscard]] RecompileResult Recompile(const RecompileRequest& request); diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 0656a8fb3f..2e1cdbbfcf 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -84,7 +84,7 @@ std::filesystem::path ShaderCacheDirectory() { namespace ShaderRecompiler::ShaderDiskCache { #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__) -static_assert(sizeof(CompiledShaderArtifact) == 264, "CompiledShaderArtifact changed: update the artifact encoder"); +static_assert(sizeof(CompiledShaderArtifact) == 240, "CompiledShaderArtifact changed: update the artifact encoder"); static_assert(sizeof(ShaderInvocation) == 200, "ShaderInvocation changed: update the invocation encoder"); static_assert(sizeof(RecompileResult) == 472, "RecompileResult changed: update EncodeResult and DecodeResult"); static_assert(sizeof(DescriptorBinding) == 592, "DescriptorBinding changed: update the binding encoder"); @@ -101,7 +101,7 @@ static_assert(sizeof(StageInput) == 56, "StageInput changed: update the info enc static_assert(sizeof(StageOutput) == 48, "StageOutput changed: update the info encoder"); static_assert(sizeof(IrBindingLayout) == 80, "IrBindingLayout changed: update the layout encoder"); static_assert(sizeof(IrDescriptorBinding) == 32, "IrDescriptorBinding changed: update the layout encoder"); -static_assert(sizeof(BindingAllocationResult) == 160, "BindingAllocationResult changed: update the allocation encoder"); +static_assert(sizeof(BindingAllocationResult) == 184, "BindingAllocationResult changed: update the allocation encoder"); static_assert(sizeof(CompiledBindingLayout) == 88, "CompiledBindingLayout changed: update the allocation encoder"); static_assert(sizeof(BindingLayout) == 16, "BindingLayout changed: update BuildKey"); static_assert(sizeof(IrResourcePlan) == 712, "IrResourcePlan changed: update EncodePlan and DecodePlan"); @@ -486,6 +486,7 @@ void encodeArtifact(Writer& writer, const CompiledShaderArtifact& result) { writer.Value(result.vertexOffsetConflict); writer.Value(result.instanceOffsetConflict); writer.Value(result.hostSubgroupSize); + writer.Value(result.fullSubgroups); writer.Values(std::span(result.parameterExports)); writer.List(result.fragmentParameters, [](Writer& out, const FragmentParameter& parameter) { out.Value(parameter.location); @@ -532,6 +533,7 @@ void decodeArtifact(Reader& reader, CompiledShaderArtifact& result) { reader.Value(result.vertexOffsetConflict); reader.Value(result.instanceOffsetConflict); reader.Value(result.hostSubgroupSize); + reader.Value(result.fullSubgroups); reader.Values(result.parameterExports); reader.List(result.fragmentParameters, 10, [](Reader& in, FragmentParameter& parameter) { in.Value(parameter.location); @@ -1694,13 +1696,15 @@ void EncodeResult(const RecompileResult& result, std::vector& out) { encodeArtifact(writer, result); encodeInvocation(writer, result); writer.Value(result.workgroupMemoryDwords); + writer.Values(std::span(result.shaderDataUserWords)); } -bool DecodeResult(std::span bytes, RecompileResult& result) { - Reader reader(bytes); +bool DecodeResult(std::span bytes, RecompileResult& result, std::shared_ptr owner) { + Reader reader(bytes, std::move(owner)); decodeArtifact(reader, result); decodeInvocation(reader, result); reader.Value(result.workgroupMemoryDwords); + reader.Values(result.shaderDataUserWords); result.cacheHit = false; return reader.Done() && result.runtimeAbiVersion == RuntimeAbi::Version; } diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index b03d85a809..7322efa496 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -53,7 +53,7 @@ void BuildKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, s [[nodiscard]] std::string EntryName(std::span key); void EncodeResult(const RecompileResult& result, std::vector& out); -[[nodiscard]] bool DecodeResult(std::span bytes, RecompileResult& result); +[[nodiscard]] bool DecodeResult(std::span bytes, RecompileResult& result, std::shared_ptr owner = nullptr); [[nodiscard]] std::vector EncodeEntry(std::span key, const CompiledVariant& variant); [[nodiscard]] LoadStatus DecodeEntry(std::span file, std::span key, CompiledVariant& variant, std::shared_ptr owner = nullptr); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp index 9d22da5885..e4a861594f 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp @@ -5,6 +5,7 @@ #include "Optimization/BindingAllocator.hpp" #include "Optimization/ShaderStageInputInfo.hpp" #include +#include #include #include #include @@ -24,12 +25,21 @@ struct SpirvTargetOptions { bool registrationModules = false; bool fragmentDescriptorSet = false; bool float16DenormPreserve = false; + bool spannedWaves = false; +}; + +struct SpirvEmitInfo { + bool fullSubgroups = false; +}; + +struct SpannedWaveUnsupported : std::runtime_error { + using std::runtime_error::runtime_error; }; class SpirvEmitter { public: [[nodiscard]] std::vector Emit(const IrProgram& program, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target) const; - [[nodiscard]] std::vector Emit(const IrProgram& program, const ShaderStageInputInfo& inputInfo, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target) const; + [[nodiscard]] std::vector Emit(const IrProgram& program, const ShaderStageInputInfo& inputInfo, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target, SpirvEmitInfo* info = nullptr) const; }; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 2ac59b0477..5c89545900 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -100,6 +100,8 @@ struct SpirvEmitterState { std::uint32_t laneCount = 1; std::uint32_t hostSubgroupSize = 0; bool splitSubgroup = false; + bool spannedWave = false; + std::uint32_t spannedExchangeVariable = 0; bool narrowSubgroupClock = false; bool float64DenormPreserve = false; bool registrationModules = false; @@ -185,6 +187,7 @@ struct SpirvEmitterState { std::uint32_t sampleMaskInputVariable = 0; std::uint32_t quadSampleCoverage = 0; std::uint32_t subgroupLocalInvocationIdVariable = 0; + std::uint32_t subgroupIdVariable = 0; std::uint32_t perVertexVariable = 0; std::uint32_t pointSizeVariable = 0; std::uint32_t clipDistanceVariable = 0; @@ -211,6 +214,7 @@ struct SpirvValueEmitContext { std::uint32_t Ballot(const IrValue* predicate); std::uint32_t FirstLane(std::uint32_t ballot); std::uint32_t Shuffle(const IrValue& inst, std::size_t index, std::uint32_t lane); + std::uint32_t ShuffleInHalf(const IrValue& inst, std::size_t index, std::uint32_t lane); std::uint32_t Result(const IrValue& inst); std::uint32_t Define(const IrValue& inst, std::uint32_t value); std::uint32_t ResourceIndex(const IrValue* value, IrOpcode opcode); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp index b38bd34c1e..a24284d37c 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp @@ -8,8 +8,10 @@ namespace ShaderRecompiler { bool HostSubgroupSingleLane(const SpirvEmitterState& state); void EmitSubgroupBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate); +void EmitHostSubgroupBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate); void EmitSubgroupBallotFindLsb(SpirvEmitterState& state, std::uint32_t result, std::uint32_t ballot); void EmitSubgroupShuffle(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane); +void EmitSubgroupShuffleInHalf(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane); void EmitSubgroupElect(SpirvEmitterState& state, std::uint32_t result); std::uint32_t EmitSubgroupLocalInvocationId(SpirvEmitterState& state); std::uint32_t EmitHostSubgroupLane(SpirvEmitterState& state, std::uint32_t lane); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index 757a979fa5..dc224a7a2e 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -186,11 +186,10 @@ std::uint32_t EmitDsMaskedLaneRead(SpirvEmitterState& state, std::uint32_t sourc if (state.laneCount == 2) { lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), lane, ConstantU32(state, 31u)); } - const auto physicalLane = EmitHostSubgroupLane(state, lane); const auto shuffled = state.module.AllocateId(); - EmitSubgroupShuffle(state, TypeU32(state), shuffled, source, physicalLane); + EmitSubgroupShuffleInHalf(state, TypeU32(state), shuffled, source, lane); const auto sourceExec = state.module.AllocateId(); - EmitSubgroupShuffle(state, TypeBool(state), sourceExec, exec, physicalLane); + EmitSubgroupShuffleInHalf(state, TypeBool(state), sourceExec, exec, lane); const auto sourceActive = Binary(state, spv::OpLogicalAnd, TypeBool(state), sourceExec, EmitSubgroupLaneActiveBool(state, lane)); return Select(state, TypeU32(state), sourceActive, shuffled, ConstantU32(state, 0u)); } @@ -566,7 +565,7 @@ std::optional MatchHalfWaveScan(const IrValue* value, IrShaderStag std::optional EmitHalfWaveReduction(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t lane) { auto& state = ctx.state; - if (state.program.WaveSize() != 64u || (lane & 31u) != 31u) return std::nullopt; + if (state.program.WaveSize() != 64u || (lane & 31u) != 31u || state.spannedWave) return std::nullopt; const auto stage = state.program.Resources().stage; auto scan = MatchHalfWaveScan(inst.Argument(0), stage, false); const bool narrow = !scan && HostSubgroupNarrowerThanWave(state) && state.hostSubgroupSize == 32u; @@ -969,7 +968,7 @@ std::uint32_t EmitDppMoveU32(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; const auto flags = inst.Flags(); const auto target = EmitDppTargetLane(state, flags.control); - const auto shuffled = ctx.Shuffle(inst, 0, target.lane); + const auto shuffled = ctx.ShuffleInHalf(inst, 0, target.lane); if (flags.fetchInactive) { return EmitNative(state, target.valid, shuffled, ConstantU32(state, 0u)); } @@ -1079,7 +1078,7 @@ std::uint32_t EmitPermlane16U32(SpirvValueEmitContext& ctx, const IrValue& inst) state.module.AddFunction(spv::OpShiftRightLogical, TypeU32(state), shifted, selected, shift); state.module.AddFunction(spv::OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15u)); state.module.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, rowValue, index); - const auto shuffled = ctx.Shuffle(inst, 0, target); + const auto shuffled = ctx.ShuffleInHalf(inst, 0, target); if (flags.fetchInactive) { return shuffled; } @@ -1107,9 +1106,9 @@ std::uint32_t EmitPermuteU32(SpirvValueEmitContext& ctx, const IrValue& inst) { auto result = ConstantU32(state, 0u); for (std::uint32_t source = 0; source < 32u; ++source) { const auto from = Binary(state, spv::OpBitwiseOr, TypeU32(state), base, ConstantU32(state, source)); - const auto value = ctx.Shuffle(inst, 0, from); - const auto address = ctx.Shuffle(inst, 1, from); - const auto active = ctx.Shuffle(inst, 2, from); + const auto value = ctx.ShuffleInHalf(inst, 0, from); + const auto address = ctx.ShuffleInHalf(inst, 1, from); + const auto active = ctx.ShuffleInHalf(inst, 2, from); const auto target = Binary(state, spv::OpBitwiseAnd, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)), ConstantU32(state, 31u)); const auto hit = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, Binary(state, spv::OpIEqual, TypeBool(state), target, own)); result = Select(state, TypeU32(state), hit, value, result); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index 4b0d60d921..3ae3e3744e 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -1,6 +1,7 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" +#include #include namespace ShaderRecompiler { @@ -115,7 +116,14 @@ void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition) { if (OrderedPixelShader(state)) { state.module.AddFunction(spv::OpEndInvocationInterlockEXT); } - state.module.AddFunction(state.program.Resources().stage == IrShaderStage::Pixel ? spv::OpKill : spv::OpReturn); + if (state.program.Resources().stage != IrShaderStage::Pixel) { + state.module.AddFunction(spv::OpReturn); + } else if (std::ranges::find(state.supportedExtensions, "SPV_KHR_terminate_invocation") != state.supportedExtensions.end()) { + state.module.EmitExtension("SPV_KHR_terminate_invocation"); + state.module.AddFunction(spv::OpTerminateInvocation); + } else { + state.module.AddFunction(spv::OpKill); + } } EmitLabel(state, next); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 2bd877635a..0d629adaca 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -173,6 +173,13 @@ std::uint32_t SpirvValueEmitContext::Shuffle(const IrValue& inst, std::size_t in return value; } +std::uint32_t SpirvValueEmitContext::ShuffleInHalf(const IrValue& inst, std::size_t index, std::uint32_t lane) { + if (!state.spannedWave) return Shuffle(inst, index, lane); + const auto result = state.module.AllocateId(); + EmitSubgroupShuffleInHalf(state, TypeId(state, inst.Argument(index)->Type()), result, Arg(inst, index), lane); + return result; +} + std::uint32_t SpirvValueEmitContext::Result(const IrValue& inst) { if (const auto found = definitions.find(&inst); found != definitions.end()) { return found->second; @@ -233,7 +240,7 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Co return Emit(program, ShaderStageInputInfo {}, bindings, target); } -std::vector SpirvEmitter::Emit(const IrProgram& program, const ShaderStageInputInfo& inputInfo, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target) const { +std::vector SpirvEmitter::Emit(const IrProgram& program, const ShaderStageInputInfo& inputInfo, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target, SpirvEmitInfo* info) const { if (program.Resources().stage != IrShaderStage::Compute && program.Resources().stage != IrShaderStage::Vertex && program.Resources().stage != IrShaderStage::Pixel && program.Resources().stage != IrShaderStage::Mesh && program.Resources().stage != IrShaderStage::Local && program.Resources().stage != IrShaderStage::TessellationControl && program.Resources().stage != IrShaderStage::TessellationEvaluation) { FailProgram(program, "binary SPIR-V emitter received an unsupported shader stage"); } @@ -262,23 +269,31 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.splitSubgroup = program.WaveSize() == 32u && target.subgroupSize > 32u; if (state.splitSubgroup && (state.requirements.subgroupBallot || state.requirements.subgroupShuffle)) state.requirements.subgroupLocalInvocationId = true; const auto* workgroup = ShaderWorkgroupInputFor(state); - state.laneCount = workgroup != nullptr && program.WaveSize() == 64u && workgroup->hostSubgroupSize == 32u ? 2u : 1u; + const bool halfHost = workgroup != nullptr && program.WaveSize() == 64u && workgroup->hostSubgroupSize == 32u; + state.spannedWave = halfHost && target.spannedWaves && program.Resources().stage == IrShaderStage::Compute && !state.requirements.computeDerivatives && std::max(workgroup->threadsNum[0], 1u) * std::max(workgroup->threadsNum[1], 1u) * std::max(workgroup->threadsNum[2], 1u) == 64u; + state.laneCount = halfHost && !state.spannedWave ? 2u : 1u; + if (state.spannedWave) { + state.requirements.subgroupLocalInvocationId = true; + state.requirements.subgroupBallot = true; + state.requirements.subgroupShuffle = true; + } if (state.laneCount == 2u) state.sharedLaneValues = WaveUniformValues(program); if (program.Resources().stage == IrShaderStage::Compute && workgroup != nullptr) { // The key comes from a subgroup ballot (ReadFirstLane), so the slot is uniform over the // workgroup only when the workgroup is one wave held by one host subgroup; a wave64 program // kept at one lane per invocation spans two subgroups (see WaveLdsScope). const auto threads = std::max(workgroup->threadsNum[0], 1u) * std::max(workgroup->threadsNum[1], 1u) * std::max(workgroup->threadsNum[2], 1u); - const bool oneSubgroup = state.laneCount == 2u || (program.WaveSize() == workgroup->hostSubgroupSize && workgroup->hostSubgroupSize <= 32u); + const bool oneSubgroup = state.laneCount == 2u || state.spannedWave || (program.WaveSize() == workgroup->hostSubgroupSize && workgroup->hostSubgroupSize <= 32u); state.tableIndexNonUniform = threads > program.WaveSize() || !oneSubgroup; } state.waveLdsScope = WaveLdsScope(program, workgroup, state.laneCount); if (const char* guard = std::getenv("APS5_LOOP_GUARD")) state.loopGuardLimit = static_cast(std::strtoul(guard, nullptr, 0)); // Stopped invocations would leave the wave LDS barriers incomplete. - state.bdaStopsInvocations = state.waveLdsScope == 0 && BdaInvocationsMayStop(program) && program.Resources().stage != IrShaderStage::Mesh; + state.bdaStopsInvocations = state.waveLdsScope == 0 && !state.spannedWave && BdaInvocationsMayStop(program) && program.Resources().stage != IrShaderStage::Mesh; EmitModuleHeader(state, bindings); EmitProgram(state); state.module.EmitEntryPoint(ExecutionModelForStage(state.program.Resources().stage), state.mainFunc, "main", state.interfaceVariables); + if (info != nullptr) info->fullSubgroups = state.spannedWave; return state.module.Finalize(); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index db0ca5d84b..d2ce577837 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -741,9 +741,6 @@ std::uint32_t StoreTexel(SpirvValueEmitContext& ctx, const ImageEmitAccess& acce auto& state = ctx.state; const auto& mem = access.mem; const auto sint = SintStorageFormat(access.image); - if (sint.format != IrBufferFormat::Invalid && mem.dataBits == 16u) { - ctx.Fail(access.inst, "stores 16-bit data to an image of a SINT format"); - } std::uint32_t values[4] = {}; const auto dmask = EffectiveDmask(mem); for (std::uint32_t component = 0; component < 4u; component++) { @@ -766,7 +763,7 @@ std::uint32_t StoreTexel(SpirvValueEmitContext& ctx, const ImageEmitAccess& acce } values[component] = integer ? raw : mem.dataBits == 16u ? EmitF16BitsToF32(state, raw) : Unary(state, spv::OpBitcast, TypeF32(state), raw); if (component < sint.componentCount) { - values[component] = EmitFormatStoreComponent(state, sint, component, values[component]); + values[component] = mem.dataBits == 16u ? EmitD16StoreComponent(state, sint, component, values[component]) : EmitFormatStoreComponent(state, sint, component, values[component]); } } const auto texel = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp index 168803014c..9b36762390 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp @@ -165,7 +165,7 @@ void DefineInputs(SpirvEmitterState& state) { for (const auto& input : state.program.Info().inputs) { state.inputs.push_back(SpirvInputBinding {input}); } - if (state.laneCount == 2u) { + if (state.laneCount == 2u || state.spannedWave) { const auto addBuiltin = [&](StageInputKind kind, std::uint32_t components, const char* name) { if (std::none_of(state.inputs.begin(), state.inputs.end(), [kind](const SpirvInputBinding& input) { return input.kind == kind; @@ -293,6 +293,10 @@ void DefineInputs(SpirvEmitterState& state) { if (state.requirements.subgroupLocalInvocationId) { const auto variable = DefineInterfaceVariable(state, TypeU32(state), spv::StorageClassInput, "gl_SubgroupInvocationID"); state.subgroupLocalInvocationIdVariable = variable; + if (state.spannedWave) { + state.subgroupIdVariable = DefineInterfaceVariable(state, TypeU32(state), spv::StorageClassInput, "gl_SubgroupID"); + state.module.AddAnnotation(spv::OpDecorate, state.subgroupIdVariable, spv::DecorationBuiltIn, spv::BuiltInSubgroupId); + } state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBuiltIn, spv::BuiltInSubgroupLocalInvocationId); if (state.program.Resources().stage == IrShaderStage::Pixel) { state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationFlat); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp index 849e954bbe..4c05ffe094 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp @@ -196,6 +196,13 @@ namespace ShaderRecompiler } std::uint32_t EmitLocalInvocationIndex(SpirvEmitterState& state) { + if (state.spannedWave) { + const auto subgroup = state.module.AllocateId(); + const auto invocation = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), subgroup, state.subgroupIdVariable); + state.module.AddFunction(spv::OpLoad, TypeU32(state), invocation, state.subgroupLocalInvocationIdVariable); + return EmitAddU32(state, EmitBinaryU32(state, spv::OpShiftLeftLogical, subgroup, ConstantU32(state, 5u)), invocation); + } const auto variable = InputVariableForKind(state, StageInputKind::LocalInvocationIndex); if (variable == 0) { FailEmit("LocalInvocationIndex was not declared"); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index b7ee109d7f..e152d36092 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1365,7 +1365,7 @@ std::uint32_t LockedLdsUpdate(SpirvEmitterState& state, std::uint32_t type, std: state.module.AddFunction(spv::OpPhi, TypeBool(state), doneNext, served, servedMerge, done, body); state.module.AddFunction(spv::OpPhi, type, resultNext, servedResult, servedMerge, result, body); const auto ballot = state.module.AllocateId(); - EmitSubgroupBallot(state, ballot, Unary(state, spv::OpLogicalNot, TypeBool(state), doneNext)); + EmitHostSubgroupBallot(state, ballot, Unary(state, spv::OpLogicalNot, TypeBool(state), doneNext)); std::uint32_t remaining = ConstantU32(state, 0u); for (std::uint32_t component = 0; component < 4u; ++component) { const auto word = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp index 78de6340b4..763f1a1fa8 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp @@ -240,7 +240,7 @@ void DefineModule(SpirvEmitterState& state) { state.meshGuestFunc = state.module.AllocateId(); state.module.EmitCapability(spv::CapabilityMeshShadingEXT); state.module.EmitExtension("SPV_EXT_mesh_shader"); - state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeOutputTrianglesEXT); + state.module.AddExecutionMode(state.mainFunc, mesh.outputPrimitiveSize == 2u ? spv::ExecutionModeOutputLinesEXT : spv::ExecutionModeOutputTrianglesEXT); state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeOutputVertices, mesh.maxVertices); state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeOutputPrimitivesEXT, mesh.maxPrimitives); } @@ -343,6 +343,11 @@ void DefineModule(SpirvEmitterState& state) { std::uint32_t localX = workgroup->threadsNum[0] != 0u ? workgroup->threadsNum[0] : derivativeDefault; std::uint32_t localY = workgroup->threadsNum[1] != 0u ? workgroup->threadsNum[1] : derivativeDefault; std::uint32_t localZ = workgroup->threadsNum[2] != 0u ? workgroup->threadsNum[2] : 1u; + if (state.spannedWave) { + localX = 64u; + localY = 1u; + localZ = 1u; + } if (state.laneCount == 2) { localX = ((localX * localY * localZ + 63u) / 64u) * 32u; localY = 1u; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp index 6efac49c44..055881c70e 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp @@ -26,6 +26,59 @@ namespace ShaderRecompiler return value; } + std::uint32_t SpannedLane(SpirvEmitterState& state) { + return EmitBinaryU32(state, spv::OpBitwiseAnd, EmitLocalInvocationIndex(state), ConstantU32(state, 63u)); + } + + std::uint32_t SpannedSlot(SpirvEmitterState& state, std::uint32_t index) { + if (state.spannedExchangeVariable == 0u) { + state.spannedExchangeVariable = state.module.DefineGlobalVariable(TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, 64u), spv::StorageClassWorkgroup); + state.module.AddName(state.spannedExchangeVariable, "wave_exchange"); + } + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, spv::StorageClassWorkgroup), pointer, state.spannedExchangeVariable, index); + return pointer; + } + + void SpannedBarrier(SpirvEmitterState& state) { + state.module.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask)); + } + + std::uint32_t SpannedLoad(SpirvEmitterState& state, std::uint32_t index) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, SpannedSlot(state, index)); + return value; + } + + void SpannedBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate) { + const auto lane = SpannedLane(state); + SpannedBarrier(state); + state.module.AddFunction(spv::OpAtomicStore, SpannedSlot(state, EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 1u))), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, 0u)); + SpannedBarrier(state); + const auto bit = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 1u), EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 31u))); + const auto previous = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicOr, TypeU32(state), previous, SpannedSlot(state, EmitBinaryU32(state, spv::OpShiftRightLogical, lane, ConstantU32(state, 5u))), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsMaskNone), Select(state, TypeU32(state), predicate, bit, ConstantU32(state, 0u))); + SpannedBarrier(state); + const auto low = SpannedLoad(state, ConstantU32(state, 0u)); + const auto high = SpannedLoad(state, ConstantU32(state, 1u)); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4u), result, low, high, ConstantU32(state, 0u), ConstantU32(state, 0u)); + } + + void SpannedShuffle(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane) { + const bool boolean = type == TypeBool(state); + if (!boolean && type != TypeU32(state) && type != TypeF32(state)) throw SpannedWaveUnsupported("a spanned wave shuffles only 32-bit and boolean values"); + std::uint32_t word = value; + if (boolean) word = Select(state, TypeU32(state), value, ConstantU32(state, 1u), ConstantU32(state, 0u)); + else if (type != TypeU32(state)) word = Unary(state, spv::OpBitcast, TypeU32(state), value); + SpannedBarrier(state); + state.module.AddFunction(spv::OpStore, SpannedSlot(state, SpannedLane(state)), word); + SpannedBarrier(state); + const auto read = SpannedLoad(state, EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 63u))); + if (boolean) state.module.AddFunction(spv::OpINotEqual, type, result, read, ConstantU32(state, 0u)); + else if (type != TypeU32(state)) state.module.AddFunction(spv::OpBitcast, type, result, read); + else state.module.AddFunction(spv::OpCopyObject, type, result, read); + } + } bool HostSubgroupSingleLane(const SpirvEmitterState& state) { @@ -33,6 +86,14 @@ namespace ShaderRecompiler } void EmitSubgroupBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate) { + if (state.spannedWave) { + SpannedBallot(state, result, predicate); + return; + } + EmitHostSubgroupBallot(state, result, predicate); + } + + void EmitHostSubgroupBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate) { if (HostSubgroupSingleLane(state)) { const auto zero = ConstantU32(state, 0u); const auto own = Select(state, TypeU32(state), predicate, ConstantU32(state, 1u), zero); @@ -43,6 +104,19 @@ namespace ShaderRecompiler } void EmitSubgroupBallotFindLsb(SpirvEmitterState& state, std::uint32_t result, std::uint32_t ballot) { + if (state.spannedWave) { + const auto low = state.module.AllocateId(); + const auto high = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, ballot, 0u); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, ballot, 1u); + const auto lowFirst = state.module.AllocateId(); + const auto highFirst = state.module.AllocateId(); + state.module.AddFunction(spv::OpExtInst, TypeU32(state), lowFirst, GlslStd450(state), GLSLstd450FindILsb, low); + state.module.AddFunction(spv::OpExtInst, TypeU32(state), highFirst, GlslStd450(state), GLSLstd450FindILsb, high); + const auto lowActive = Binary(state, spv::OpINotEqual, TypeBool(state), low, ConstantU32(state, 0u)); + state.module.AddFunction(spv::OpSelect, TypeU32(state), result, lowActive, lowFirst, EmitAddU32(state, highFirst, ConstantU32(state, 32u))); + return; + } if (HostSubgroupSingleLane(state)) { const auto own = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), own, ballot, 0u); @@ -53,6 +127,10 @@ namespace ShaderRecompiler } void EmitSubgroupShuffle(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane) { + if (state.spannedWave) { + SpannedShuffle(state, type, result, value, lane); + return; + } if (HostSubgroupSingleLane(state)) { state.module.AddFunction(spv::OpCopyObject, type, result, value); return; @@ -60,6 +138,14 @@ namespace ShaderRecompiler state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, result, ConstantU32(state, spv::ScopeSubgroup), value, lane); } + void EmitSubgroupShuffleInHalf(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane) { + if (!state.spannedWave) { + EmitSubgroupShuffle(state, type, result, value, EmitHostSubgroupLane(state, lane)); + return; + } + state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, result, ConstantU32(state, spv::ScopeSubgroup), value, EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 31u))); + } + void EmitSubgroupElect(SpirvEmitterState& state, std::uint32_t result) { if (HostSubgroupSingleLane(state)) { state.module.AddFunction(spv::OpCopyObject, TypeBool(state), result, ConstantBool(state, true)); @@ -69,12 +155,14 @@ namespace ShaderRecompiler } std::uint32_t EmitSubgroupLocalInvocationId(SpirvEmitterState& state) { + if (state.spannedWave) return SpannedLane(state); const auto value = HostInvocationId(state); if (state.splitSubgroup) return EmitBinaryU32(state, spv::OpBitwiseAnd, value, ConstantU32(state, 31u)); return state.laneHalf == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32)); } std::uint32_t EmitHostSubgroupLane(SpirvEmitterState& state, std::uint32_t lane) { + if (state.spannedWave) return lane; if (!state.splitSubgroup) return lane; const auto base = EmitBinaryU32(state, spv::OpBitwiseAnd, HostInvocationId(state), ConstantU32(state, ~31u)); return EmitAddU32(state, base, lane); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 774e16aa09..1f76ed7314 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -51,7 +51,7 @@ std::uint32_t EmitBuiltinU32(SpirvEmitterState& state, StageInputKind kind, std: if (kind == StageInputKind::LayerExported) { return state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::LayerExported, 0u); } - if (state.laneCount == 2 && (kind == StageInputKind::LocalInvocationId || kind == StageInputKind::GlobalInvocationId)) { + if ((state.laneCount == 2 || state.spannedWave) && (kind == StageInputKind::LocalInvocationId || kind == StageInputKind::GlobalInvocationId)) { const auto* workgroup = ShaderWorkgroupInputFor(state); if (workgroup == nullptr) { throw std::runtime_error("workgroup input info is missing for a local/global invocation builtin"); @@ -767,11 +767,11 @@ void DefineMeshOutputs(SpirvEmitterState& state) { } state.meshAllocation = MeshArray(state, spv::StorageClassWorkgroup, TypeU32(state), 2u); state.meshPrimitiveData = MeshArray(state, spv::StorageClassPrivate, TypeU32(state), state.laneCount); - state.meshPrimitives = MeshArray(state, spv::StorageClassOutput, TypeU32Vector(state, 3u), mesh.maxPrimitives); + state.meshPrimitives = MeshArray(state, spv::StorageClassOutput, TypeU32Vector(state, mesh.outputPrimitiveSize), mesh.maxPrimitives); state.meshCull = MeshArray(state, spv::StorageClassOutput, TypeBool(state), mesh.maxPrimitives); state.interfaceVariables.push_back(state.meshPrimitives); state.interfaceVariables.push_back(state.meshCull); - state.module.AddAnnotation(spv::OpDecorate, state.meshPrimitives, spv::DecorationBuiltIn, spv::BuiltInPrimitiveTriangleIndicesEXT); + state.module.AddAnnotation(spv::OpDecorate, state.meshPrimitives, spv::DecorationBuiltIn, mesh.outputPrimitiveSize == 2u ? spv::BuiltInPrimitiveLineIndicesEXT : spv::BuiltInPrimitiveTriangleIndicesEXT); state.module.AddAnnotation(spv::OpDecorate, state.meshCull, spv::DecorationBuiltIn, spv::BuiltInCullPrimitiveEXT); state.module.AddAnnotation(spv::OpDecorate, state.meshCull, spv::DecorationPerPrimitiveEXT); if (state.numWorkgroupsVariable == 0u) { @@ -837,16 +837,16 @@ void EmitMeshEntryPoint(SpirvEmitterState& state) { EmitIfCondition(state, isPrimitive, [&]() { const auto packed = MeshLoad(state, state.meshPrimitiveData, spv::StorageClassPrivate, TypeU32(state), ConstantU32(state, half)); std::array vertex {}; - for (std::uint32_t component = 0; component < 3u; component++) { + for (std::uint32_t component = 0; component < mesh.outputPrimitiveSize; component++) { const auto field = state.module.AllocateId(); state.module.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), field, packed, ConstantU32(state, component * 10u), ConstantU32(state, 9u)); vertex.at(component) = state.module.AllocateId(); state.module.AddFunction(spv::OpExtInst, TypeU32(state), vertex.at(component), GlslStd450(state), GLSLstd450UMin, field, lastVertex); } - const auto triangle = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 3u), triangle, vertex[0], vertex[1], vertex[2]); - const auto trianglePointer = MeshElement(state, state.meshPrimitives, spv::StorageClassOutput, TypeU32Vector(state, 3u), index); - state.module.AddFunction(spv::OpStore, trianglePointer, triangle); + const auto indices = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, mesh.outputPrimitiveSize), indices, std::span(vertex.data(), mesh.outputPrimitiveSize)); + const auto indicesPointer = MeshElement(state, state.meshPrimitives, spv::StorageClassOutput, TypeU32Vector(state, mesh.outputPrimitiveSize), index); + state.module.AddFunction(spv::OpStore, indicesPointer, indices); const auto nullBit = EmitBinaryU32(state, spv::OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u)); const auto culled = state.module.AllocateId(); state.module.AddFunction(spv::OpINotEqual, TypeBool(state), culled, nullBit, ConstantU32(state, 0u)); diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index c174855b88..c0e248f919 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -74,6 +74,7 @@ private: IrU32 flushF16Result(IrU32 bits); IrF32 flushF16InputValue(IrF32 value); IrU32 convertF32ToF16Bits(IrF32 value, std::initializer_list sources); + IrU32 flushTinyF16Conversion(IrF32 value, IrU32 bits); void writeF16Conversion(const RdnaOperand& operand, IrF32 value, std::initializer_list sources); IrU32 readU32(const RdnaOperand& operand); IrU32 flushF32Denormal(IrU32 bits); diff --git a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp index 5de4543f7b..992dcf3388 100644 --- a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp +++ b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp @@ -217,7 +217,8 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues target.maxPrimitives = mesh->maxPrimitives; target.provokingVertex = mesh->provokingVertex; target.esgsItemSize = mesh->esgsItemSize; - if (target.threadsNum[0] == 0u || target.maxVertices == 0u || target.maxPrimitives == 0u || target.provokingVertex > 2u) throw std::runtime_error("ShaderInputInfoBuilder: invalid mesh configuration"); + target.outputPrimitiveSize = mesh->outputPrimitiveSize; + if (target.threadsNum[0] == 0u || target.maxVertices == 0u || target.maxPrimitives == 0u || (target.outputPrimitiveSize != 2u && target.outputPrimitiveSize != 3u) || target.provokingVertex >= target.outputPrimitiveSize) throw std::runtime_error("ShaderInputInfoBuilder: invalid mesh configuration"); } ShaderStageInputInfo result; result.vertex = &vertexStorage; diff --git a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp index cf6c3a789f..f64a0c72ee 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp @@ -165,7 +165,7 @@ void TranslationContext::write16Bits(const RdnaOperand& operand, IrU32 value) { } bool TranslationContext::directedF16Rounding() const { - return floatMode.has_value() && ((floatMode->floatMode >> 2u) & 3u) != 0u && ((floatMode->floatMode >> 6u) & 3u) == 3u; + return floatMode.has_value() && ((floatMode->floatMode >> 2u) & 3u) != 0u; } IrU32 TranslationContext::flushF16Bits(IrU32 bits, std::uint32_t keepBit) { @@ -220,12 +220,28 @@ IrU32 TranslationContext::convertF32ToF16Bits(IrF32 value, std::initializer_list return IrU32(ir.Select(ir.ULessThan(magnitude.Value(), ir.Constant(0x7f800000u)), result.Value(), native.Value())); } +IrU32 TranslationContext::flushTinyF16Conversion(IrF32 value, IrU32 bits) { + if (!floatMode.has_value() || (floatMode->floatMode & 0x80u) != 0u) return bits; + const std::uint32_t rounding = (floatMode->floatMode >> 2u) & 3u; + const IrU32 word(ir.BitCastU32(value.Value())); + const IrU32 magnitude(ir.BitwiseAnd(word.Value(), ir.Constant(0x7fffffffu))); + const IrU32 sign(ir.BitwiseAnd(ir.ShiftRightLogical(word.Value(), ir.Constant(16u)), ir.Constant(0x8000u))); + IrValue* limit = &ir.Constant(rounding == 0u ? 0x387ff000u : 0x38800000u); + if (rounding == 1u || rounding == 2u) { + const IrU1 away(ir.IEqual(sign.Value(), ir.Constant(rounding == 1u ? 0u : 0x8000u))); + limit = &ir.Select(away.Value(), ir.Constant(0x387fe001u), ir.Constant(0x38800000u)); + } + const IrU1 tiny(ir.ULessThan(magnitude.Value(), *limit)); + return IrU32(ir.Select(tiny.Value(), sign.Value(), bits.Value())); +} + void TranslationContext::writeF16Conversion(const RdnaOperand& operand, IrF32 value, std::initializer_list sources) { - if (!directedF16Rounding()) { + if (!directedF16Rounding() && (!floatMode.has_value() || (floatMode->floatMode & 0x80u) != 0u)) { writeF16(operand, value, sources); return; } - write16Bits(operand, convertF32ToF16Bits(applyF16ResultModifiers(operand, value), sources)); + const IrF32 modified = applyF16ResultModifiers(operand, value); + write16Bits(operand, flushTinyF16Conversion(modified, convertF32ToF16Bits(modified, sources))); } void TranslationContext::writeF16(const RdnaOperand& operand, IrF32 value, std::initializer_list sources) { diff --git a/core/shader/recompiler/UserDataUse.hpp b/core/shader/recompiler/UserDataUse.hpp new file mode 100644 index 0000000000..c193714270 --- /dev/null +++ b/core/shader/recompiler/UserDataUse.hpp @@ -0,0 +1,19 @@ +#ifndef CORE_SHADER_RECOMPILER_USERDATAUSE_HPP +#define CORE_SHADER_RECOMPILER_USERDATAUSE_HPP + +#include +#include +#include + +namespace ShaderRecompiler { + +struct SrtUserDataUse { + static constexpr std::uint32_t TrackedWords = 64; + std::uint64_t asBase = 0; + std::uint64_t asValue = 0; + std::vector> reads; +}; + +} + +#endif diff --git a/core/shader/recompiler/tests/RequestSerializerTests.cpp b/core/shader/recompiler/tests/RequestSerializerTests.cpp index b8d4b0e10e..f30aae1501 100644 --- a/core/shader/recompiler/tests/RequestSerializerTests.cpp +++ b/core/shader/recompiler/tests/RequestSerializerTests.cpp @@ -206,11 +206,11 @@ int main() { Require(request.graphics->draw.indexAddress == 0xabcdef3400u && request.graphics->draw.indexCount == 57u && request.graphics->draw.indexElementBytes == 4u && request.graphics->draw.instanceCount == 9u); Require(request.target.float16DenormPreserve == preserve); auto upgraded = bytes; - upgraded[4] = 30u; + upgraded[4] = 32u; if (fixture.pixel) upgraded.insert(upgraded.begin() + 258u, {0u, 0u, 0u}); if (fixture.pixel) InsertPixelDefaultsAfterFrontFace(upgraded); if (fixture.tessellation) upgraded.insert(upgraded.end() - 35u, 5u, 0u); - upgraded.insert(upgraded.end(), sizeof(std::uint64_t), 0u); + upgraded.insert(upgraded.end(), sizeof(std::uint64_t) + 1u, 0u); Require(serializer.Serialize(request) == EncodeBase64(upgraded)); auto truncated = bytes; truncated.pop_back(); @@ -245,11 +245,11 @@ int main() { layerRejected = std::string_view(error.what()) == "RequestSerializer: truncated data"; } Require(layerRejected); - layered[4] = 30u; + layered[4] = 32u; layered.insert(layered.begin() + 259u, {0u, 0u}); InsertPixelDefaultsAfterFrontFace(layered); if (fixture.tessellation) layered.insert(layered.end() - 35u, 5u, 0u); - layered.insert(layered.end(), sizeof(std::uint64_t), 0u); + layered.insert(layered.end(), sizeof(std::uint64_t) + 1u, 0u); Require(serializer.Serialize(layerReplay.request) == EncodeBase64(layered)); } } @@ -270,10 +270,10 @@ int main() { Require(replay.request.context.pixel->rasterPrimitive == ShaderRecompiler::RasterPrimitive::Rectangle); Require(replay.request.context.vertex->paClVsOutCntl == 0x00050a05u); Require(replay.request.target.float16DenormPreserve == preserve); - bytes[4] = 30u; + bytes[4] = 32u; bytes.insert(bytes.begin() + 259u, {0u, 0u}); InsertPixelDefaultsAfterFrontFace(bytes); - bytes.insert(bytes.end(), sizeof(std::uint64_t), 0u); + bytes.insert(bytes.end(), sizeof(std::uint64_t) + 1u, 0u); Require(serializer.Serialize(replay.request) == EncodeBase64(bytes)); auto truncated23 = DecodeBase64(encoded23); @@ -394,10 +394,10 @@ int main() { Require(request.graphics->draw.indexAddress == 0xabcdef3400u && request.graphics->draw.indexCount == 57u && request.graphics->draw.indexElementBytes == 4u && request.graphics->draw.instanceCount == 9u); Require(request.target.float16DenormPreserve == preserve); auto upgraded = bytes; - upgraded[4] = 30u; + upgraded[4] = 32u; if (fixture.pixel) upgraded.insert(upgraded.begin() + 259u, {0u, 0u}); if (fixture.pixel) InsertPixelDefaultsAfterFrontFace(upgraded); - upgraded.insert(upgraded.end(), sizeof(std::uint64_t), 0u); + upgraded.insert(upgraded.end(), sizeof(std::uint64_t) + 1u, 0u); Require(serializer.Serialize(request) == EncodeBase64(upgraded)); for (const bool dither : {false, true}) { if (!fixture.pixel && dither) continue; @@ -407,12 +407,12 @@ int main() { const auto replay25 = serializer.Deserialize(EncodeBase64(main25)); Require(!fixture.pixel || (replay25.request.context.pixel->layerExported == layer && replay25.request.context.pixel->ditheredAlphaToCoverage == dither && replay25.request.context.pixel->reversedBlend == 0u)); Require(replay25.request.target.float16DenormPreserve == preserve && (!fixture.tessellation || replay25.request.graphics->tessellation->primitiveGroupSize == 37u)); - main25[4] = 30u; + main25[4] = 32u; if (fixture.pixel) { main25.insert(main25.begin() + 260u, 0u); InsertPixelDefaultsAfterFrontFace(main25); } - main25.insert(main25.end(), sizeof(std::uint64_t), 0u); + main25.insert(main25.end(), sizeof(std::uint64_t) + 1u, 0u); Require(serializer.Serialize(replay25.request) == EncodeBase64(main25)); } } @@ -426,9 +426,9 @@ int main() { const auto replay = serializer.Deserialize(encoded); Require(replay.request.target.float16DenormPreserve == preserve); auto bytes = DecodeBase64(encoded); - Require(bytes.size() >= 8u && bytes[4] == 30u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u); + Require(bytes.size() >= 8u && bytes[4] == 32u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u); auto future = bytes; - future[4] = 31u; + future[4] = 33u; bool futureRejected = false; try { static_cast(serializer.Deserialize(EncodeBase64(future))); @@ -446,5 +446,10 @@ int main() { Require(rejected); } + for (const bool spanned : {false, true}) { + request.target.spannedWaves = spanned; + Require(serializer.Deserialize(serializer.Serialize(request)).request.target.spannedWaves == spanned); + } + return 0; } diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 7c9dd25c52..661128c3f9 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -95,11 +95,13 @@ flowchart LR - [`Recompiler.cpp`](../../core/shader/recompiler/Recompiler.cpp) runs the stages in this order. With `ANYPS5_ENABLE_SPIRV_TOOLS`, the SPIR-V is also validated and optimized with SPIRV-Tools. - `ShaderRecompiler::Recompile` keeps compiled variants in memory, and `ShaderDiskCache` stores them on disk so later runs reuse them. Before the writable cache, a miss is also looked up in read-only prebuilt packs and loose entry folders under `shader_cache/prebuilt`; pack entries are ordinary encoded cache entries behind a sorted key-hash index, validated by magic, compiler source version and index checksum when mapped, and a miss or rejected entry falls through to the writable cache and the compiler. Cache format 35 also stores the finite stencil-reference set, its replay-safety proof and reserved push-data word in the artifact and binding layouts. Entries from formats 31 to 34 are rejected. +- The Vulkan pipelines are journaled as well ([`PipelineJournal.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp)): a graphics pipeline the store builds (`CachedPipeline`) or a compute pipeline a dispatch builds is appended to `pipelines.journal` next to the shader cache entries, as the materialized result of each stage (the specialized SPIR-V, bindings and push constants, through `ShaderDiskCache::EncodeResult`) plus the recipe the pipeline was built from (`GraphicsPipelineRecipe`: the stages' identities, vertex input layout, descriptor set layout keys and the state fields the `Pipeline` constructor reads, `CanonicalPipelineState`). When a `VulkanDevice` is created, the journal is loaded and its pipelines are built on a pool of `hardware_concurrency() / 2` threads (`APS5_PIPELINE_JOURNAL_THREADS`) while the title loads, so the first draw or dispatch of a configuration seen on an earlier run finds its pipeline resident. The pipeline stores key on `RecompileResult::Identity`, a hash of the module words, specialization constants, subgroup width and bindings that is the same on every run (`SharedSpirv::ContentHash`, computed once per shared module), rather than on the process-lifetime variant ids, and a graphics pipeline is built outside the store's mutex, so the replay threads and the queue workers build in parallel. A draw that arrives before its replayed entry builds the pipeline itself; the first inserted wins. A replayed entry enters the store at its least recently used end and a full store takes none, so the title's own pipelines are never evicted for a replayed one, and the optimized relink of a replayed pipeline (`OptimizedPipeline::Schedule`) waits for the first draw that binds it, so the relink thread serves the title's pipelines first. A device created without a window starts its replay at its first draw or dispatch rather than at creation, since `Driver::AttachWindow` usually replaces it before the title draws. The journal records a pipeline once (modules and recipes are deduplicated by content), keeps at most `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024), replays the first `APS5_PIPELINE_JOURNAL_ENTRIES` recorded (1024, the pipeline store's bound, `APS5_PIPELINE_STORE_ENTRIES`; a run reaches those first), and is disabled with `APS5_NO_PIPELINE_JOURNAL=1` or without a shader cache directory. The file header names the device (`PipelineJournalDeviceFingerprint`, the properties the driver's pipeline cache file is named by), so a journal written for another GPU or driver is started over rather than replayed; appends and reads hold `pipelines.journal.lock` against another process sharing the directory, and a device cancels its replay before it is replaced (`PrepareForReplacement`) or destroyed, so a replaced headless device never inserts entries the window's device then sweeps. `APS5_PROFILE_DRAW=1` prints a `[pipeline-journal]` summary when the replay finishes and at teardown. - A compute shader whose LDS plus its lock dword exceeds the host's `maxComputeSharedMemorySize` is not given a `Workgroup` array. `Recompile` sets `ShaderInfo::sharedMemoryBytes` for it and the SPIR-V backend addresses the LDS in a coherent storage buffer at descriptor set 1 (`WorkgroupMemoryDescriptorSet`), with a slice of `RecompileResult::workgroupMemoryDwords` per workgroup, indexed by `WorkgroupID` and `NumWorkGroups`. `VulkanDevice` keeps one such buffer, sized from the dispatch dimensions, and binds it after the resource set; programs that fit keep using shared memory. - `Driver::RegisterShader` splits shader preparation in two. `PlanRegistered` runs on the registering guest thread: it validates the registers and entry point, decodes the code and builds the recompile request, so invalid registrations still fail there. The RDNA to IR translation and resource plan (`PrepareShader`) run on a background pool of `hardware_concurrency() / 2` detached threads (at least 2, 64 MiB stacks). -- Queue 0's packet worker decodes, captures and prepares a draw and hands its recording (`Graphics::Draw` under the GPU lock) to a committer thread in [`DrawPipeline.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp), so the next packet is decoded while the previous draw is recorded. Each queued item carries the guest ranges it will write (`DrawWriteRanges`: written buffers and storage images, color targets with their DCC and CMASK, depth and stencil). A later packet that reads or writes one of those ranges, or that the pipeline can't order, drains it first; labels, submission ends, buffer fills and (with `APS5_PIPELINE_DMA=1`) DMA packets are queued as items instead. An exception thrown by a commit is rethrown on the worker at its next enqueue or drain, as an inline draw error would be. On by default; `APS5_PIPELINED_DRAWS=0` records every draw inline, and `APS5_EXACT_DRAW_WRITES=0` widens the color and depth ranges to estimates. +- Queue 0's packet worker decodes, captures and prepares a draw and hands its recording (`Graphics::Draw` under the GPU lock) to a committer thread in [`DrawPipeline.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp), so the next packet is decoded while the previous draw is recorded. Each queued item carries the guest ranges it will write (`DrawWriteRanges`: written buffers and storage images, color targets with their DCC and CMASK, depth and stencil). A later packet that reads or writes one of those ranges, or that the pipeline can't order, drains it first; labels, submission ends, buffer fills and (with `APS5_PIPELINE_DMA=1`) DMA packets are queued as items instead. With `APS5_PIPELINE_DISPATCH=1` (off by default), a `DISPATCH_DIRECT` whose dispatch-cache entry hits with a recipe is queued as an item too, carrying the dispatch's written buffers and storage images (`DispatchWriteRanges`); the capture and the cache lookup stay on the worker and the recipe is recorded on the committer. A dispatch still drains when its lookup misses, when what it reads overlaps a write still pending, when the hit is verified against data or when its writes cannot be ranged (bindless access, GDS, fault buffer). An exception thrown by a commit is rethrown on the worker at its next enqueue or drain, as an inline draw error would be. On by default; `APS5_PIPELINED_DRAWS=0` records every draw inline, and `APS5_EXACT_DRAW_WRITES=0` widens the color and depth ranges to estimates. - The shader registry maps a code address to every header registered for that code, in registration order. Static, graphics and rectangle ABI resolution pick the header at the bound address (or one equal to it, `SameHeader`, which also takes a hull header fused by `sceAgcFuseShaderHalves` (type 3) as the registered back half (type 7) it copies, the fused registers aside; graphics ABI resolution then uses the fused header's shader registers); draws and dispatches pick the program by the stage's binary type (`RegisteredProgram`), and a front entry matching both type 2 and type 4 throws. Within a type the last registration wins; re-registering an identical earlier header makes its snapshot current again without preparing it again, and changed code drops the older headers. - The snapshot is published as pending. `SourceHandleFor`, `InvocationFor` and `ShaderPreparationTransaction::Edit` / `Read` wait for the pending job before reading `PreparedShaders`, and rethrow its failure. `APS5_SYNC_SHADER_PREPARE=1` prepares on the registering thread instead. +- `APS5_LAZY_SHADER_PREPARE=1` retains registration validation and decoding but discards the temporary plan without queueing preparation. The snapshot is deferred, not pending; draws and dispatches prepare and cache it on first use. Explicit ABI resolution still prepares its requested stages. This opt-in mode takes precedence over synchronous registration preparation and trades background CPU/memory use for possible first-use stalls; disk caches and shader output are unchanged. - On devices with graphics pipeline libraries (`SpirvTarget::registrationModules`), a vertex or pixel artifact is emitted in registration form: the buffer V# words, image modes, swizzles and mip ranges, depth-compare state, export mappings and vertex component selectors it used to specialize on per draw are read at run time (image `ResourceMetadata` and a few state dwords in the shader data), descriptor heaps keep every element in place (an unused one gets a null or placeholder descriptor), and the pixel stage's bindings live in descriptor set 1 so each stage's library depends only on its own set layout (`INDEPENDENT_SETS`). An artifact above `registrationWordLimit` SPIR-V words is re-emitted specialized, and `CompiledShaderArtifact::registrationModule` records which form it has. For artifacts in registration form, `RegistrationResult` materializes the module once, and `PrebuildStageLibrary` builds its pre-rasterization or fragment-shader library on a separate one-thread pool (`APS5_PREBUILD_THREADS`) with a fixed default state, pairing it with a stub stage. A later draw with that state finds both libraries in the store and only links. - A draw or dispatch whose state matches no artifact prepared at registration (a title that builds its static ABI state itself, without `sceAgcDriverResolve*Abi`) gets one prepared at use by `SourceHandleFor` or `InvocationFor`, logged once per artifact and kept in the snapshot for the next use. - A compute shader whose `s_swappc_b64` calls a user-data pointer cannot be linked at registration, because the callee's address is only known at dispatch. `InvocationFor` in [`ShaderRegistry.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) reads each callee from guest memory, appends one copy per call after the caller (`LinkUserDataCalls`) and prepares that linked code, so a different callee gets its own prepared entry. @@ -107,7 +109,26 @@ flowchart LR - A draw whose `PA_SC_AA_CONFIG` selects 2, 4 or 8 samples renders into host-only multisampled images. [`MultisampleTarget.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) keeps one image per device, surface address, format, extent and sample count (`MultisampleTargetView`); the pipeline, render pass and library keys carry `State::samples`, and a multisampled depth target is a multisampled depth surface. The guest sample planes and FMASK are never read or written, so the registry also guards the target's guest range: a CPU read (`GuestMemory::SetReadGuard`), a DMA copy, and a sampled or storage image read of it throw. A `CB_COLOR_CONTROL` resolve mode over a multisampled CB0 runs as a render pass whose resolve attachment is CB1's resident image (`ResolveMultisampleTarget`). A zero fill of the target's CMASK (`NoteColorMetadataFill`, from fill kernels and CP DMA fills) clears the image at its next use. - A texture over a depth surface samples the depth image itself when it views one 2D slice of the plane's own format (`DepthSurfaceTexture` in [`DepthSurface.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp)). A view of several slices (a 2D array or a cube) or an integer view of a D16 plane is served by a `DepthPlaneCopy`, an image the slices are copied into at every use: from the depth surface bound at each slice's address, else from guest memory. Such a texture is never kept in the resource templates (`RefreshedPerUse`). - Guest memory reaches the GPU through host imports (`VK_EXT_external_memory_host`) where the driver accepts them, and through copies otherwise ([`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp)). A registered range larger than the import window (64 MiB) is imported per window for buffer and fill accesses and whole for an address-based build. Live imports stay under a budget, with the least recently used import retired first (see TechnicalDebt, Host import windows and budget). -- On Windows, a host import of private `MEM_WRITE_WATCH` memory (singly mapped direct memory) stays on `GetWriteWatch` when `ProbeImportWriteProtection` shows the driver reports nothing written after the second GPU read and a GPU write, and a CPU store is collected (`decideImportWatch`). The driver marks every page of such an import written, asynchronously, until a GPU use of it completes (a read on the settle queue ends it even while main-queue work that used the import first is still running, and the first use of an earlier import of the same pages after that read marks nothing), so `settleImport` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) copies 4 bytes out of the new import on a second queue of the graphics family, or on a queue of another copy-capable family when the graphics family has only one (`Context::settleQueue`, own pool and mutex) inside `ImportWatched`'s window and waits for it, for imports of 1 MiB or more; the main queue can't be used there because `importAllocation` holds the imports lock and `GpuMutex` comes before it in the lock order. A smaller import is compared when it is made, so a title that maps, imports and unmaps a small buffer around every submission does not block its mapping thread. Its 4-byte read is queued instead: the next import lookup that holds `GpuMutex` while the imports match the registry (`settleUnwatchedImports`) records the queued reads into one batch on the settle queue, and once the batch's fence has signalled a later lookup returns the import's whole 64 KiB blocks to the write watch (`GuestMemory::Rewatch` drops their marks and stamps every block as a CPU store), except while storage results are pending over them. A settled import whose range had already left the watch returns the same way, and an import retired while its read is in flight is kept with that batch and freed once the batch has completed, without waiting for it; a guest unmap of its memory waits for the batch instead. Imports through a section alias keep tracking on the guest views, and every other Windows host import is compared. A device without such a queue keeps the comparisons. +- On Windows, a host import of private `MEM_WRITE_WATCH` memory (singly mapped direct memory) stays on `GetWriteWatch` when `ProbeImportWriteProtection` shows the driver reports nothing written after the second GPU read and a GPU write, and a CPU store is collected (`decideImportWatch`). The driver marks every page of such an import written, asynchronously, until a GPU use of it completes (a read on the settle queue ends it even while main-queue work that used the import first is still running, and the first use of an earlier import of the same pages after that read marks nothing), so `settleImport` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) copies 4 bytes out of the new import on a second queue of the graphics family, or on a queue of another copy-capable family when the graphics family has only one (`Context::settleQueue`, own pool and mutex) inside `ImportWatched`'s window and waits for it, for imports of 1 MiB or more; the main queue can't be used there because `importAllocation` holds the imports lock and `GpuMutex` comes before it in the lock order. A smaller import is compared when it is made, so a title that maps, imports and unmaps a small buffer around every submission does not block its mapping thread. Its 4-byte read is queued instead: the next import lookup that holds `GpuMutex` while the imports match the registry (`settleUnwatchedImports`) records the queued reads into one batch on the settle queue, and once the batch's fence has signalled a later lookup returns the import's whole 64 KiB blocks to the write watch (`GuestMemory::Rewatch` drops their marks and stamps every block as a CPU store), except while storage results are pending over them. A settled import whose range had already left the watch returns the same way, and an import retired while its read is in flight is kept with that batch and freed once the batch has completed, without waiting for it; a guest unmap of its memory waits for the batch instead. Imports through a section alias keep tracking on the guest views, and every other Windows host import is compared. A device without such a queue keeps the comparisons. On Linux the settle queue is made when the device imports dma-buf memory, and every dma-buf import of 1 MiB or more is settled right after it is made, for another reason: the kernel maps an import's pages for the GPU at the first submission that uses it, which took about 2 s for a 3 GiB import on RADV and held up the title's frame. The import's buffer is shared with the settle queue's family instead of aliased, since a sparse import is bound to many memory objects. + +### Log output + +```mermaid +flowchart LR + callers["Any thread:
WriteLogLine,
AGC_LOG_DEFERRED"] --> queue["Queue, 16 MiB:
a full queue blocks the caller"] + queue --> logger["Logger thread:
format, join lines per stream,
fwrite and flush"] + logger --> streams["stdout, stderr"] + exits["Abort, terminate, crash report,
_Exit"] -- "bounded flush, 2 s" --> queue +``` + +`WriteLogLine` ([`LogMacros.hpp`](../../core/libs/prx/libc/include/general/LogMacros.hpp), behind `APS5_LOG_OUT` and `APS5_LOG_ERR`) formats a line on the calling thread and hands it to the module's `LogSink()` when one is installed; without a sink it writes and flushes it there. `Driver` installs [`AsyncLog`](../../core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp) as the sink of `libSceAgcDriver`, so the queue workers no longer wait for the console or the log file for those lines. `APS5_LOG_SYNC` (any value) skips the installation and restores the synchronous path. + +- **Queue.** One logger thread drains a queue that holds at most 16 MiB of waiting lines; the batch the logger is writing comes on top. A caller that finds the queue full blocks until the logger has taken the pending batch, so no line is dropped. The logger writes entries in queue order and joins consecutive entries of one stream; it writes them with one `fwrite` and flush when the text reaches 64 KiB, the stream changes or the batch ends. `AGC_LOG_DEFERRED` queues the format string with numeric, enum and `LogLiteral` arguments and formats them on the logger thread; `APS5_LOG_TIMESTAMPS` stamps a line when it is queued. +- **Order.** Lines that pass through the sink, on stdout and stderr alike, appear in call order. A direct `fprintf(stderr, ...)` or `printf` does not pass through it and is written at once, so it can appear before older queued lines; the coalesced `deferred write-back failed` notes are of this kind. `APS5_LOG_SYNC=1` keeps one order for all of them. +- **Flush.** `Flush` waits without a limit until every line queued before the call has been written, including while another thread holds the stream. It runs when the driver fails and in `Driver::stop`. At exit `AsyncLog` runs a bounded flush (`FlushFor`, 2 s) instead: on Windows the `atexit` handlers of a DLL run after the process has ended its other threads, so the logger thread may be gone with lines still queued, and a `Flush` there would wait forever. It also registers that bounded flush with `LibcSetEndFlush_nid_no_patch`; libc runs it where the process ends without `atexit`: `abort()` and `std::terminate` (a `SIGABRT` handler that the first registration chains in front of the previous one, so `raise(SIGABRT)` still ends the process), guest `abort()`, the Windows crash report of a fatal exception and guest `_Exit`. A title that installs its own `SIGABRT` disposition replaces that handler. The bounded flush polls two counters and takes no lock, so a thread that dies holding the queue lock or a stream cannot stall it. After 2 s it gives up, later calls return at once, and the lines still queued are lost. +- **Statistics.** A `[log]` line every 10 s in which lines were logged reports lines, bytes, the time spent by the callers and by the logger, and the queue peak; `APS5_LOG_STATS=0` hides it. Identical `deferred write-back failed` notes print three times per 10 s and are then counted in one `repeated N more times` line. + +`agc_async_log` checks the order of deferred and written lines, long lines, the repeat limit, a `Flush` that waits for a held stream, a bounded flush that gives up, and a full queue that blocks its caller. `agc_async_log_fatal` starts the test executable as a child that queues lines behind a held stream and then calls `abort()`, calls `std::terminate()`, raises an access violation (Windows) or `SIGABRT` (elsewhere), or aborts without ever releasing the stream; the first four must leave every line in the file in order, the last must still end after the bound. ### Runtime image and sampler tables diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 2dfcffda8d..ff5632bf5f 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -14,7 +14,7 @@ Python must be found during CMake configuration for Python tests to be registere This mode builds the conversion tool and its tests on Linux, Windows and macOS, including Apple Silicon. macOS uses AppleClang from the Xcode command-line tools; Windows uses the MinGW-w64 toolchain below. The executable is `build-relinker/core/relinker/relinker` (`relinker.exe` on Windows with Ninja). -The output remains x86-64 Linux ELF or Windows PE. Converted games need system libraries built for the target OS and a compatible x86-64 host. This mode does not build those libraries or provide macOS game execution. Tests inspect both output formats; execution checks run only on their compatible hosts. +The output is x86-64: Linux ELF by default, Windows PE with `--windows` or Mach-O with `--macos` ([usage](../user/USAGE.md#input-and-conversion)). Converted games need system libraries built for the target OS and a compatible x86-64 host. This mode does not build those libraries, and releases ship them for Linux and Windows only. Tests inspect all three output formats; execution checks run only on their compatible hosts. Use a separate build directory for the full build. diff --git a/docs/dev/TRACING.md b/docs/dev/TRACING.md new file mode 100644 index 0000000000..8297f4ab04 --- /dev/null +++ b/docs/dev/TRACING.md @@ -0,0 +1,74 @@ +# Tracing + +Set a switch before starting the emulator. Unless noted, any value enables it. Most output goes to the emulator log or standard error. Performance captures are written to files as described in [Build](BUILD.md#performance-trace). Traces can be large and can affect timing. + +## Kernel and libraries + +| Switch | Output | +| --- | --- | +| `APS5_TRACE_AFFINITY` | Thread role, host thread ID, requested CPU mask and whether pinning succeeded. | +| `APS5_TRACE_AJM` | Submitted audio jobs, decoded output and unimplemented AJM calls. | +| `APS5_TRACE_APR` | APR file reads, including path, offset, size and destination. | +| `APS5_TRACE_AUDIOOUT` | Audio output port parameters when a port is opened. | +| `APS5_TRACE_AUDIOOUT2` | AudioOut2 context and port parameters, attributes, pushes, queue polls and periodic queue summaries. | +| `APS5_TRACE_CALLS` | On Windows, records calls to the imports selected by this comma-separated library list (or `all`); crash and abort reports include the last 64 calls and most-called imports. The executable must be converted with `--trace-imports`; see [Usage](../user/USAGE.md#tracing-system-library-calls). | +| `APS5_TRACE_EXIT` | Guest `exit` and `_Exit` calls and process-exit backtraces. | +| `APS5_TRACE_FIBER` | Fiber initialization, switches and returns. | +| `APS5_TRACE_MEMORY` | Direct-memory allocation, mapping and protection operations. | +| `APS5_TRACE_PREPARATION_WAIT` | Shader preparation tasks moved to the front of the queue to satisfy a waiting caller. | +| `APS5_TRACE_SSE4A` | Each emulated SSE4A instruction once per guest instruction address (Windows). | +| `APS5_TRACE_SYNC` | Event-flag operations and their wait and wake results. | +| `APS5_TRACE_TIMERS` | Timing measurements for the host wait primitives used by the emulator. | +| `APS5_TRACE_USLEEP` | Every 2,000 guest `usleep` calls, with caller addresses and requested delays. | +| `APS5_TRACE_VIDEODEC` | Video decoder API calls and decode results. | +| `APS5_TRACE_WAITS` | Blocking waits aggregated by guest call site, including timeout counts and longest waits, every three seconds. | + +## GPU and memory + +| Switch | Output | +| --- | --- | +| `APS5_TRACE_BIGBUF` | Each copied or written-back guest-memory range of at least 1 MiB. | +| `APS5_TRACE_BDA` | Shader memory accesses that make a program use the buffer-device-address path. | +| `APS5_TRACE_BUFFER_WRITTEN` | Guest buffer elements classified as writable or read-only after descriptor population. | +| `APS5_TRACE_CAPSYNC` | Waits for GPU writes needed by CPU accesses, including submission and range details. | +| `APS5_TRACE_CONTEXT_STATE` | PM4 context-state operations and registers written while context state is pushed. | +| `APS5_TRACE_COPY_HLE` | Pending host-side copies, their ranges, GPU batches and known writers. | +| `APS5_TRACE_DCC_KEYS` | Display and storage-image DCC key reads, writes, fills and clears. | +| `APS5_TRACE_DISPATCH_CACHE` | Compute dispatch cache hits, misses and changed user-data words. | +| `APS5_TRACE_DISPATCH_IO` | Compute dispatch resources; value `2` also prints small user-data buffers. | +| `APS5_TRACE_DRAW_CACHE` | Draw-cache lookups, hits, misses, and key differences, reported every ten seconds. | +| `APS5_TRACE_DRAWS` | Draw inputs and per-draw timing phases. With `APS5_PROFILE_DRAW`, also prints the detailed draw lines. | +| `APS5_TRACE_DRAW_PROGRAMS` | One line per draw listing selected shader stages, roles, code addresses and entry addresses. | +| `APS5_TRACE_FILL_COVER` | Initial fill/image coverage mismatches and refused image-layer clears. | +| `APS5_TRACE_FLUSH` | Storage images flushed to guest memory and ranges left for the CPU. | +| `APS5_TRACE_GPU` | Queue submissions, packet execution, waits and display-buffer scanouts. | +| `APS5_TRACE_HOST_IMPORT` | Host-import mapping attempts, memory-map details, failures and retirement. | +| `APS5_TRACE_IMAGE_TABLE` | Image-table resource materialization and descriptor selection. | +| `APS5_TRACE_INDIRECT_DRAWS` | Indirect draw packets that pass the draw precheck. | +| `APS5_TRACE_LABEL` | `RELEASE_MEM` packets that target the address in this hexadecimal value. | +| `APS5_TRACE_LATE_LABELS` | GPU label writes observed after their waiting submission was handled. | +| `APS5_TRACE_QUERY` | Every sixteenth guest-memory page query, including cost and caller address. | +| `APS5_TRACE_RESOURCE_KEY_MISSES` | Resource-content key changes that cause draw or dispatch resource-cache misses. | +| `APS5_TRACE_SHADOW` | Unit-shadow publish, seed, eviction and retirement events; a number limits publish output. | +| `APS5_TRACE_SKIPS` | GPU operations skipped by diagnostics, with a bounded number of recompile-request dumps per reason. | +| `APS5_TRACE_STAGING` | Distinct guest-buffer ranges admitted to the write-staging window. | +| `APS5_TRACE_TEXTURES` | Sampled texture addresses, dimensions, format and nonzero-byte counts. | +| `APS5_TRACE_UNREADABLE` | Call sites that check inaccessible guest-memory ranges or fail to read them. | +| `APS5_TRACE_UPLOAD` | Reasons storage-image data is uploaded again. | +| `APS5_TRACE_BARRIERS` | Set to a positive integer to print that many skipped barrier decisions and their ranges. | + +## Performance captures + +See [Build](BUILD.md#performance-trace) for the trace format, summary fields and capture limits. + +| Switch | Output | +| --- | --- | +| `APS5_TRACE_FRAMES=:` | Records a window of presents by zero-based frame number. | +| `APS5_TRACE_SECONDS=:` | Records a window of presents by elapsed seconds after the first present. Cannot be set with `APS5_TRACE_FRAMES`. | +| `APS5_TRACE_SUMMARY_EVERY=` | Writes periodic summary lines without recording trace events. Can be combined with either window switch. | +| `APS5_TRACE_EVENTS=` | Sets the per-thread event-buffer capacity for a frame or seconds window (1–4,194,304; default 262,144). | +| `APS5_TRACE_DIR=` | Writes performance capture and summary files to this directory instead of the working directory. | + +## Shader registers + +`APS5_TRACE_SHADER_REGS=all` prints register snapshots for every shader. Set it to a shader code address in hexadecimal to print one shader's registers. diff --git a/docs/dev/debt/libSceAgc.md b/docs/dev/debt/libSceAgc.md index 8a7ce86545..7d871117ed 100644 --- a/docs/dev/debt/libSceAgc.md +++ b/docs/dev/debt/libSceAgc.md @@ -20,7 +20,7 @@ - [sceAgcDcbSet{Cx,Sh,Uc}RegistersIndirectGetSize](../../../core/libs/prx/libSceAgc/DcbState/src/Registers.cpp) (libSceAgc) - the count argument is ignored: PPSA28416 calls them with leftover register contents (0xf5f5039c), so the real functions likely take none; the size is 20 bytes either way - [sceAgcUpdateInterpolantMapping_0100](../../../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - name from its NID (`k0E7vkgqAuE`); the `(regs, vs, ps)` arguments are the declared ones, behaviour unknown - [sceAgcWriteDataPatchSetDst, sceAgcWriteDataPatchSetCachePolicy and their Async forms](../../../core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp) (libSceAgc) - the signatures `(packet, uint8 value)` are inferred from the other patch functions, as no public declaration was found; the value is taken in the encoding of the `dst` and `cachePolicy` arguments of `sceAgcDcbWriteData` (graphics) and `sceAgcAcbWriteData` (Async, compute), and the patched packet is the one those calls would write. `sceAgcAsyncWriteDataPatchSetAddressOrOffset` is assumed to patch the address like `sceAgcWriteDataPatchSetAddressOrOffset`, as the compute WRITE_DATA packet keeps it in the same dwords -- [sceAgcCreateInterpolantMapping_10000](../../../core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp) (libSceAgc) - name from its NID (`dbOlWdppb4o`), not in the aerolib NID list; same arguments as `sceAgcCreateInterpolantMapping_0100`; the `SPI_PS_INPUT_CNTL` bits for `is_f16` 2 inputs follow KytyPS5, not confirmed on hardware +- [sceAgcCreateInterpolantMapping_10000](../../../core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp) (libSceAgc) - name from its NID (`dbOlWdppb4o`); same arguments as `sceAgcCreateInterpolantMapping_0100`; PPSA26344 passes three pointers at each of nine observed call sites (`0x9F914E`, `0x9F91E2`, `0x9F9216`, `0x1604BBD`, `0x16122D7`, `0x1618285`, `0x161838C`, `0x16184EE`, `0x1618607`), consistent with that argument assumption, but does not read the return value there, so its return type is not established. The `SPI_PS_INPUT_CNTL` bits for `is_f16` 2 inputs follow KytyPS5, not confirmed on hardware - [sceAgcGetDataPacketPayloadRange](../../../core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp) (libSceAgc) - signature, the `{base, size in bytes}` output and the type 0 range (header + 1, one dword longer than the type 1 payload) from KytyPS5 only, matching `sceAgcGetDataPacketPayloadAddress_0090`; prosper returns header + 2 for both types from that address function - [sceAgcAsyncCondExecPatchSetCommandAddress, sceAgcAsyncCondExecPatchSetEnd](../../../core/libs/prx/libSceAgc/Patch/src/CondExec.cpp) (libSceAgc) - no reference has a body for them; they are assumed to patch the COND_EXEC packet like `sceAgcCondExecPatchSetCommandAddress` and `sceAgcCondExecPatchSetEnd`, since `sceAgcAcbCondExec` writes the same packet as `sceAgcDcbCondExec`, as `sceAgcAsyncRewindPatchSetRewindState` does for REWIND - [sceAgcAcbAtomicGds_0900](../../../core/libs/prx/libSceAgc/Acb/src/Control.cpp) (libSceAgc) - name from its NID (`gQkqkLttcpw`), signature unknown diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 5762dcca0c..53683267fa 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -17,6 +17,7 @@ - [Indices far past every vertex fetch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - when a draw's highest index is at least 64Ki vertices past the largest per-vertex record count, every index past that count is moved to it in a copy of the indices, so the zero-padded vertex copies are not sized to the original index (PPSA04159 draws a list of `0xffffffff` with primitive restart off, which asked for 160 GiB). Each attribute fetched by vertex index reads zeros there as at the original index (out-of-bounds select 0 or 1); a draw with another out-of-bounds select keeps its indices. The vertex index the shader sees changes for the moved indices, which a shader that reads it directly would notice - [Depth clear draws](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - a draw with `DB_RENDER_CONTROL.DEPTH_CLEAR_ENABLE` is taken to write `DB_DEPTH_CLEAR` to every pixel it covers whatever the depth test, depth bounds and depth bias say: it is drawn with an always-passing depth write and the viewport depth range pinned to the clear value. The draw is rasterized as submitted, so its coverage is the draw's own. Whether the DB still applies the depth test or HiZ to such a draw is not measured - [GS_FAST_LAUNCH geometry](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - `VGT_SHADER_STAGES_EN.GS_FAST_LAUNCH = 1` with a point list and `GE_CNTL` groups of one vertex and one primitive decodes as one point per subgroup, with the primitive limit taken from `GE_NGG_SUBGRP_CNTL.PRIM_AMP_FACTOR` and the GS/ES VGPR counts not required. Follows how RADV drives mesh shaders on gfx10.3 (`ac_nir_lower_ngg.c`, the workgroup index comes from the vertex id of the first thread), not measured on the console. Fast launch 2, other group sizes or input primitives and indexed fast-launch draws still throw. PPSA03671's fast-launch draws decode, but the title stops earlier in a wave64 vertex program, so they have not reached the GPU yet. +- [Line-strip geometry output](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - a geometry program with `VGT_GS_OUT_PRIM_TYPE = 1` (line strip) runs as a mesh shader with `OutputLinesEXT`: each primitive export is one line whose two vertex indices are packed like the three of a triangle, as Mesa's NGG lowering exports line strips (`ac_nir_lower_ngg_gs.c`), and each input primitive gets `VGT_GS_MAX_VERT_OUT - 1` lines where a triangle strip gets `- 2` triangles. A subgroup gets at least `GE_MAX_OUTPUT_PER_SUBGROUP` line slots, because Mesa's lowering exports one primitive, possibly null, per emitted vertex; whether the console's compiler does the same is not known, and triangle strips keep `- 2` per input. A fast-launch subgroup takes its line count from `GE_NGG_SUBGRP_CNTL.PRIM_AMP_FACTOR` as for triangles. Line output counts as lines for the polygon offset decode, as on the vertex path. The lines are drawn 1 pixel wide with Vulkan's default line rasterization, like lines on the vertex path: `PA_SU_LINE_CNTL` is not read, and neither its width nor the console's line rules were measured. Points, rectangle lists, per-stream types and line strips from a vertex program on the mesh path still throw. - [Alpha-to-coverage](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - only `DB_ALPHA_TO_MASK = 0xaa01` is accepted: the enable bit with the four quad offsets at 2 and no `OFFSET_ROUND`, the undithered setting [radeonsi](https://gitlab.freedesktop.org/mesa/mesa/-/blob/bffd91947efc1932da99aca69fc540d6d4201b0c/src/gallium/drivers/radeonsi/si_state.c#L371-L381) programs. Draws are single-sampled, so the pixel shader [writes](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) `SampleMask` 0 when its MRT0 alpha is below one half or NaN and all ones otherwise, and runs with late fragment tests. That rule was measured on a Radeon 660M (gfx1035, RADV with `RADV_DEBUG=noatocdithering`, which programs the same register) for 0, -0, 0.25, the two floats below one half, one half, the two above, 0.75, 1, 2, -1, both infinities and both NaNs, and is pinned in [AlphaToCoverage.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/AlphaToCoverage.cpp); it is not measured on gfx1013. Vulkan's `alphaToCoverageEnable` is not used: the NVIDIA driver 615.78.08 dithers single-sample coverage (an alpha of 0.25 writes half the pixels of an RTX 3050 Laptop target). Other offsets, `OFFSET_ROUND`, a sample-mask export, and an MRT0 that is missing, swizzled, packed or exported without a floating-point alpha throw. A draw without a color target on MRT0 keeps alpha-to-coverage from its MRT0 export, as [radeonsi exports MRT0 as `32_AR` for alpha-to-coverage without a color buffer](https://gitlab.freedesktop.org/mesa/mesa/-/blob/mesa-24.0.0/src/gallium/drivers/radeonsi/si_state_shaders.cpp#L2425-L2431). A draw that enables alpha-to-coverage with no MRT0 export still throws: what the console covers then is not measured. Default dithered `DB_ALPHA_TO_MASK = 0x18701` additionally uses a per-quad sample-mask table [measured on gfx1036](https://github.com/boykopovar/AnyPS5/pull/4451#issuecomment-6124848921) for eight standard samples with FP16_ABGR or 32_ABGR exports; other sample counts, custom locations, EQAA combinations and dithered UNORM16/SNORM16 exports throw. The dithered rule is not measured on gfx1013. - [MSAA_ENABLE on a single-sample target](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - `PA_SC_MODE_CNTL_0.MSAA_ENABLE` with `PA_SC_AA_CONFIG.MSAA_NUM_SAMPLES = 0` is drawn like the same state without the bit when sample 0 of the four `PA_SC_AA_SAMPLE_LOCS_PIXEL_*_0` registers is at the pixel center and the draw rasterizes filled triangles with alpha-to-coverage and conservative rasterization off. [RADV](https://gitlab.freedesktop.org/mesa/mesa/-/blob/745709a000482a4a24c91faef5c5a40bd653a863/src/amd/vulkan/radv_cmd_buffer.c#L13606-13609) programs this state for one-sample custom sample locations on GFX10 and later, where the bit makes the rasterizer use the programmed location of sample 0 ([061bc615](https://gitlab.freedesktop.org/mesa/mesa/-/commit/061bc6151a11597f720535f00e79a5752e24aff6)); that the bit changes nothing else for such a draw is inferred from that and is not measured on hardware. Other locations of sample 0, absent location registers, points, lines, rectangles, alpha-to-coverage and conservative rasterization throw - [A multisampled target rasterized without MSAA_ENABLE](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - `PA_SC_AA_CONFIG.MSAA_NUM_SAMPLES` > 0 with `PA_SC_MODE_CNTL_0.MSAA_ENABLE` clear is drawn with the target's sample count and every sample at the pixel center, as a single-sample scan into a multisampled target already is. On GFX10 and later the bit is what makes the rasterizer use the programmed sample locations ([RADV 061bc615](https://gitlab.freedesktop.org/mesa/mesa/-/commit/061bc6151a11597f720535f00e79a5752e24aff6)), and with multisampling disabled GL rasterizes at the pixel center and gives the fragment full coverage, which radeonsi programs by clearing the bit; that the hardware does the same with `MSAA_NUM_SAMPLES` > 0 is inferred from these and is not measured on hardware. Points, lines, a depth target and alpha-to-coverage throw. SONIC SUPERSTARS (PPSA06888) draws such a 2x target. @@ -43,7 +44,7 @@ ### Functional - [Initial stencil state](../../../core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp) uses zero for `DB_STENCIL_CONTROL`, `DB_STENCILREFMASK` and `DB_STENCILREFMASK_BF` (context `0x10b`-`0x10d`), including after `CLEAR_STATE`. The public V0, V4, V5 and V7-V11 [register-default tables](../../../core/libs/prx/libSceAgc/Command/include/RegisterDefaultsData.hpp) supply these values. Linux's [GFX10 clear-state table](https://github.com/torvalds/linux/blob/master/drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h) agrees on the control register but uses `0x01000000` for the reference masks; the console reset values have not been measured. Planet Coaster (PPSA01736 v01.017) enables stencil tests without writing the control register. -- [Type-0 texture descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (`TYPE` = 0 in dword 3) with a nonzero base are bound as null images, like a zero base. The behaviour comes from a gfx1036 (RDNA2) measurement in issue #4139 (image sample, load and resinfo return zero, no fault); it was not measured on the console's gfx1013. +- [Type-0 and type-1 texture descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (`TYPE` = 0 or 1 in dword 3) with a nonzero base are bound as null images, like a zero base. TYPE 0 comes from a gfx1036 (RDNA2) measurement in issue #4139 (image sample, load and resinfo return zero, no fault). TYPE 1 comes from a gfx1010 (RDNA1) measurement in #4139 and #5828 (`image_sample_lz`, `image_sample_b`, `image_load` and `image_get_resinfo` return zero, no fault) with format 22 only; TYPE 1 is not measured on RDNA2, with other formats or with depth-compare reads (`image_sample_c*`). Neither type was measured on the console's gfx1013. - [Multisampled color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) lose their guest-read guards and cannot be reused after a successful allocation removal or unmap covers their full guarded range. Retired Vulkan images remain allocated until device teardown to preserve outstanding references. Partial unmaps keep the guard; partial allocation reuse and guest access to live multisampled data remain unsupported. - [Stencil reference export replay](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) on devices without `VK_EXT_shader_stencil_export` supports at most sixteen values proved from constant, select and phi exports, with an export proved on every kept path. It records instances, then their triangles, then the reference classes of each triangle. One pipeline filters fragments at shader termination against a dynamic push value before late stencil and depth tests. The [Vulkan stencil-reference replacement rule](https://docs.vulkan.org/refpages/latest/refpages/source/FragStencilRefEXT.html) discards bits above the attachment's stencil width; supported stencil attachments have eight bits. Numeric subgroup observations throw except bounded own-lane mask plumbing and group control with kept-lane and derivative-helper coverage proved. WQM inputs must exclude helper contributions, since [non-quad subgroup operations may treat helpers as inactive](https://docs.vulkan.org/glslext/latest/glslext/khr/GL_KHR_shader_subgroup.html). Readonly guarded SRT reads may report internal host faults; these draws wait and check faults before publishing results, and loops throw. Indirect draws, restart, strips, fans, tessellation, mesh, multisampling, early tests, ordered pixel execution, guest shader memory writes, other fault-producing memory access, clocks, primitive ID inputs, attachment feedback and pipeline face/channel splits throw. An exhausted stage push range throws. Equivalence to console ordering has not been measured on console hardware. @@ -64,21 +65,24 @@ - [Covered-centroid raster state](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) accepts PA_SC_AA_CONFIG.COVERED_CENTROID_IS_CENTER only when the pixel program is skipped or SPI_PS_INPUT_ENA and SPI_PS_INPUT_ADDR have no active perspective or linear centroid input. Bit 29 follows [PS5_Mesa GFX10.3 registers](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/registers/gfx103.json); [RADV](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/vulkan/radv_cmd_buffer.c) sets it on GFX10.3 and later. Active centroid interpolation and unspecified input registers still throw. Console covered-pixel centroid results were not measured. - [Compute LDS beyond the device's shared-memory limit](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) lives in a coherent storage buffer with one slice of `LDS bytes / 4 + 1` dwords per workgroup (the last dword is the atomic lock), shared by all such dispatches and grown in 1 MiB steps up to `maxStorageBufferRange`; a dispatch whose slices do not fit throws. It is slower than shared memory and its cost was not measured. An indirect dispatch of such a program always reads its group counts on the CPU. Its results are pinned against the in-limit semantics in `agc_driver_lds_beyond_device_limit`, not measured on hardware. - [AGC buffer allocations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) retry host/device out-of-memory once after reclaiming idle pool buffers and fully unused slab blocks. Live buffers and blocks with live slots are retained. Live images and other retained resources have no global memory bound; allocation failure still throws when reclaim cannot satisfy the request. +- [Pipeline journal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp) (libSceAgcDriver): the journal is append-only and never compacted, so a module a later run no longer builds pipelines from stays in the file until it is deleted, and the whole file is read into memory while the replay runs (its modules reference that buffer until the last replayed pipeline was built). Only the first `APS5_PIPELINE_JOURNAL_ENTRIES` pipelines recorded are replayed (a record is never rewritten, so a pipeline past the bound stays past it), and the pipeline store evicts the least recently used entry past `APS5_PIPELINE_STORE_ENTRIES` (1024 each): a title with more configurations rebuilds the rest at the draw as before. A compute pipeline is replayed over the set layout `DescribeSetLayouts` derives from the result's bindings, which a dispatch that finds it resident checks against its `ShaderResources` layout only under `APS5_VERIFY_SET_LAYOUTS=1`. Recording and replay are best effort: a record whose write fails is lost, an entry whose build fails is skipped, and both are counted in the `[pipeline-journal]` line rather than stopping the run. A replayed pipeline that the title never asks for again costs its build time and its driver memory (its optimized relink is deferred to its first draw, so a run whose first frames bind many replayed pipelines queues their relinks then); nothing ages recipes out of the file. Two runs sharing the file take turns under its lock file, but each only knows the modules it loaded or wrote itself, so a module the other run appended meanwhile may be appended again (the loader keeps the first copy). Records a run appends after another run's load are replayed by that run only after its next start. The file header's format number is the only guard against a change of the recipe codec, `CanonicalPipelineState` or the store key that keeps the structures' sizes (the recompiler's source version does not cover the driver's sources). - [Graphics pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp) (libSceAgcDriver): a pipeline is fast-linked at the draw and relinked with `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` on a background thread, and binds the optimized one from the next draw on. Draws recorded before that use the fast link, whose vertex stage keeps the outputs the fragment stage does not read: on RADV (Radeon 660M, Mesa 26.2.4) 346 of 483 Astro's Playroom vertex shaders paired with a fragment stage that reads none of them used more VGPRs (median 8 more, up to 2.5 times), and on an RTX 4090 13 of 15 PPSA36082 vertex shaders up to 68 instead of 38 (#2242). The optimized link compiles to the same code as a whole pipeline - [Prebuilt pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver): each registered vertex or pixel shader in registration form gets its pre-rasterization or fragment-shader library built on one background thread, with a fixed state the draw may not use: triangle list, counter-clockwise front face, no depth clamp, depth bias or conservative rasterization, first provoking vertex, one sample. Libraries are kept until the device is torn down; on Astro's Playroom 1948 of them added 0.44 GB of peak memory on RADV. - [`APS5_THREAD_PRIORITY=rt`](../../../core/libs/prx/libSceAgcDriver/Execution/src/ThreadPriority.cpp) (libSceAgcDriver) lowers the process's hard `RLIMIT_RTTIME` when it is above `APS5_THREAD_RTTIME_US` or rtkit's `RTTimeUSecMax`, because [rtkit](https://github.com/heftig/rtkit/blob/c295fa849f52b487be6433e69e08b46251950399/rtkit-daemon.c#L592-L637) (`verify_process_rttime`) reads the hard limit from `/proc//limits` and refuses real-time scheduling when it is higher. An unprivileged process cannot raise the hard limit again, so it stays lowered for the process and its threads even when rtkit then refuses the request (burst limit, user or start-time checks) and the thread throws. rtkit's `MaxRealtimePriority` and `RTTimeUSecMax` are read first, so an unreachable rtkit or a priority above its maximum leaves the limit untouched. The lowered limit only matters for threads that later run under a real-time policy -- AMD image descriptors with `TYPE=0` are treated as null for read-only image slots even when the base and other words are nonzero: `image_load` on a gfx1035 with such a T# returns 0,0,0,0, and the same descriptors were seen in direct reads in PPSA01341 on an RTX 4070. Writable and atomic image slots and entries of an image table keep faulting because there is no hardware measurement for them, and reserved `TYPE=1` also faults (gfx1035 reads it as zeros too); gfx1013 behavior is unverified +- AMD image descriptors with `TYPE=0` are treated as null for read-only image slots even when the base and other words are nonzero: `image_load` on a gfx1035 with such a T# returns 0,0,0,0, and the same descriptors were seen in direct reads in PPSA01341 on an RTX 4070. Writable and atomic image slots and entries of an image table keep faulting because there is no hardware measurement for them, and the reserved types 2-7 also fault. `TYPE=1` is treated as null in the same read-only slots: gfx1035 reads it as zeros too, and gfx1010 returns zeros for `image_sample_lz`, `image_sample_b`, `image_load` and `image_get_resinfo` with format 22 (#5828); it is not measured on RDNA2, with other formats or with depth-compare reads (`image_sample_c*`). gfx1013 behavior is unverified - On a device without VK_EXT_robustness2's `nullDescriptor`, such as MoltenVK, a null image descriptor in a typed heap is a [placeholder image](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PlaceholderImages.cpp) of that heap's type, 1×1 and cleared to zero, which every null descriptor of the heap shares. Reads give zeros, as `nullDescriptor` allows, but stores and atomics to it are kept, so a later read of a null storage image can return them, and a depth comparison compares against depth 0. A heap whose placeholder the device cannot create, for example 1D or 3D depth, 64-bit atomics without 64-bit image atomics, or multisampled storage without `shaderStorageImageMultisample`, throws when a null descriptor reaches it. - [Draw input snapshots](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver): `PrepareDrawBindings` caps read-only input buffer snapshots at 64 MiB (overridable via `APS5_DRAW_SNAPSHOT_MAX_MIB`). Larger inputs (streaming heaps, bindless pools) or draws exceeding the cumulative budget skip the per-draw snapshot and bind in place, avoiding multi-gigabyte CPU copy stalls (such as Ratchet & Clank: Rift Apart PPSA01474 stalling at frame ~380 copying 36 GB every 5 s). If the title rewrites those large ranges on the CPU before GPU completion, the in-place binding observes the modification, as verified by `drawSnapshotCapTests`. - [Pending storage-image results](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) are stored when something reads their memory, not when the GPU work that wrote them completes. A title that waits for that work, frees the image and reuses its memory can have the new data overwritten by the late store (#1970). A title's own CPU reads are not seen either, so a title that reads the results after waiting for a label reads stale memory. `APS5_STORE_AT_FLIP=1` stores every pending result at each flip and `APS5_STORE_AT_LABEL=1` before each label the GPU writes; both store every pending image, not only the ones the waited-for work wrote, and neither is on by default. - [Pending storage-image results over memory outside the write watch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) leave to the CPU the 64 KiB blocks whose bytes differ from the ones saved at the upload, the fill, the image copy, the previous store or the moment their range left the watch (`Unwatch`, or a host import that does not stay watched), and the blocks the watch saw the CPU write before that moment. A store made on the GPU saves its bytes through a host-visible copy that becomes the saved bytes when its batch completes; a store or a refresh that needs them earlier waits for that batch. A unit without a write generation saves the guest bytes when results land in it outside the watch. One whose range leaves the watch while its results are pending, and ranges that leave the watch through a failed collect, have no saved bytes; storing such results throws. - Windows host-pointer imports of the guest mapping itself are compared instead of write-watched, except writable imports of private `MEM_WRITE_WATCH` memory when the import probe keeps the write watch and a settle queue exists: those of 1 MiB or more stay watched, and smaller ones return to the watch once their settle has completed (see Pages under an import window): a driver can mark imported pages dirty after the synchronous import window. Shared memory imported through a separate writable alias retains protection-based tracking on the guest views; driver access to the import alias does not disarm those views. [GuestMemory](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. Excluded blocks keep their write-watch marks as a hint only (`UntrackedHint`): a storage image or a sampled texture snapshot over them compares its bytes again after a mark in its range, a driver store, a mapping change or a recorded GPU write instead of at every collect epoch or lookup, and a mark the driver makes without a CPU store costs one comparison. This preserves CPU edits through byte comparisons but limits snapshot reuse on excluded blocks to draws of one collect epoch and disables unit shadows for them; restoring tracking on import retirement without replacing the backing remains unimplemented. - [Pages under an import window](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) (libSceAgcDriver) whose write protection was resolved while a host import that leaves its range watched was made are compared with the bytes they held before the import and count as stored over only when those differ. A CPU store of the bytes already there within the window cannot be told apart from an import that resolves the write protection without storing, as the NVIDIA driver (610.57.04) does for every imported page, so such a store does not win over pending results. Where host-pointer and dma-buf imports differ in whether they stay watched, a failed host-pointer import and the dma-buf import after it are two windows, and the pages the failed one resolved count as stored over. On Windows, the NVIDIA driver (617.42) marks every page of a host import of private `MEM_WRITE_WATCH` memory written, asynchronously, until a GPU use of the import completes; a 4-byte read of it on the settle queue ends that even while main-queue work that used the import first is still running, and an earlier import of the same pages whose first use runs after that read marks none of them (`agc_import_settle_first_use`, `agc_import_settle_earlier_import`). Such an import stays watched only after a 4-byte read of it on the settle queue has completed inside the window. Each one waits for that read, which keeps its window open for one round trip on the settle queue, so a same-value CPU store from another thread in that time counts as unchanged too. An import made where no settle queue exists is compared instead. An import of under 1 MiB is not settled inside its window, because the settle blocks the importing thread and a title that imports a 64 KiB buffer around every submission lost about 19 % of its frame rate to that wait (Dripl1x, #6060). It is compared until its read, submitted in a batch by a later import lookup under `GpuMutex` while the imports match the registry, has completed; a later lookup then returns its whole blocks to the write watch, each stamped as a CPU store, so whatever was derived from them while they were compared reads as changed once. Its blocks stay compared while storage results are pending over them, and those of an import retired before that stay compared. +- [Settled dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a dma-buf import of 1 MiB or more is read by a 4-byte copy on the settle queue right after it is made (`settleImport`), so the kernel maps its pages for the GPU there and not in the first submission that uses it (about 2 s for a 3 GiB import on RADV). The import's buffer is shared with the settle queue's family. An import made at map time settles on the guest thread that maps the range, inside its map call, so that thread waits while the kernel maps the pages: 0.47 to 2.2 s for the 3 GiB range of Hades (PPSA03355) and 0.49 to 4.3 s for the 4.5 GiB range of ASTRO BOT (PPSA21567) on RADV. Both titles map these ranges at start-up; a title that maps large ranges during gameplay stalls its mapping thread for as long. An import made from a submission settles under the GPU mutex, so its frame still pays for it; a smaller import, and every import on a device with no settle queue (one without a copy-capable queue besides the main one), pays in its first submission. A range imported through a host pointer, as lavapipe imports shared memory, is not settled - [Sparse dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a shared-backed range larger than one 64 MiB udmabuf buffer is imported as one sparse buffer bound to the 64 MiB buffers, on a queue of its own when the device has a second family that binds sparse memory. The path is decided before anything is imported, and a failure after that throws. A range takes the single udmabuf import instead, which udmabuf's `size_limit_mb` refuses above 64 MiB by default so the range is copied, when its offset in the backing file is not a multiple of the buffer's sparse block size (64 KiB on RADV), when the sparse buffer does not accept every memory type a chunk import can have, when the backing file ends inside its last chunk, or when it is imported at map time on a device with no second family (the bind then needs the GPU mutex); the NVIDIA proprietary driver imports no dma-buf memory - [Unnormalized S# coordinates](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view. - An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) maps clamp mode 0 (repeat) on X and Y to clamp-to-edge as required by Vulkan unnormalized coordinates, and assigns the magnification filter when minification and magnification filters differ (since unnormalized coordinates sample at LOD 0). Anisotropy, TRUNC_COORD, MC_COORD_TRUNC and other clamp modes still throw. Secondary LOD bias (LOD_BIAS_SEC, bits [19:14]) is accepted and ignored, matching RDNA2 hardware measurements across explicit and implicit samples. High on Life (PPSA15388, #4736) uses secondary LOD bias and unnormalized coordinates with clamp mode 0 and differing min/mag filters during home base and city rendering. - [Background link-time-optimized relink of library pipelines](../../../core/libs/prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp) (libSceAgcDriver) is skipped on every GPU under NVIDIA's proprietary Vulkan driver (`VK_DRIVER_ID_NVIDIA_PROPRIETARY`), so those pipelines keep their fast-linked form. The optimized link loses the device when bound on an RTX 3080 (Ampere, #2866) and an RTX 4060 (Ada, #3434); an RTX 5060 and an RTX 5080 (Blackwell) on the same driver do not, so the scope is wider than the evidence. No standard Vulkan query tells the architecture apart, and the driver version that fixes it is unknown. `APS5_NO_PIPELINE_LTO` disables the relink on every driver. - [8_SRGB and 8_8_SRGB textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp) (libSceAgcDriver) that the device samples as R8_SRGB or R8G8_SRGB use its own decode, which need not match gfx1035: NVIDIA's R8_SRGB is one bf16 step off in 29 of 256 codes. +- [PM4 submission rings](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/SubmissionRing.cpp) (libSceAgcDriver) keep only the first 512 dwords of each submission, so the packets past them are not printed. The [failure report](../../../core/libs/prx/libc/src/FailureReport.cpp) dump does not allocate from the C runtime heap: packet names come from static strings, each line is formatted with integer and string conversions into a 512-byte stack buffer and written with one `WriteFile`. It still runs on whatever state the crash left. It waits about 100 yields for each ring's lock and otherwise prints that queue as busy, so a crash while recording a submission loses that queue's ring. It reads the rings without checking them, so a crash that corrupted them can fault inside the dump; such a fault during the report of an unhandled exception stops in that report's `Sleep(INFINITE)` and the process hangs instead of ending. The reports run at most once per process. Linux has no crash report: there the rings are printed only by `APS5_STALL_WATCHDOG_SECS`. - [WAIT_REG_MEM](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp) reports a diagnostic after `APS5_GPU_WAIT_TIMEOUT_MS` without treating the timeout as completion. An unsatisfied wait keeps polling and checking driver failures; a missing producer can therefore leave its queue waiting indefinitely. Existing recorded-label shortcuts remain in use. - [COND_EXEC](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp) reads its condition after the earlier packets of its queue complete. Without PFP_SYNC_ME the console's graphics PFP reads it ahead of the ME, so a title that races the read sees a later value here. - A [flip inside a COND_EXEC range](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/CommandValidation.cpp) throws: the flip is reserved with the video output when the buffer is submitted, and how the console withdraws a skipped one is unknown. @@ -114,7 +118,7 @@ - [Multisampled targets in CB passes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - a CB eliminate or DCC pass over a multisampled target throws, since its texels are on the host and only a CB resolve and an FMASK decompress (`CB_COLOR_CONTROL` mode 5) are modeled. The FMASK decompress only applies the pending CMASK fast clear to the host image; it requires a CMASK address, and the target's DCC and FMASK contents are not interpreted. DCC over a multisampled target is accepted only in that pass and throws everywhere else. A CB resolve into a fast-clear destination throws: the resolve writes the destination's texels but not its CMASK, which would still read as cleared. - [Multisampled target guest access](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) (libSceAgcDriver) - the memory a multisampled color target throws on (CPU read, DMA copy, texture or storage image read) uses fragment-aware 64 KiB render-target blocks. [AMD Addrlib's GFX10 thin-block rule](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/imported/addrlib/src/core/addrlib2.cpp#L1680-L1712) gives the block width and height from element bytes and fragment count; its [surface-size calculation](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/imported/addrlib/src/gfx10/gfx10addrlib.cpp#L4000-L4005) rounds both extents to those blocks and multiplies by element bytes and fragments. Thus a 1920x1080 4x RGBA8 surface occupies 0x1fe0000 bytes, leaving the next allocation at +0x2000000 outside the guard. The 2x and 8x block orientations differ from treating the fragments as a larger element. Console address-layout equivalence has not been measured. Other tile modes retain the conservative `max(CB bytes * samples, width * height * element bytes * samples)` bound, which may still reject a distinct allocation. Targets are kept per device for the device's lifetime, one per surface address, format, extent and sample count, and are not evicted. - [CB resolve sample count](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - a draw with CB_COLOR_CONTROL in resolve mode is a CB resolve when either PA_SC_AA_CONFIG or CB_COLOR0_ATTRIB.NUM_SAMPLES is multisampled, and takes its sample count from CB0. Inferred, not from documentation: PPSA36082 resolves its 4x targets with no sample count in PA_SC_AA_CONFIG, and radeonsi sets PA_SC_AA_CONFIG to 0 for a multisampled framebuffer when rasterizer multisampling and line smoothing are off. -- [Custom MSAA sample locations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) support a uniform 1x1 pattern for 2, 4 or 8 samples through VK_EXT_sample_locations. Patterns varying across the guest's 2x2 pixel grid use a 2x2 sample location grid, anchored at the framebuffer origin like the guest's while the window offset is zero, on devices whose `maxSampleLocationGridSize` for the sample count is a multiple of 2x2, and throw on others. The RTX 5070 reports 2x4 for 2 samples, 2x2 for 4 and 1x2 for 8; an AMD Radeon integrated GPU reports 2x2 for each. Devices without the required sample count, four subpixel bits or the full guest coordinate range refuse custom patterns. Depth images remain in GENERAL between draws, and render passes cannot continue across pattern changes. Guest offsets retain the existing decoder's signed four-bit, sixteenth-pixel interpretation; PS5 coverage and interpolation equivalence have not been measured on hardware. +- [Custom MSAA sample locations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) support a uniform 1x1 pattern for 2, 4 or 8 samples through VK_EXT_sample_locations. Patterns varying across the guest's 2x2 pixel grid use a 2x2 sample location grid, anchored at the framebuffer origin like the guest's while the window offset is zero, on devices whose `maxSampleLocationGridSize` for the sample count is a multiple of 2x2, and throw on others. The RTX 5070 reports 2x4 for 2 samples, 2x2 for 4 and 1x2 for 8; an AMD Radeon integrated GPU reports 2x2 for each. Devices without the required sample count, four subpixel bits or the full guest coordinate range refuse custom patterns. A multisampled rasterizer drawing into one-sample color targets, and a one-sample depth target if any, with MSAA_ENABLE set, no alpha-to-coverage and no coverage pixel input is drawn with one sample at sample 0's location in each pixel of the grid, as the targets keep only sample 0's coverage. Without PS_ITER_SAMPLE such a draw also refuses centroid inputs, and keeps PERSP_SAMPLE, LINEAR_SAMPLE and ANCILLARY: a pixel shaded once is taken to get sample 0's barycentrics and sample index 0. With PS_ITER_SAMPLE only sample 0's invocation reaches the targets, so its sample and centroid barycentrics are taken to be sample 0's and the draw is shaded once. Both are inferred, not measured. ANCILLARY with POS_X_FLOAT or POS_Y_FLOAT throws, since reading the sample index makes the host shade per sample, which moves FragCoord from the pixel center to sample 0's location. By the same Vulkan rule, PERSP_CENTER and LINEAR_CENTER inputs on this path are interpolated at sample 0's location on the host, where the console uses the pixel center; one-sample depth images are created with `SAMPLE_LOCATIONS_COMPATIBLE_DEPTH` when the device supports it, as multisampled ones already were; this is how VK_AMD_mixed_attachment_samples defines the case, and it is not measured on PS5 hardware. Other mixed sample counts throw. Depth images remain in GENERAL between draws, and render passes cannot continue across pattern changes. Guest offsets retain the existing decoder's signed four-bit, sixteenth-pixel interpretation; PS5 coverage and interpolation equivalence have not been measured on hardware. - [DCC display buffers](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) are presented only when all keys of the console's DCC extent agree on uncompressed or a fast-clear code, padding keys included; other keys, and register-clear keys over a pending image, throw. - [Storage image write-back](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) stores the GPU results over a CPU-stamped range whose guest bytes equal the copy taken when the image's layers got their generation (images up to 1 MiB whose guest bytes were settled at that point, with no unit shadow, pending image results or recorded GPU store over them; other images keep the previous rule: the CPU's bytes win). A CPU store that rewrites the baseline value after the GPU changed it is not told apart from no store, and the GPU result wins there. The 1 MiB limit is a choice, not a console value. - [16-bit float display buffers](../../../core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp) (libSceAgcDriver, libSceVideoOut) - pixel formats `0xC001000622000000` and `0xC001000600000000` are taken as R16G16B16A16 and B16G16R16A16 float, eight bytes per texel, as [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/59183dbfa81ed7efdff6069177fed50de0552725/src/graphics/host_gpu/renderer/image/imageInfo.h) maps them and [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/VideoOut/VideoOutTypes.cs) names them (`Rgba16Float`, `Bgra16Float`); PPSA11386 registers 3840x2160 tiled buffers of the first. The tiled layout is assumed to be the 8-byte SW_64KB_R_X equation of `TextureSwizzleEquations.hpp` applied to full coordinates, as addrlib applies it, by analogy with the 32-bit scanout; `ColorTargetLayout` applies the equations to in-block coordinates, which for 8-byte texels differs in every odd 64-row block row (address bit 11 takes y bit 6) and for 16-byte texels also differs, so a float display buffer written through `WriteColorTarget` and presented from guest memory would show those rows misplaced. The values are assumed to be linear light and are encoded with the sRGB curve for the 8-bit output. Bits 32-39 of these formats are `0x06`, as in the 10-bit formats SharpProspero lists without sRGB transfer, while its sRGB formats have `0x00`, its 10-bit PQ formats `0x04` (with `0x07` in bits 40-47) and its float BT.2100 PQ formats `0x07`: they show only that these formats are not the sRGB ones. NaN and negative values give 0, values of 1 and above give 255, alpha is linear. Neither the layout nor the transfer is verified against console or title frames, and HDR output is not modelled. DCC metadata on a float buffer throws. diff --git a/docs/dev/debt/libSceConvertKeycode.md b/docs/dev/debt/libSceConvertKeycode.md index 25d15eaa60..458e175688 100644 --- a/docs/dev/debt/libSceConvertKeycode.md +++ b/docs/dev/debt/libSceConvertKeycode.md @@ -2,7 +2,7 @@ ### Unknown function info -- [sceConvertKeycodeGetCharacter](../../../core/libs/prx/libSceConvertKeycode/Export.cpp) (libSceConvertKeycode) - the signature `(uint16_t keycode, uint32_t, uint32_t keyboardType, char16_t* character)` is inferred from PPSA12361, which passes a 16-bit keycode, 0, the type from `sceConvertKeycodeGetImeKeyboardType` and a pointer it reads one UTF-16 unit from when the result is 0. The meaning of the second argument is unknown. PRAGMATA (PPSA02530, #5726) and Onimusha: Way of the Sword (PPSA27836) pass `0x4000000` as a constant: Onimusha calls it for every keycode from 0 to 0x8F to build a table of key names, upper-cases `a` to `z` itself and skips keycodes that fail or return 0, so that bit is ignored and the unshifted character is returned as for 0. Any other bit throws, and so does a keyboard type other than 0. +- [sceConvertKeycodeGetCharacter](../../../core/libs/prx/libSceConvertKeycode/Export.cpp) (libSceConvertKeycode) - the signature `(uint16_t keycode, uint32_t, uint32_t keyboardType, char16_t* character)` is inferred from PPSA12361, which passes a 16-bit keycode, 0, the type from `sceConvertKeycodeGetImeKeyboardType` and a pointer it reads one UTF-16 unit from when the result is 0. The meaning of the second argument is unknown. PRAGMATA (PPSA02530, #5726) and Onimusha: Way of the Sword (PPSA27836) pass `0x4000000` as a constant: Onimusha calls it for every keycode from 0 to 0x8F to build a table of key names, upper-cases `a` to `z` itself and skips keycodes that fail or return 0, so that bit is ignored and the unshifted character is returned as for 0. Any other bit throws, and so does a keyboard type other than 0. The returned character for HID usage `0x64` has not been measured on a console; `\` is an unverified US-layout guess based on the Windows VK mapping, which does not establish the console result. ### Functional diff --git a/docs/dev/debt/libScePsml_debug.md b/docs/dev/debt/libScePsml_debug.md index 3042611621..a24b3c1e9a 100644 --- a/docs/dev/debt/libScePsml_debug.md +++ b/docs/dev/debt/libScePsml_debug.md @@ -3,7 +3,7 @@ ### Unknown function info - [scePsmlMfsr2CreateContext, scePsmlMfsr2CreateSharedResources, scePsmlMfsr2GetContextInitRequirement, scePsmlMfsr2GetDispatchPackets, scePsmlMfsr2GetDispatchPacketsSizeInDwords, scePsmlMfsr2GetSharedResourcesInitRequirement, scePsmlMfsr2Init, scePsmlMfsr2ReleaseContext, scePsmlMfsr2ReleaseSharedResources](../../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_mfsr2) - unknown signatures, imported by PPSA23566; declared without parameters -- [scePsmlMfsrGetContextBufferRequirement1100, scePsmlMfsrCreateContext1100, scePsmlMfsrGetDispatchMfsrPacket1100](../../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_debug) - argument layouts unknown; only 0x8A810001 (not initialized) is returned +- [scePsmlMfsrGetContextBufferRequirement1100, scePsmlMfsrCreateContext1100, scePsmlMfsrGetDispatchMfsrPacket1100](../../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_debug) - argument layouts unknown; PPSA26344 imports but does not call the first two; at `0x87EFA` it passes `(rdi, rsi=&stack, rdx)` as three pointers to `GetDispatchMfsrPacket1100`. Only 0x8A810001 (not initialized) is returned - [scePsmlMfsrEnableSpecularAA, scePsmlMfsrEnableQuality8K](../../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_specularAA and libScePsml_quality8K) - imported by PPSA01317, which fails to load without them; signatures unknown, so they throw. The names are not in the aerolib NID list; they hash to the imported NIDs `xNZM9Y5lf-M` and `lcZD9tJraAw` (`tools/nid_names.py`) - [scePsmlMfsrGetDispatchMfsrPacket1000](../../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml) - unknown signature, imported by PPSA25344; declared without parameters. libScePsml and libScePsml_debug are built from the same source - [scePsmlMfsrCreateContext1300, scePsmlMfsrGetDispatchMfsrPacket1300](../../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml) - unknown signatures, imported by PPSA28416 next to `scePsmlMfsrGetContextBufferRequirement1100`; declared without parameters. The names are not in the aerolib NID list; they hash to the imported NIDs `kLF7WtQ1-Dg` and `DDVUyPyYHTA` (`tools/nid_names.py`) diff --git a/docs/dev/debt/libSceSystemService.md b/docs/dev/debt/libSceSystemService.md index 3e5bcecac2..3fa7ec0b3a 100644 --- a/docs/dev/debt/libSceSystemService.md +++ b/docs/dev/debt/libSceSystemService.md @@ -11,10 +11,11 @@ - [sceSystemServiceLaunchApp](../../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - AnyPS5 runs one title and cannot start another, so every call throws without reading its arguments. The (title ID, null-terminated argv, launch parameters) signature follows [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/SystemService/SystemService.cs#L141-L148), whose `AppLauncher` Prospero Explorer (PPSA99109) and Prospero Multi Tools (PPSA99110) import it through and which passes a 0x20-byte launch context or null; the console's argument checks and errors are unknown - [sceSystemServiceLaunchWebBrowser](../../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - the console's error for an unavailable web browser is unknown; it returns the kernel's `EOPNOTSUPP` (0x8002002D) for any arguments. The (uri, param) signature follows [RommPS](https://github.com/s0liton/RommPS/blob/754d4d60f25938a5136b9e2efcd074313c4386c5/platform/ps5/app/rommps/examples/rommps/kit/credits.cpp#L86-L95), which passes a URL and a null param and only tests the result for zero; the param layout and the console's argument checks are unknown -- [sceSystemServiceParamGetString](../../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - the system name (parameter 6) is the fixed `PS5`, not the console's name; other parameters and buffers of 1 to 64 bytes throw +- [sceSystemServiceParamGetString](../../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - the system name (parameter 6) is the fixed `PS5`, not the console's name; other parameters and buffers of 1 to 64 bytes throw. PPSA26344 at `0xA87935` passes parameter 6 with a 65-byte buffer - [sceSystemServiceParamGetInt](../../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - console language numbers 0 to 29 follow the PS4 SDK enum in [OpenOrbis](https://github.com/OpenOrbis/OpenOrbis-PS4-Toolchain/blob/06fcf7fd108fa080035cdd3259ed22eb07e0508c/include/orbis/_types/sys_service.h#L54-L83) (21 Arabic). 30 (Ukrainian) is not in it and follows PPSA99505 ([Jelly5](https://github.com/02dnot/Jelly5/blob/552b866d89b2a58b27f7f7b0484dc743ce235526/app/src/app/i18n.cpp#L29-L60)) and PPSA99002 ([ProsperoLight](https://github.com/blackbearreloaded/ProsperoLight/blob/b708bbde3457bef1073a9ab6959957cb691e8e2f/include/i18n.hpp#L252-L309)), whose numbers agree with the enum. Not confirmed on a console, and neither title starts on `main` yet. Parameter 400 returns the same number as parameter 1: PPSA01325 asks for 400 first, asks for 1 when that fails, and uses either answer as an index into the same 31 languages; in the PS5 10.01 and 12.60 `libSceSystemService` modules parameter 400 reads the same registry value as parameter 1, without parameter 1's mapping of newer language numbers by SDK version, and returns 0x80a10003 when an imported check that is not identified returns 1; the 4.03 module has no parameter 400. Neither the mapping nor that error is modelled, and the parameter's name is not known. Without `ANYPS5_LANGUAGE` the language is English (United States); the date format, time format, time zone, summer time and enter button stay fixed ### Functional - [sceSystemServiceLoadExec](../../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - besides `"exit"`, it starts the title's own `/app0/eboot.bin` or another executable under `/app0` that the relinker converted with `--executable`, which is looked up beside the running one (`/app0/picker/picker.bin` as `picker.picker.bin` with the running executable's extension). The main executable is the one `APS5_EBOOT` names, which the first call sets to its own path. After the guest shutdown the old process ends without running guest exit handlers or host destructors. `"exit"` ends the process after the host shutdown without running the title's `atexit` and `__cxa_atexit` handlers, since the console's shell closes the title instead; a line the title writes just before through its own thread, such as a forwarded standard error line, may not be printed. [VLC for PS5](https://github.com/ZiZc3/VLC/blob/86bb7d474a303f62bb3c2fc12f728db4846085b8/src/platform_ps5.c#L565-L571) (PPSA85300) relaunches itself with no arguments; RetroArch (PPSA99169) starts its frontend picker with `{"", NULL}`, and the picker starts `eboot.bin` with `{"--ps5-mode=retroarch", NULL}` or `/app0/es-de/es-de.bin` ([picker.cpp](https://github.com/mihawk-99/PS5_RetroArch/blob/bead80725cc11e3c5c65bd9424b0a4494ae36d45/frontends/picker/picker.cpp#L444-L446)), so the list holds only arguments. A list with a first entry becomes the whole guest `argv`, with no program name in front ([LoadExec.cpp](../../../core/libs/prx/libSceSystemService/LoadExec.cpp)): a [PS5 RetroArch console transcript](https://github.com/mihawk-99/PS5_RetroArch/blob/bead80725cc11e3c5c65bd9424b0a4494ae36d45/evidence/loadexec-second-image/relaunch-test.jsonl) records a restart with one argument entering with `argc=1`, `argv[0]="--ps5-relaunch=1"`, and a launch from the home screen with `argc=1`, `argv[0]=""` ([relaunch_ps5.cpp](https://github.com/mihawk-99/PS5_RetroArch/blob/bead80725cc11e3c5c65bd9424b0a4494ae36d45/src/relaunch_ps5.cpp#L161-L164)). For such a list Linux replaces the process with `fexecve`: the PID and the standard streams stay, the other descriptors are closed and the working directory becomes the executable's. Windows starts a suspended process that inherits only the standard streams and a handle of the old process, and its entry stub waits for the old process to end before it loads the title's libraries, so a shell or script that started the title sees it end. A null list or one without a first entry keeps the earlier launch: the new executable gets only its own path as its command line (`CreateProcessW` on Windows, `execv` on Linux), so its `argv[0]` is the host path, and the old process ends without waiting for it. What the console passes for such a list is not known, nor are its argument limits and error codes. Paths outside `/app0`, unconverted executables and macOS hosts throw - [Running application ID](../../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - `sceSystemServiceGetAppIdOfRunningBigApp` reports 1 as the running title's app ID. The value is AnyPS5's own: the console's shell assigns app IDs and their values are not known here, so other calls that take an app ID must use the same one. `sceSystemServiceKillApp` with that ID and the arguments -1, 0 and 0, which [SwanStationPS5](https://github.com/darkxex/SwanStationPS5/blob/4cdc981d1a64ffb7f2af73f7a27a9e91a3cb5eb4/src/main.c#L1050-L1058) passes to close itself and [ps5-payload-dev's hbldr](https://github.com/ps5-payload-dev/websrv/blob/46374edee6627885abac717de24da7a1d73af6f1/src/ps5/hbldr.c#L425-L435) passes to close the running game, ends the title as its own `exit(0)` would: the guest's shutdown and `atexit` handlers run and the process exits with status 0. On the console the call returns and the shell then ends the title, and whether those handlers run there is unknown; here there is no shell, so the call does not return. Another app ID or other arguments throw. +- [LoadExec descriptors](../../../core/libs/prx/libSceSystemService/LoadExec.cpp) (libSceSystemService) - on Linux, a call with an argument list replaces the process with `fexecve`, which first closes every descriptor above 2; a call without one uses `execv`, and the guest's descriptors stay open across it. On both paths descriptor 2 first gets back the host standard error a guest redirect kept, so the runtime's output does not follow the title's redirect (RetroArch's trace file, VLC's log) into the next executable. Which descriptors the console passes to the new process is not known diff --git a/docs/dev/debt/libSceVideoOut.md b/docs/dev/debt/libSceVideoOut.md index 6eb7fa25dc..28155782ee 100644 --- a/docs/dev/debt/libSceVideoOut.md +++ b/docs/dev/debt/libSceVideoOut.md @@ -14,9 +14,9 @@ - [sceVideoOutRegisterBuffers2](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - buffer option 8 (STRICT_COLORIMETRY) is presented as option 0, with the stored values unchanged; what the console changes for it is unknown. Only options 0 and 8 are accepted - [sceVideoOutAddBufferAttributeOption](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut, NID `MBlv8YWoyOk`) - the signature `(VideoOutBufferAttribute2*, uint64_t option)` comes from [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/60977b29103452b9681ae10c7a9011d44965bf16/src/SharpProspero/Interop/VideoOut/VideoOut.cs#L174-L176) and the call site in PPSA25380, which passes 0x20 and ignores the result. The option is ORed into `attribute->option`; whether the console validates the bits, keeps them elsewhere or fails on an unknown option is unknown. A null attribute throws. Option 0x20 makes `sceVideoOutRegisterBuffers2` throw, as every option other than 0 and 8; what it selects for an overlay buffer is unknown - [sceVideoOutRegisterBuffers2](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - DCC buffers (category 1): `dcc_control` is taken as the CB_DCC_CONTROL block layout, only bits 0x10026c are accepted; `dcc_cb_register_clear_color` as the register-clear texel, only 32-bit values are presented -- [sceVideoOutIsOutputSupported](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - mode 1 is supported; mode 15 is supported when the current host display runs at 120 Hz or above. Configuring mode 15 requires attribute3 bit 0x40; this single-bit mask is inferred from the measured full value 0x80040, not isolated by a console measurement. Vblanks run at 119.88 Hz until mode 1 restores 59.94 Hz. Display modes are read through SDL; a move to a slower display rejects further high-frame-rate presentation. Selecting another host display mode and VRR are not implemented. The console answers per display and per title: with a 120 Hz TV it also answers 1 for mode 15 (119.88 Hz), and [PS5_Vulkan](https://github.com/mihawk-99/PS5_Vulkan/blob/5b5e4fc2d80fdb67a4f61ba9e6a8a424026bb8a5/docs/HARDWARE_FINDINGS.md) measured that `sceVideoOutConfigureOutput` accepts 15 only from a title whose param.json declares high-frame-rate output (attribute3 0x80040). Modes 4, 7, 8, 12-14, 16-17 and 19 return `VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE` here, as they did on the measured display. Mode 18 throws, because one console answered 0x80290016 and another 0x8029001E. The name `VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE` for 0x8029001E is ours. Word 3 of the output options is accepted with any value, as on the console, and its meaning is unknown. PPSA12544 probes `0xd000000a`, which the console answers with 0x8029001E, so the title selects 3840 pixels +- [sceVideoOutIsOutputSupported](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - mode 1 is supported; mode 15 is supported when the current host display runs at 120 Hz or above. Configuring mode 15 requires attribute3 bit 0x40; this single-bit mask is inferred from the measured full value 0x80040, not isolated by a console measurement. Vblanks run at 119.88 Hz until mode 1 restores 59.94 Hz. Display modes are read through SDL; a move to a slower display rejects further high-frame-rate presentation. Selecting another host display mode and VRR are not implemented. The console answers per display and per title: with a 120 Hz TV it also answers 1 for mode 15 (119.88 Hz), and [PS5_Vulkan](https://github.com/mihawk-99/PS5_Vulkan/blob/5b5e4fc2d80fdb67a4f61ba9e6a8a424026bb8a5/docs/HARDWARE_FINDINGS.md) measured that `sceVideoOutConfigureOutput` accepts 15 only from a title whose param.json declares high-frame-rate output (attribute3 0x80040). Modes 4, 7, 8, 12-14, 16-17 and 19 return `VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE` here, as they did on the measured display. Mode 18 throws, because one console answered 0x80290016 and another 0x8029001E. The name `VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE` for 0x8029001E is ours. Word 3 of the output options is accepted with any value, as on the console, and its meaning is unknown. PPSA12544 probes `0xd000000a`, which the console answers with 0x8029001E, so the title selects 3840 pixels; PPSA26344 at `0xCD0946` passes `(handle, 0xf, nullptr, nullptr, 0)` and branches on the result equalling 1 - [sceVideoOutGetResolutionStatus](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the 48-byte layout follows the declarations in ps5-opengl and EVO-PLAYER-PS5 (#1578), which agree; not confirmed on a console. `screenSizeInInch` is 0, the value EDID uses for an undefined size, since the host display's size is not read; `flags` is 0, its bits are unknown. The pane equals the full size because `sceVideoOutSetWindowModeMargins` applies no margins -- [sceVideoOutOpen](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the open param's first word is unknown (PPSA21564 passes 16; it is not the byte size, since the affinity mask is at offset 16); only 16 is accepted. Whether the param continues past offset 24 is also unknown +- [sceVideoOutOpen](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the open param's first word is unknown (PPSA21564 passes 16; it is not the byte size, since the affinity mask is at offset 16); only 16 is accepted. PPSA26344's only call at `0xCCF5EA` passes `(0xFF, 0, 0, nullptr)`. Whether a non-null param continues past offset 24 is also unknown - [sceVideoOutGetBufferLabelAddress](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) exposes a port-owned contiguous array of 16 aligned 64-bit labels and returns its count. Astro Bot PPSA21564's bundled AGC driver passes a pointer output and addresses each label at `base + index * 8`. The count, zero initialization on registration, and release of the previous displayed buffer when its replacement completes follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/23c41333822fe8eca4c609b43119180af63ae4ac/src/core/libraries/videoout/video_out.cpp) and its driver; PS5 console equivalence has not been measured. Labels live until the port closes, and reopening creates a new array. Error precedence follows the reference for null output and the existing open-port policy for handles. Exact scanout release timing, repeated same-buffer flips and concurrent guest label stores remain unverified. EOP-flip registration and interrupt tokens remain unimplemented; exposing these labels does not implement that separate API. - [GPU flips](../../../core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp) (libSceVideoOut) - a command-buffer flip of a buffer index with no registered buffer when the GPU reaches it presents black and completes like any flip (flip event, count, `flipPendingNum`). The console's behaviour is unknown; PPSA05684 requires that it completes: its renderer submits a flip of buffer 0 while another thread waits for `sceVideoOutIsFlipPending` to reach 0 before registering the buffers, and `flipPendingNum` counts that flip from its submission as in shadPS4 - [sceVideoOutAddPreVblankStartEvent](../../../core/libs/prx/libSceVideoOut/src/Event.cpp) (libSceVideoOut) - how long before the vblank the console signals pre-vblank start is unknown. The event is triggered on each emulated vblank tick immediately before the vblank event, with its own count as the event data diff --git a/docs/dev/debt/libSceVoiceQoS.md b/docs/dev/debt/libSceVoiceQoS.md index 7b3937b6a9..5700fed07c 100644 --- a/docs/dev/debt/libSceVoiceQoS.md +++ b/docs/dev/debt/libSceVoiceQoS.md @@ -3,7 +3,7 @@ ### Unknown function info - [sceVoiceQoSSetConnectionAttribute](../../../core/libs/prx/libSceVoiceQoS/Export.cpp) (libSceVoiceQoS, NID `KCGOxg8iX7s`) is a throwing stub declared without parameters, like the other endpoint calls of the library: PPSA04159 fails to load without it and its signature is unknown. The name hashes to the imported NID (`tools/nid_names.py`) -- [sceVoiceQoSInit](../../../core/libs/prx/libSceVoiceQoS/Export.cpp) (libSceVoiceQoS) - `0x804E0905` for a second initialization comes from [prosper](https://github.com/mattias800/prosper)'s reading of the module and was not measured: on a PS5 a valid call blocks inside a homebrew payload, so it never returns there. The app type and the block are not used, as no endpoint is implemented +- [sceVoiceQoSInit](../../../core/libs/prx/libSceVoiceQoS/Export.cpp) (libSceVoiceQoS) - `0x804E0905` for a second initialization comes from [prosper](https://github.com/mattias800/prosper)'s reading of the module and was not measured: on a PS5 a valid call blocks inside a homebrew payload, so it never returns there. PPSA26344's two call sites (`0xE307BF`, `0x151C45D`) pass an allocated pointer, a 0x40000-byte block and app type 0x10000000; the app type and block are not used, as no endpoint is implemented ### Functional diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index eb244ed912..907400a5c0 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -50,6 +50,7 @@ - [sinhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `sinhf` of the `libc.prx` bundled with PPSA28319 (disassembly): FreeBSD msun's [`e_sinhf.c`](https://github.com/freebsd/freebsd-src/blob/5aa554c7e5b6fd9042986df07f12ac5687e41392/lib/msun/src/e_sinhf.c) from before `__ldexp_expf`, which squares `expf(|x| / 2)` between `logf(FLT_MAX)` and the overflow threshold, calling msun's [`expm1f`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/s_expm1f.c) and [`expf`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_expf.c), whose paths for the arguments it passes are inlined here. Past the threshold Sony's error helper sets `errno` to `ERANGE` before returning `x * 1e37`; msun itself sets no `errno`. That code, run on a Windows host, gives the same bits as this implementation for every finite argument up to the threshold, of both signs; it was not run on Linux. That the system `libSceLibcInternal`, which Ruffle (PPSA68091) and RetroArch (PPSA99169) import while bundling a zero-return `libc.prx`, has the same code is assumed - [asinl](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `asinl` of the `libc.prx` bundled with PPSA28319 (disassembly), FreeBSD msun's [`e_asinl.c`](https://github.com/freebsd/freebsd-src/blob/9f0432041b8fb69832f10c32da50a67f34feeb61/lib/msun/src/e_asinl.c) with the ld80 [`invtrig.c`](https://github.com/freebsd/freebsd-src/blob/9f0432041b8fb69832f10c32da50a67f34feeb61/lib/msun/ld80/invtrig.c) coefficients, which the binary holds as the same 80-bit constants. For |x| > 1, the infinities and NaN, Sony's error helper raises `FE_INVALID` and sets `errno` to `EDOM` before returning `(x - x) / (x - x)`, so a NaN argument also sets `EDOM`; msun itself sets no `errno`. 20 million random arguments in (-1, 1) give the same bits as that code run on a Windows host. The result needs the x87 precision control at 64 bits, as FreeBSD amd64 starts threads (control word `0x37f`), but guest threads do not set it: on Windows a thread started by `_beginthreadex` runs at 53 bits (`0x27f`), where `asinl(0.75)` returns mantissa `0xd91a98ae3406e000` instead of `0xd91a98ae3406e041`. The system `libSceLibcInternal` that RetroArch (PPSA99169) imports is assumed to have the same code - [atanhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `atanhf` of the `libc.prx` bundled with PPSA28319 (disassembly): FreeBSD msun's [`e_atanhf.c`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_atanhf.c) calling msun's [`log1pf`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/s_log1pf.c), whose paths for the positive arguments it passes are inlined here; the exported `log1pf` is still the host's. For |x| > 1, the infinities and NaN, Sony's error helper raises `FE_INVALID` and sets `errno` to `EDOM` before returning `(x - x) / (x - x)`, and for |x| = 1 it raises `FE_DIVBYZERO` and sets `errno` to `ERANGE` before returning `x / 0`; msun itself sets no `errno`. That code, run on a Windows host, gives the same bits, `errno` and floating-point exceptions as this implementation for all 2^32 arguments; it was not run on Linux. That the system `libSceLibcInternal`, which Dusklight (PPSA99305) imports while bundling a zero-return `libc.prx`, has the same code is assumed +- [getenv, setenv, unsetenv, putenv](../../../core/libs/prx/libc/src/Environment.cpp) (libc) keep one guest environment that starts empty, like the `environ` libkernel exports, and never reads or changes the host's. A title sees only the variables it sets itself; host variables such as `HOME` and `PATH` name host paths a guest cannot use (PS5MC, PPSA99420, exits with `XBMC_Run returned -1` when it gets the host's `HOME`). Whether the console starts a title with any variables set is not measured - On macOS the [write watch](../../../core/libs/prx/libc/src/GuestWriteWatch.cpp) write-protects watched pages and records the first write to each from a SIGBUS/SIGSEGV handler, because macOS has no counterpart to `userfaultfd` write-protect or `MEM_WRITE_WATCH`. The protection is real, so: - a system call that writes into a protected page fails with `EFAULT` instead of faulting. `GuestArena::HostWrite` lifts the protection for the calls it brackets (file reads and socket receives); a host call that writes guest memory without it (`clock_gettime` and the like into a watched page) fails; - code that changes the protection of watched pages, or maps over them, has to call `GuestWriteWatchProtectionChanged`, and code that asks the kernel for a page's protection has to ask `GuestWriteWatchProtection` first, since a watched page is read-only between a collection and the next write. Nothing calls either yet: the macOS memory paths are not in the tree; @@ -81,3 +82,4 @@ - [std::_Throw_Cpp_error](../../../core/libs/prx/libc/src/exception/StandardExceptions.cpp) (libSceAmpr) follows Dinkumware's `thread0.cpp` as shipped with [Visual C++ 2013](https://github.com/ganboing/crt.vc12/blob/f1d47126bf13d1c9dccaa6c53b72217fd0474968/src/thread0.cpp) and in [microsoft/STL](https://github.com/microsoft/STL/blob/da52dc0fcadbd83690cfd1ed32408d2847439493/stl/src/thread0.cpp), without console verification. `_Cpp_error` codes 0 to 6 throw a guest `std::system_error` whose `error_code` holds the FreeBSD errno of the `errc` value (EBUSY 16, EINVAL 22, ESRCH 3, ENOMEM 12, EPERM 1, EDEADLK 11, EAGAIN 35) and the generic category, and whose `what()` is the table message, `": "` and the generic category's message for that errno, selected by the first matching errno entry in VC12's [`syserror.cpp`](https://github.com/ganboing/crt.vc12/blob/f1d47126bf13d1c9dccaa6c53b72217fd0474968/src/syserror.cpp); another code throws `std::out_of_range` on the host instead of reading past the table. The guest `system_error` is taken to be [VC12's](https://github.com/ganboing/crt.vc12/blob/f1d47126bf13d1c9dccaa6c53b72217fd0474968/src/system_error): the 16-byte `runtime_error` followed by `error_code {int, const error_category*}`, 32 bytes, so its copy constructor and assignment copy the code too. The generic category is a static libc object with VC12's `error_category` virtual order (destructors, `name`, `message`, `default_error_condition`, the two `equivalent`), not the console's `_Error_objects::_Generic_object`, and the new `generic_category()` export returns that object; a title's inline category object in a bundled libc can still have a different identity. Its `message` and direct message export construct the existing [GuestLocale::String](../../../core/libs/prx/libc/include/GuestLocale.hpp) model: a reserved eight-byte word, 16-byte inline buffer, length and capacity, with guest `operator new` storage above 15 bytes. This 40-byte string layout and virtual return ABI have not been verified against a console library. The deleting destructor still throws. `_Syserror_map` covers the 72 distinct Dinkumware `errc` values present in FreeBSD 11.4 [errno.h](https://github.com/freebsd/freebsd-src/blob/releng/11.4/sys/sys/errno.h), returning null for other values; category messages use `unknown error` for those. First-match aliases are `EAGAIN`/`EWOULDBLOCK` -> `operation would block` and `ENOTSUP`/`EOPNOTSUPP` -> `not supported`. STREAMS-only errno assignments absent from that header (`ENODATA`, `ENOSR`, `ENOSTR`, `ETIME`) are not inferred - [Call tracing](../../../core/libs/prx/libc/src/specifics/windows/CallTrace.cpp) (`--trace-imports`, `APS5_TRACE_CALLS`) exists on Windows only and records the entry of a call, not its arguments or its result. One process-wide ring keeps the last 65536 calls, so a thread that calls often pushes the others out. Calls are also counted per traced target since start-up; an import bound to a target already traced under another name is counted under the first name. The thunk saves the integer argument registers and `xmm0` to `xmm7`; arguments in wider vector registers pass only as long as the recording code stays free of AVX instructions, which is not tested - [_Dtest](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows Dinkumware double classification matching _FDtest: finite values return -1, subnormals -2, zeros 0, infinities 1, and NaNs 2. +- [app0.zar](../../../core/libs/prx/libc/src/AppArchive.cpp) (libc) - a [ZArchive](https://github.com/Exzap/ZArchive) beside `app0/` is a read-only layer under it: a path missing from `app0/` is looked up in the archive, case-insensitively. It is not a console format; titles ship their files in a PFS image instead. Resolving an archive-only path through any call other than the archive-aware libkernel ones throws, so `fopen`, `stat`, `access`, `opendir` and the rest do not read the archive yet, and `opendir` of a directory that exists in both places, `app0/` itself included, and `fdopen` of an archived descriptor throw instead of missing its entries or reading the null device. A damaged compressed block throws instead of returning `EIO` diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 5396a00c3a..7d2d75e609 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -200,3 +200,4 @@ - [getpriority](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) follows FreeBSD 11.4's [`sys_getpriority`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/kern_resource.c#L93-L165) for the only guest process, which is taken to run at nice 0: `PRIO_PROCESS` and `PRIO_PGRP` with 0 or its own pid (the guest is taken to lead its own process group, see `kill`) and `PRIO_USER` with 0 or its user id 0 return 0, any other id fails with `ESRCH` and any other `which` with `EINVAL`. As in FreeBSD's libc, where `getpriority` is the bare system call, errno is left unchanged on success. The nice value of a title process on the console is unverified, and the host nice of the process or of the threads `APS5_THREAD_PRIORITY=high` raises is not reported. `setpriority` accepts the same target ids (any other id fails with `ESRCH`, any other `which` with `EINVAL`), clamps the value to FreeBSD's `PRIO_MIN`-`PRIO_MAX` (-20 to 20) and stores it as the guest process's nice value, which `getpriority` returns; the host scheduling class is not changed, and whether a title process may lower the value without privilege is unverified - [rtprio_thread, pthread_setconcurrency, pthread_getconcurrency](../../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - `rtprio_thread` follows FreeBSD 11.4's `rtprio_thread(2)`. Every guest thread here runs under `SCHED_FIFO` or `SCHED_RR` (priority 256-767), whose rtprio class is real-time; the console's mapping from that priority to the rtprio `prio` range (0-31) is unmeasured, so `RTP_LOOKUP` and `RTP_SET` throw for such a thread rather than report an impossible priority. A thread that is not real-time reports `RTP_PRIO_NORMAL` (3) with 0, and `RTP_SET` accepts only that value. Another function fails with `EINVAL`, a null `rtprio` with `EFAULT`, a thread other than the calling one throws, and an id that is not a live guest thread with `ESRCH`. `pthread_setconcurrency` stores 0 or more and returns `EINVAL` for a negative level (the error itself, as FreeBSD's libthr does, with errno untouched); `pthread_getconcurrency` returns the level, which is a hint and changes nothing about the host's concurrency. Morrowind (`mshivam019/morrowind-ps5`) imports all three at start-up - [sceKernelVirtualQuery](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) returns [VirtualQueryInfo](../../../core/libs/SceTypes.hpp) with a one-byte flag allocation unit so Windows and SysV hosts return the same 72-byte representation, with the name at byte 33 and GPU mask at byte 65. This preserves the existing SysV representation; the console offsets beyond the address, protection and size fields observed in Astro Bot's AGC calls are not hardware-measured. Size and offset assertions plus a raw-byte named-mapping regression guard the host ABI. +- [app0.zar files](../../../core/libs/prx/libkernel/File/src/ArchivedFile.cpp) (libkernel) - `sceKernelOpen`, `open`, `sceKernelStat` and `sceKernelCheckReachability` resolve a path missing from `app0/` in `app0.zar`. An archived file is a read-only host null device whose reads, positioned and vectored reads, AIO reads, seeks, `fstat` and `FIONREAD` are served from the archive, with the position shared by `dup`, `dup2` and `F_DUPFD`; writes fail with `EBADF` as on a read-only descriptor, opening one for writing or truncation fails with `EROFS` and `O_CREAT | O_EXCL` with `EEXIST`. `fstat` and `sceKernelStat` report mode `0444` (`0555` for a directory), the node index as inode and the archive file's device and times. Opening an archived directory or a directory present both on disk and in the archive throws, as `getdents` would miss entries. On an archived descriptor `mmap`, `flock`, record locks, `fstatfs`, `ftruncate`, `fsync`, `fdatasync`, `getdirentries`, `fchmod` and `futimes` throw instead of acting on the null device; `mmap` is `noexcept`, so its refusal terminates the process. `lstat` and the other path calls on archived files are not implemented diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 9a5d65d1aa..ec051f71ed 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -86,7 +86,7 @@ - [v_interp_p1ll_f16, v_interp_p1lv_f16 and v_interp_p2_f16](../../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp) read the 16-bit halves of each vertex's parameter through `VK_KHR_fragment_shader_barycentric` and throw without it, as fixed-function interpolation cannot split a 32-bit attribute. P10 and P20 are the vertex differences rounded to f16 to nearest even, which is not confirmed on hardware. v_interp_p1ll_f16 and v_interp_p1lv_f16 round the exact `P10 * I + P0` once to f32 and v_interp_p2_f16 rounds the exact `P20 * J + S2` once to f16, as measured on RDNA2 (#697); the exact sums are computed in f64, so these instructions need the Float64 capability. They follow the shader's float mode, measured on RDNA2 (gfx1035) in all eight combinations of `IEEE_MODE` and kept/flushed f32 and f16 denormals over 768 special and 1,024 random rows. With IEEE=1 a NaN operand is returned quieted; with IEEE=0 it is returned unquieted, except that v_interp_p2_f16's f16 result is always quiet. Flushed f32 denormals flush I/J, v_interp_p2_f16's f32 S2 and the f32 result; flushed f16 denormals flush P0, P10/P20, v_interp_p1lv_f16's f16 S2 and the f16 result. Denormal modes 1 and 2 (flush only inputs or only outputs) and rounding other than to nearest even are not measured and throw. An unknown mode follows IEEE=0 with denormals kept. Lavapipe has no barycentrics, so the draw test is skipped in CI; `agc_driver_interpolation_modes` checks the arithmetic in every mode against the hardware rows on any device with Float64. - Without `VK_KHR_fragment_shader_barycentric`, a pixel shader whose I/J VGPRs [only feed `v_interp_p1_f32`/`v_interp_p2_f32` of their own pair](../../../core/shader/recompiler/Optimization/src/HostInterpolationChecker.cpp) reads the attribute Vulkan interpolated. Any other read of them or of a raw vertex parameter takes the weights and the three vertex values from a [generated geometry stage](../../../core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp), which needs `geometryShader` and a vertex shader before it: tessellation, mesh and rect-list pipelines, primitives other than triangles, more than 32 fragment input locations and interfaces beyond the device's geometry limits throw. - [v_fma_f64 and v_div_fmas_f64](../../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp) with a subnormal operand or result round once, as IEEE 754 fusedMultiplyAdd does with f64 denormals kept, measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1, `v_div_fmas_f64` with VCC clear and set. The f16/f64 denormal field of `FLOAT_MODE` (bits 7:6) is not read: when it flushes, the hardware flushes the operands and the result of `v_fma_f64` (tininess before rounding) and only the result of `v_div_fmas_f64`, while these keep them. Shaders with f64 instructions declare `DenormPreserve` for 64-bit floats only when `shaderDenormPreserveFloat64` is true and Vulkan reports `denormBehaviorIndependence` as `ALL`; other independence modes are not handled by the current backend because they require coupled execution modes. See [VkShaderFloatControlsIndependence](https://docs.vulkan.org/refpages/latest/refpages/source/VkShaderFloatControlsIndependence.html). Elsewhere (including lavapipe), the driver may flush f64 subnormals, and `agc_driver_f64_fma_subnormals` is skipped. -- [SINT storage images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8- and 16-bit components saturate each stored 32-bit value to the component range, as adriadam10 measured for image stores on RDNA2 (gfx1035) in #944 (448 rows match); the PS5's gfx1013 is not measured. +- [SINT storage images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8- and 16-bit components saturate each stored 32-bit value to the component range, as adriadam10 measured for image stores on RDNA2 (gfx1035) in #944 (448 rows match); the PS5's gfx1013 is not measured. A D16 store sign-extends each 16-bit half before that saturation, as measured on RDNA2 (gfx1035) for `image_store d16` to all eight SINT formats with DMASK 0xf, 0x7, 0x5, 0x3, 0x2, 0xa and 0x1 (1792 rows match). - [`StoreTexel`](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) writes channel 1 of a two-channel format from the data slot whose DST_SEL is Y, for SINT and UINT alike. With DST_SEL X,Y,Z,W, gfx1035 writes it from the W slot, as adriadam10 measured in #944. - [v_ldexp_f16](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the `FLOAT_MODE` f16 denormal field: bit 0 preserves the half input and bit 1 the half result. Otherwise a denormal input is flushed to its signed zero, and a result whose exact scaled value is below the minimum normal (2^-14) is flushed to its signed zero before rounding, so `v_ldexp_f16 0x07ff, -1` gives +0 instead of 0x0400. Measured on RDNA2 (gfx1036) with round to nearest, IEEE 0 and 1 and denorm16 0..3, over every half magnitude 0x0000..0x0fff of both signs scaled by 2^-1 and 2^0 (16384 rows per mode): no mismatch. NaN and infinity handling and destination high-half preservation are unchanged. Shaders without a known float mode keep the previous lowering. - [v_mullit_f32](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted only with IEEE=1, as measured at IEEE=0 too; a positive denormal src2 counts as positive); with f32 denormals flushed it follows the f32 denormals entry below, and denormal modes 1 and 2 throw. @@ -116,7 +116,7 @@ - The [output modifier](../../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) (`mul:2`, `mul:4`, `div:2`) follows the shader's float mode as measured on RDNA2 (gfx1035): the hardware applies it only with IEEE mode off and output denormals of the result's precision flushed (bit 1 of the `FLOAT_MODE` denormal field clear), and ignores it otherwise. On f32 results it is applied as the hardware does with f32 denormals flushed; a shader without a known float mode is translated with IEEE mode off and f32 denormals flushed, the mode titles use. On f16 and f64 results it is ignored, and it throws in the modes where the hardware applies it. - The [clamp modifier](../../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follows `DX10_CLAMP`, measured on RDNA2 (gfx1035) for f32, f16, packed f16 and f64 results at IEEE=0 and IEEE=1: with `DX10_CLAMP`=1 (and in shaders without a known float mode) a NaN result clamps to +0, including the quieted NaN that `v_pk_min_f16` and `v_pk_max_f16` return for a signaling NaN operand with IEEE=1 (measured on RDNA2 gfx1036 at both `IEEE_MODE` and both `DX10_CLAMP` settings); with `DX10_CLAMP`=0 a NaN result is written unchanged and every other value clamps to [0, 1] as before. f64 results follow the same rule on hardware, but a clamped f64 result with `DX10_CLAMP`=0 throws until it has an execution test. With `DX10_CLAMP`=0 the NaN's bits are those the translation already produces for the instruction without clamp, so where those are left to the driver (f32 and scalar f16 arithmetic) the payload is the driver's, not the hardware's input propagation. `v_dot2_f32_f16` and the integer clamps are unchanged. - [FP16_OVFL](../../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follows the shader's float mode as measured on RDNA2 (gfx1036) at IEEE=0 and IEEE=1 with f16 denormals kept and flushed: with `FP16_OVFL`=1 an f16 result that rounds to ±infinity is written as ±0x7bff unless a source operand is infinite, so `v_rcp_f16`, `v_rsq_f16` and `v_log_f16` of ±0 and `v_exp_f16` of 16 or more are clamped while an infinite input passes through, and `v_div_fixup_f16` keeps the infinity it writes for a zero denominator or an infinite numerator and clamps any other infinite quotient, a NaN one included. `v_cvt_pkrtz_f16_f32` rounds toward zero and never overflows. The sources are checked as the f32 values the translation computes with; the rule is applied to `v_fma_mix*_f16` with f32 sources and to `v_interp_p2_f16` without a measurement of infinite f32 sources or parameters. Round modes other than nearest even are not modelled, so the overflow threshold is the nearest-even one (65520). -- [Directed f16 rounding](../../../core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp) for `v_cvt_f16_f32`, `v_cvt_f16_i16`, `v_cvt_f16_u16` and `v_ldexp_f16` follows `FLOAT_MODE` bits 3:2 only when f16 denormals are preserved (bits 7:6 = 3), measured on RDNA2 (gfx1036). With f16 denormals flushed (bits 7:6 = 0), or flushed on one side (modes 1 and 2, not measured), the hardware also flushes the f16 result and the directed path does not model that, so those shaders keep round-to-nearest-even. The `clamp` modifier with directed rounding is not measured. +- [Directed f16 rounding](../../../core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp) for `v_cvt_f16_f32`, `v_cvt_f16_i16`, `v_cvt_f16_u16` and `v_ldexp_f16` follows `FLOAT_MODE` bits 3:2 in every f16 denormal mode, measured on RDNA2 (gfx1036 with f16 denormals preserved, gfx1030 in all sixteen combinations of f16 rounding and denormal mode). When bit 7 flushes f16 results, `v_cvt_f16_f32`, `v_cvt_f16_i16` and `v_cvt_f16_u16` write a zero of the result's sign for a value that, rounded in that mode with an unbounded exponent, is below 2^-14, so a value just below 2^-14 that rounds up to it stays 0x0400; `v_ldexp_f16` keeps its rule above. The `clamp` modifier is applied before the rounding. On gfx1030 every row matches: 8,996 f32 inputs around the half denormal, normal and overflow boundaries with and without `clamp`, every 16-bit integer, and 2,470 `v_ldexp_f16` operands. - [Comparison sampling of a color texture](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_sample_c`/`image_sample_c_lz` on a texture that is not R32 float or R16 unorm) is emulated in the shader, since Vulkan compares only depth formats: the red channel is fetched, compared and, for bilinear filtering, blended (compare, then filter). Only 2D, 2D array and cube views, float/unorm/snorm formats, wrap, clamp-to-edge and clamp-to-border addressing (border red 0 for the black borders, 1 for opaque white) and point or bilinear filtering at one level are implemented, with texel offsets added to the texel indices; LOD biases and clamps are accepted, as they cannot select another level of a single-level sample. Half-border addressing, the border color table, gradients, gathers, other dimensions, anisotropy (except on the 8-bit formats below) and mip filtering throw. Under bilinear filtering, taps past a clamp-to-border edge compare the border value and blend with the texel taps, so results near the edge depend on the filter weights. AMD documentation does not say how a color format's reference is handled: it is clamped to [0, 1] for unorm and [-1, 1] for snorm, as Vulkan does for unorm depth, and not clamped for float. 8-bit sRGB and UNORM (`Format8Srgb`, `Format8_8Srgb`, `Format8_8_8_8Srgb`, `Format8UNorm`, `Format8_8UNorm`, `Format8_8_8_8UNorm`) return 0 for every compare function, measured with `image_sample_c_lz` on gfx1035 over references -0.5..1.5 and texel codes 0..255. A bilinear cube of those 8-bit formats without `DISABLE_CUBE_WRAP`, and an anisotropic S# on them, return 0 for the same reason; float, snorm and the other unorm bilinear cubes without that bit still throw. Bilinear weights are full float, not the hardware's fixed-point subtexel weights - [Negative color-texture comparison wrap indices](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (libSceAgcDriver) use explicit `OpSRem` normalization to implement modulo; the negative-offset point-wrap regression fails with direct `OpSMod` and passes with normalization on Mesa 22.3.6 llvmpipe (LLVM 15.0.6), but negative-index wrap has not been measured on PS5 hardware. - [Converted unorm images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view): samples, gathers, LOD queries, 16-bit data and a DST_SEL of the missing W channel throw. @@ -146,3 +146,4 @@ - [SDWA source selects on `v_cvt_f32_ubyte1`, `v_cvt_f32_ubyte2` and `v_cvt_f32_ubyte3`](../../../core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp) (shader recompiler) - byte N is taken from source 0 after `SRC0_SEL` and `SRC0_SEXT` select and extend it, since SDWA supplies source 0 (RDNA 2 ISA guide 6.8 and 13.3.7) and the opcode reads `S0.u[8N+7:8N]` (12.8). Not measured on hardware. - [Flag structurization](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a graph that every structurizer attempt refuses, irreducible and non-terminating graphs included, is rebuilt with one reach flag per block and one again flag per loop (`structurizeWithFlags`): each block keeps one copy and runs behind a guard on its flag, and each strongly connected component becomes a loop whose single latch repeats it while a back edge set its flag. The output has about 8 times the input's blocks and one route variable per block, and ends in an empty Return that never runs. Only blocks that start the top-level order unconditionally run unguarded, so resource planning also sees the static paths that skip a guarded block and can plan extra descriptor candidates for them. The attempts still run first. Shared merge splitting refuses an attempt once the graph has grown past 16 times its block count at the start of splitting (at least 64 blocks); succeeding attempts grew at most 10.6 times on random graphs of up to 128 blocks, and an attempt that would succeed only beyond the budget leaves the graph to a later attempt or to this construction. Splitting that adds one block per round stays within that budget until its 4 rounds per block run out, so the attempts can still take more than 10 s (9 of 2,000 random graphs of up to 128 blocks, 63 s at most). `APS5_FORCE_STRUCTURIZER_FALLBACK=1` builds every graph this way, for tests - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) use only the base address, stride and record count for resources accessed exclusively by scalar loads; their fourth word is normalized to zero, following AMD RDNA2 ISA section 7.2.2. Vector and mixed accesses retain descriptor validation. Native Vulkan readbacks cover ignored fourth-word fields on Windows/NVIDIA; their console behavior has not been measured. +- [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - `APS5_SPANNED_WAVE=1`, off by default, runs a single-wave wave64 compute program at one lane per invocation on a 32-lane host subgroup and exchanges ballots, shuffles and reductions between the two host subgroups through workgroup memory and barriers. The driver turns it on only where compute subgroups can be pinned at 32 full lanes (`subgroupSizeControl` and `computeFullSubgroups`), passes it as `SpirvTarget::spannedWaves` (part of the cache key and the serialized request), and creates the pipelines of spanned programs with `REQUIRE_FULL_SUBGROUPS`, since the lane number `SubgroupId * 32 + SubgroupLocalInvocationId` needs two full subgroups. Those barriers are correct only where every invocation of the workgroup reaches them, which holds for the guest's wave-uniform control flow. The emulated 64-bit LDS atomics ([LockedLdsUpdate](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp)) run only on the lanes the exec mask keeps, so their lock loop ballots within its own host subgroup, as its elect already does. Any other cross-lane exchange emitted inside per-lane control flow would hang a spanned wave in the same way, and the emitter does not reject one: there is no uniformity check. The only check so far is manual: disassemble the module (`spirv-dis`), list the selection and loop constructs that enclose each `OpControlBarrier`, and read the condition of each enclosing selection. In Ghost of Yōtei's program 0x80003af600 (PPSA26344), 108 of 111 barriers sat only in loops on wave-uniform conditions and 3 inside per-invocation conditionals, all in the lock loop. Other programs and titles have not been scanned. An earlier, cheaper ballot (one barrier instead of three) also lost the device for a reason that is not understood, which may be another barrier placement problem. Measured on an RTX 4090 (Windows): an offline replay of that program and two Yōtei runs that previously lost the device (NVIDIA TDR) now run past it, and [LdsMaskWaterfall.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp) runs both modes and covers a half-inactive wave. Stray (PPSA02101) on Intel ANV and a Demon's Souls (PPSA01342) request replay showed no fault, before this fix diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 57f4ba8cdd..73c49d52c6 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -107,6 +107,8 @@ Use `sce_modules/` or `prx/` instead of `sce_module/` if that is the input direc Some applications also need their original `app0/sce_sys/param.json` at runtime. AnyPS5 reads this file for the title ID and display name; it must contain a non-empty `titleId` string and a non-empty `localizedParameters` object with a language entry that has a non-empty `titleName`. It selects the entry named by `defaultLanguage` (inside `localizedParameters`, or at the top level); if that entry is absent, it tries `en-US`, then the first object-valued language entry. Preserve the values from the title's own metadata and place the file under the same `app0/` directory as the other app resources. At startup, its presence in the working directory creates `data/`; a non-zero `downloadDataSize` also creates `download0/`. Without the file, neither directory is created. If it is missing or invalid, metadata calls such as title/window setup can fail with a `param.json` error. +Resources can also be packed into `app0.zar` beside `app0/`, a [ZArchive](https://github.com/Exzap/ZArchive) whose root is the `app0/` directory. A file present in `app0/` is read from there and any other file from the archive; the executable, `sce_module/` and `sce_sys/` stay in `app0/`. The archive is read only through `sceKernelOpen`/`open`, the read, seek and stat calls on its descriptors and `sceKernelStat`; other calls on an archived path, listing a directory that the archive also contains and opening a file in it for writing fail as described in [technical debt](../dev/debt/libkernel.md). An archive that cannot be opened stops the title at its first file access. + Use the generated files printed as `Guest module:` for bundled title modules. `libs/` is for AnyPS5 system libraries, not the original PS5 `.prx` files. Placing an original PS5 module in `libs/` on Windows makes Windows try to load it as a DLL and can fail with error 193 (not a valid Win32 application). On Windows, a self-built `libs/` also needs `libgcc_s_seh-1.dll`, `libstdc++-6.dll` and `libwinpthread-1.dll` from the `mingw64/bin` directory of the toolchain that built the libraries; the release archives already contain them. The libraries are loaded without searching `PATH`, so a copy elsewhere on the system is not used, and a library that needs one of them fails with error 126 (the specified module could not be found) although its `.prx` file is present. @@ -179,6 +181,10 @@ HDR output is available on Windows only. Titles see it when the monitor showing Recompiled shaders and the Vulkan pipeline cache are kept in a `shader_cache/` directory beside the output executable, so later runs skip recompilation. Set `ANYPS5_SHADER_CACHE_DIR` to use another directory, or set `ANYPS5_NO_SHADER_CACHE` to any non-empty value other than `0` to disable both caches. Shaders are stored in a subdirectory named after the recompiler's source version; subdirectories of other versions unused for 14 days are removed the first time a run looks up a shader. The pipeline cache is one `vk-pipelines-*.bin` file per GPU and driver version. Deleting the directory is safe: it is rebuilt on the next run. +### Pipeline journal + +Besides the recompiled shaders, the shader cache directory holds `pipelines.journal`: every Vulkan graphics pipeline the pipeline store kept and every compute pipeline a run built, with the specialized shader modules they were built from. The next run of the title builds those pipelines on background threads while it loads, so a draw or dispatch the earlier run already made does not compile its pipeline when it is first reached. The journal belongs to the GPU and driver that wrote it, like the driver's pipeline cache file next to it: a run on another GPU or driver version starts it over, and the driver's own pipeline cache decides how fast each pipeline is rebuilt. Two runs sharing the directory take turns on `pipelines.journal.lock`; a run that cannot lock it records nothing. `APS5_NO_PIPELINE_JOURNAL=1` disables recording and replay; `APS5_PIPELINE_JOURNAL_THREADS` sets the replay threads (half the CPU threads by default; `0` records without replaying), `APS5_PIPELINE_JOURNAL_ENTRIES` how many pipelines are replayed (the first recorded, which a run reaches first) and `APS5_PIPELINE_STORE_ENTRIES` how many graphics pipelines stay resident (1024 each), and `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024) the file size at which recording stops. Deleting the file loses nothing but the head start. With `APS5_PROFILE_DRAW=1` a `[pipeline-journal]` line reports what was loaded, replayed and recorded. + ### Storing GPU results at each flip GPU results in storage images and render targets stay on the GPU until something reads their memory. A title that frees such an image after waiting for its GPU work and reuses the memory can have its new data overwritten when the results are stored later ([TechnicalDebt](../dev/TechnicalDebt.md)). `APS5_STORE_AT_FLIP=1` stores every pending result when a display buffer is flipped, at the cost of one write-back per pending image and frame. Any other value stops the title at its first flip. @@ -202,6 +208,12 @@ Limits are listed in the [libc](../dev/debt/libc.md) and [relinker](../dev/debt/ Set `APS5_CRASH_INDIRECT_MEMORY=1` before launching the executable to include up to 64 bytes from each pointer found in the register memory dump. The report labels each block with its register, pointer offset and target address. Inaccessible, incomplete and executable targets are skipped. This additional dump is disabled when the variable is unset or empty. +### GPU submission rings and stall dumps + +Each GPU queue keeps its last 16 command buffer submissions: when each was submitted, its serial, size and guest address, and a copy of its first 512 dwords after indirect buffers are inlined. `APS5_PM4_RING=` keeps the last `n` instead, from 0 to 4096; 0 keeps none, and any other value stops the title at its first submission. The rings are printed to stderr, one line per submission and one per decoded packet, when the Windows crash report runs: an unhandled exception, `abort()`, an invalid CRT parameter or the timed dump of `APS5_HANG_DUMP_SECS`. + +With `APS5_STALL_WATCHDOG_SECS=`, a queue that has waited `n` seconds without progress in a `WAIT_REG_MEM`, a `REWIND` or behind queue 0 prints the rings once for that wait, and the title keeps running. A `WAIT_REG_MEM` progresses while any other packet runs or completes, as for `APS5_GPU_WAIT_TIMEOUT_MS`. `n` is from 0 to 86400 and 0, the default, disables the watchdog; any other value stops the title at its first submission, as for `APS5_PM4_RING`. While the rings are printed for a queue held behind queue 0, new submissions and the other queues wait. + ## Exit codes `0`: conversion succeeded or help was displayed. `1`: invalid arguments. `2`: conversion failed; the error is printed to stderr. With `--autorun`, successful conversion returns the launched application's exit code. diff --git a/tools/check_compile.py b/tools/check_compile.py index 1ec01a433c..c15a8dc1d0 100644 --- a/tools/check_compile.py +++ b/tools/check_compile.py @@ -40,20 +40,22 @@ def entries(build): return json.loads(result.stdout) -def entry_key(entry): +def entry_key(entry, root): directory = Path(entry["directory"]).resolve() source = Path(entry["file"]) if not source.is_absolute(): source = directory / source source = source.resolve() - common = Path(os.path.commonpath([str(directory), str(source)])) - return os.path.relpath(source, common).replace(os.sep, "/") + try: + return source.relative_to(Path(root).resolve()).as_posix() + except ValueError: + return source.as_posix() -def index(build): +def index(build, root): commands = {} for entry in entries(build): - commands.setdefault(entry_key(entry), entry) + commands.setdefault(entry_key(entry, root), entry) return commands @@ -96,7 +98,7 @@ def syntax_only(entry, source): def check(root, build, files): - commands = index(build) + commands = index(build, root) checked = 0 failed = 0 for file in files: diff --git a/tools/check_conventions.py b/tools/check_conventions.py index 000a1de445..f558f4f409 100644 --- a/tools/check_conventions.py +++ b/tools/check_conventions.py @@ -20,7 +20,7 @@ RULES = { "system-dependency": ("error", "CONTRIBUTING.md#code", "Third-party code is a submodule under 3rdparty/ built from source, not found on the system"), "title-specific": ("error", "CONTRIBUTING.md#code", "Implement the general behaviour, not what one title needs; title-specific code belongs in its sce_module"), "extension": ("error", "CONTRIBUTING.md#code", "Avoid non-standard extensions where standard C++ is enough"), - "notes-file": ("error", "CONTRIBUTING.md#branches-and-pull-requests", "Notes, investigation and agent files go in the pull request, not in the repository"), + "notes-file": ("error", "CONTRIBUTING.md#branches-and-pull-requests", "Notes, investigation and agent files go in the pull request, not in the repository; only the root AGENTS.md is allowed"), "binary": ("error", "docs/dev/CONVENTIONS.md", "Only UTF-8 text files in the repository; images go in the gist comments"), "ci-gate": ("error", "CONTRIBUTING.md#build-and-test", "A job that runs for pull requests needs a file filter: paths on the pull_request trigger, or needs eligibility and an if on its outputs"), "doc-link": ("error", "CONTRIBUTING.md#documentation", "Relative link to a file that does not exist"), @@ -393,7 +393,7 @@ def library(path): def allowed_notes(path): p = PurePosixPath(path) if p.name.lower() in AGENT_FILES: - return False + return path == "AGENTS.md" if p.suffix.lower() not in NOTES: return None return path in ("README.md", "CONTRIBUTING.md") or p.parts[0] == ".github" or p.parts[:2] in (("docs", "dev"), ("docs", "user")) diff --git a/tools/compile_slot.sh b/tools/compile_slot.sh new file mode 100755 index 0000000000..2f01329de4 --- /dev/null +++ b/tools/compile_slot.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -u +dir=${ANYPS5_COMPILE_SLOT_DIR:?} +slots=${ANYPS5_COMPILE_SLOTS:?} +while :; do + for ((i = 0; i < slots; i++)); do + [ -e "$dir/slot-$i" ] || : >> "$dir/slot-$i" + exec {fd}<"$dir/slot-$i" + if flock -n "$fd"; then + exec "$@" + fi + exec {fd}>&- + done + sleep 0.1 +done diff --git a/tools/configure_seed.py b/tools/configure_seed.py new file mode 100644 index 0000000000..31a3eead18 --- /dev/null +++ b/tools/configure_seed.py @@ -0,0 +1,40 @@ +import argparse +import re +import sys +from pathlib import Path + +CHECK = re.compile(r"^//(Have (include|includes|function|symbol|library) |Test |Result of (TRY_COMPILE|TRY_RUN|TEST_BIG_ENDIAN|check_))") +ENTRY = re.compile(r"^([^#/:][^:]*):([A-Z]+)=(.*)$") + + +def checks(cache): + found, doc = {}, "" + for line in Path(cache).read_text(encoding="utf-8").splitlines(): + if line.startswith("//"): + doc = line + continue + match = ENTRY.match(line) + if match and match.group(2) == "INTERNAL" and CHECK.match(doc): + found[match.group(1)] = (match.group(3), doc[2:]) + doc = "" + return found + + +def quote(value): + return '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("$", "\\$") + '"' + + +def main(): + parser = argparse.ArgumentParser(description="Write the configure check results of a CMake cache as an initial cache file") + parser.add_argument("cache", help="CMakeCache.txt of a configured tree") + parser.add_argument("seed", help="file to pass to cmake -C") + args = parser.parse_args() + found = checks(args.cache) + Path(args.seed).write_text("".join(f"set({name} {quote(value)} CACHE INTERNAL {quote(doc)})\n" + for name, (value, doc) in sorted(found.items())), encoding="utf-8") + print(f"{len(found)} check results") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/select_tests.py b/tools/select_tests.py new file mode 100644 index 0000000000..ef0e2bddef --- /dev/null +++ b/tools/select_tests.py @@ -0,0 +1,240 @@ +import argparse +import collections +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +FULL = re.compile(r"^CMakeLists\.txt$|^core/CMakeLists\.txt$|^core/libs/CMakeLists\.txt$|^\.github/workflows/build\.yml$" + r"|^3rdparty/|^cmake/|^\.gitmodules$|^tools/select_tests\.py$") +LOCAL = re.compile(r"(^|/)CMakeLists\.txt$|\.cmake$") +BATCH = 20000 + + +class Build: + def __init__(self, path): + cache = {} + for line in (Path(path) / "CMakeCache.txt").read_text(encoding="utf-8").splitlines(): + name, _, value = line.partition("=") + cache[name.partition(":")[0]] = value + self.path = os.path.normpath(cache["CMAKE_CACHEFILE_DIR"]) + self.source = os.path.normpath(cache["CMAKE_HOME_DIRECTORY"]) + self.program = cache.get("CMAKE_MAKE_PROGRAM") or "ninja" + self.encoding = "utf-8" + if sys.platform == "win32": + codepage = self.run("-t", "wincodepage").strip() + self.encoding = {"Build file encoding: UTF-8": "utf-8", "Build file encoding: ANSI": "mbcs"}[codepage] + + def run(self, *args): + return subprocess.run([self.program, "-C", self.path, *args], capture_output=True, text=True, + encoding=self.encoding, check=True).stdout + + def absolute(self, path): + return os.path.normcase(os.path.normpath(path if os.path.isabs(path) else os.path.join(self.path, path))) + + def repo(self, path): + absolute = self.absolute(path) + build, source = os.path.normcase(self.path), os.path.normcase(self.source) + if absolute.startswith(build + os.sep) or not absolute.startswith(source + os.sep): + return None + return os.path.relpath(absolute, source).replace(os.sep, "/") + + def query(self, targets): + found = {} + for batch in chunks(targets): + current, section = None, None + for line in self.run("-t", "query", *batch).splitlines(): + if not line.startswith(" "): + current = line.rstrip(":") + found[current] = {"inputs": [], "phony": False} + elif line.startswith(" input: "): + section = "inputs" + found[current]["phony"] = line.strip() == "input: phony" + elif line.startswith(" outputs:"): + section = None + elif section and line.startswith(" "): + found[current]["inputs"].append(line.strip().removeprefix("|| ").removeprefix("| ")) + return found + + def walk(self, roots): + nodes, todo = {}, list(roots) + while todo: + batch = sorted({t for t in todo if t not in nodes}) + if not batch: + break + nodes.update(self.query(batch)) + todo = [i for t in batch for i in nodes.get(t, {"inputs": []})["inputs"]] + return nodes + + def leaves(self): + nodes = self.walk(["all"]) + leaves, todo, seen = set(), ["all"], set() + while todo: + name = todo.pop() + if name in seen or name not in nodes: + continue + seen.add(name) + if nodes[name]["phony"]: + todo.extend(nodes[name]["inputs"]) + else: + leaves.add(name) + return sorted(leaves) + + def tests(self): + listing = json.loads(subprocess.run(["ctest", "--test-dir", self.path, "--show-only=json-v1"], + capture_output=True, text=True, check=True).stdout) + return listing, listing["tests"] + + def output(self, path): + absolute = self.absolute(path) + if not absolute.startswith(os.path.normcase(self.path) + os.sep): + return None + return os.path.relpath(absolute, os.path.normcase(self.path)).replace(os.sep, "/") + + def target(self, test): + command = test.get("command") or [] + return self.output(command[0]) if command else None + + +def chunks(names): + batch, size = [], 0 + for name in names: + if batch and size + len(name) + 1 > BATCH: + yield batch + batch, size = [], 0 + batch.append(name) + size += len(name) + 1 + if batch: + yield batch + + +def dump(build, out): + deps = collections.defaultdict(set) + current = None + for line in build.run("-t", "deps").splitlines(): + if not line.startswith(" ") and ": #deps" in line: + current = build.absolute(line.split(": #deps")[0]) + elif line.strip() and current: + path = build.repo(line.strip()) + if path: + deps[current].add(path) + listing, listed = build.tests() + graph = listing["backtraceGraph"] + targets = {build.target(t) for t in listed} - {None} + nodes = build.walk(sorted(targets)) + closures = {} + for target in sorted(targets): + files, seen, todo = set(), set(), [target] + while todo: + name = todo.pop() + if name in seen: + continue + seen.add(name) + path = build.repo(name) + if path: + files.add(path) + files |= deps.get(build.absolute(name), set()) + todo.extend(nodes.get(name, {"inputs": []})["inputs"]) + closures[target] = sorted(files) + tests = {} + for test in listed: + registered, node = set(), test.get("backtrace") + while node is not None: + registered.add(build.repo(graph["files"][graph["nodes"][node]["file"]])) + node = graph["nodes"][node].get("parent") + props = {p["name"]: p["value"] for p in test.get("properties", [])} + arguments = sorted({build.repo(a) for a in (test.get("command") or [])[1:] if os.path.isabs(a)} - {None}) + tests[test["name"]] = {"registered": sorted(registered - {None}), "arguments": arguments, + "requires": props.get("FIXTURES_REQUIRED", []), "sets": props.get("FIXTURES_SETUP", []), + "depends": props.get("DEPENDS", []), "target": build.target(test)} + with open(out, "w", encoding="utf-8") as file: + json.dump({"tests": tests, "closures": closures}, file) + print(f"{len(tests)} tests, {len(closures)} test executables, {len(deps)} objects with header deps") + + +def select(graph, changed): + full = [f for f in changed if FULL.search(f)] + if full: + return None, full + tests, closures = graph["tests"], {t: set(f) for t, f in graph["closures"].items()} + known = set().union(*closures.values(), *(t["arguments"] for t in tests.values())) + files = set(changed) + dirs = tuple(f.rpartition("/")[0] + "/" for f in changed if "/" in f and (LOCAL.search(f) or f not in known)) + picked = set() + for name, test in tests.items(): + scope = closures.get(test["target"], set()) | set(test["arguments"]) + if (test["target"] is None or files & set(test["registered"]) or files & scope + or any(f.startswith(dirs) for f in scope | set(test["registered"]))): + picked.add(name) + setups = collections.defaultdict(set) + for name, test in tests.items(): + for fixture in test["sets"]: + setups[fixture].add(name) + todo = list(picked) + while todo: + test = tests[todo.pop()] + for extra in set(test["depends"]).union(*(setups[f] for f in test["requires"])): + if extra in tests and extra not in picked: + picked.add(extra) + todo.append(extra) + return picked, [] + + +def plan(build, graph, picked): + _, listed = build.tests() + run = [i for i, t in enumerate(listed, 1) if t["name"] in picked or t["name"] not in graph["tests"]] + names = {listed[i - 1]["name"] for i in run} + keep = {build.output(a) for t in listed if t["name"] in names for a in t.get("command") or [] if os.path.isabs(a)} + skip = {build.target(t) for t in listed if t["name"] not in names} - keep - {None} + return run, [t for t in build.leaves() if t not in skip], len(listed) + + +def escape(name): + return name.replace("$", "$$").replace(":", "$:").replace(" ", "$ ") + + +def main(): + parser = argparse.ArgumentParser(description="Select the tests whose build inputs a change touches") + sub = parser.add_subparsers(dest="command", required=True) + d = sub.add_parser("dump", help="write the test graph of a built tree") + d.add_argument("build") + d.add_argument("graph") + s = sub.add_parser("select", help="write a ninja file and a ctest index list for a configured tree") + s.add_argument("build") + s.add_argument("graph") + s.add_argument("changed", help="file with one changed path per line") + c = sub.add_parser("ctest", help="run ctest on a tree, limited to the selected tests if select wrote any") + c.add_argument("build") + c.add_argument("arguments", nargs=argparse.REMAINDER, help="ctest arguments") + args = parser.parse_args() + if args.command == "ctest": + selected = Path(args.build, "select.tests") + extra = ["-I", selected.read_text(encoding="utf-8").strip()] if selected.exists() else [] + return subprocess.run(["ctest", "--test-dir", args.build, *args.arguments, *extra]).returncode + build = Build(args.build) + if args.command == "dump": + dump(build, args.graph) + return 0 + for name in ("select.ninja", "select.tests"): + Path(build.path, name).unlink(missing_ok=True) + if not os.path.exists(args.graph): + print("full run: no test graph") + return 0 + graph = json.loads(Path(args.graph).read_text(encoding="utf-8")) + changed = [line.strip() for line in Path(args.changed).read_text(encoding="utf-8").splitlines() if line.strip()] + picked, full = select(graph, changed) + if picked is None: + print(f"full run: {', '.join(full[:5])}") + return 0 + run, targets, total = plan(build, graph, picked) + Path(build.path, "select.ninja").write_text("include build.ninja\nbuild selected: phony " + + " ".join(escape(t) for t in targets) + "\n", encoding=build.encoding) + Path(build.path, "select.tests").write_text("0,0,0," + ",".join(map(str, run)) + "\n", encoding="utf-8") + print(f"selected {len(run)}/{total} tests, {len(targets)} build targets") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/test_build_eligibility.py b/tools/tests/test_build_eligibility.py index b39ec9fabe..56e41b71c6 100644 --- a/tools/tests/test_build_eligibility.py +++ b/tools/tests/test_build_eligibility.py @@ -94,7 +94,7 @@ class RelinkerEligibilityTests(unittest.TestCase): encoding="utf-8") environment = dict(os.environ, PR_NUMBER="7", GITHUB_TRIGGERING_ACTOR="maintainer") result = subprocess.run([self.node, "harness.js", "scenario.json", "gate.js"], cwd=self.root, - capture_output=True, text=True, env=environment, timeout=20) + capture_output=True, text=True, env=environment, timeout=90) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) return json.loads(result.stdout) diff --git a/tools/tests/test_check_compile.py b/tools/tests/test_check_compile.py index b691d60d45..dd27d12654 100644 --- a/tools/tests/test_check_compile.py +++ b/tools/tests/test_check_compile.py @@ -124,6 +124,24 @@ class CheckCompileTests(unittest.TestCase): self.assertEqual(result.returncode, 0) self.assertIn("1 translation units compile", result.stdout) + def test_a_build_directory_beside_the_checkout_is_matched(self): + repo = self.root / "AnyPS5" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + sibling = self.root / "sibling-build" + sibling.mkdir() + source = repo / "src" / "unit.cpp" + source.parent.mkdir() + source.write_text("int x;\n", encoding="utf-8") + script = self.compiler("import sys\nsys.exit(0)\n") + entry = {"directory": str(sibling), "command": f'"{sys.executable}" "{script}" "{source}"', + "file": str(source)} + (sibling / "compile_commands.json").write_text(json.dumps([entry]), encoding="utf-8") + result = subprocess.run([sys.executable, "-I", str(TOOL), "--build", str(sibling), "src/unit.cpp"], + cwd=repo, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("1 translation units compile", result.stdout) + def test_a_changed_file_with_a_non_ascii_name_is_checked(self): self.failing("caf\u00e9.cpp") subprocess.run(["git", "-C", str(self.root), "add", "caf\u00e9.cpp"], check=True) diff --git a/tools/tests/test_check_conventions_agent_files.py b/tools/tests/test_check_conventions_agent_files.py index 87704e1226..fcef15671a 100644 --- a/tools/tests/test_check_conventions_agent_files.py +++ b/tools/tests/test_check_conventions_agent_files.py @@ -10,7 +10,7 @@ from check_conventions import allowed_notes class AgentFileTests(unittest.TestCase): def test_agent_files_are_rejected_in_any_case(self): for name in ( - "CLAUDE.md", "claude.md", "Claude.md", "AGENTS.md", "Agents.md", "GEMINI.md", + "CLAUDE.md", "claude.md", "Claude.md", "GEMINI.md", "Copilot-Instructions.md", ".CursorRules", ".WindsurfRules", "windsurf.md", "ChatGPT.md", ".ChatGPTRules", ".aider.conf.yml", ): @@ -18,6 +18,12 @@ class AgentFileTests(unittest.TestCase): self.assertIs(allowed_notes(f"docs/dev/{name}"), False) self.assertIs(allowed_notes(name), False) + def test_only_the_root_agents_file_is_allowed(self): + self.assertIs(allowed_notes("AGENTS.md"), True) + for path in ("agents.md", "Agents.md", "docs/dev/AGENTS.md", "core/AGENTS.md", "tools/agents.md"): + with self.subTest(path=path): + self.assertIs(allowed_notes(path), False) + def test_ordinary_docs_are_still_allowed(self): self.assertTrue(allowed_notes("docs/dev/Claudette.md")) self.assertTrue(allowed_notes("README.md")) diff --git a/tools/tests/test_configure_seed.py b/tools/tests/test_configure_seed.py new file mode 100644 index 0000000000..375ea8c68c --- /dev/null +++ b/tools/tests/test_configure_seed.py @@ -0,0 +1,85 @@ +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +TOOL = Path(__file__).resolve().parents[1] / "configure_seed.py" +sys.path.insert(0, str(TOOL.parent)) + +import configure_seed + +PROJECT = """cmake_minimum_required(VERSION 3.20) +project(mini C) +include(CheckIncludeFile) +include(CheckSymbolExists) +include(CheckCSourceCompiles) +check_include_file(stdio.h MINI_HAVE_STDIO_H) +check_include_file(mini_missing_header.h MINI_HAVE_MISSING_H) +check_symbol_exists(printf stdio.h MINI_HAVE_PRINTF) +check_c_source_compiles("int main(void) { return 0; }" MINI_COMPILES) +check_c_source_compiles("int main(void) { return undefined_name; }" MINI_FAILS) +set(MINI_OPTION "value" CACHE STRING "An option, not a check result") +""" + + +@unittest.skipUnless(shutil.which("ninja") and (shutil.which("gcc") or shutil.which("cc")), "needs ninja and a C compiler") +class ConfigureSeedTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.directory = tempfile.TemporaryDirectory() + cls.root = Path(cls.directory.name) + (cls.root / "source").mkdir() + (cls.root / "source" / "CMakeLists.txt").write_text(PROJECT, encoding="utf-8") + cls.compiler = shutil.which("gcc") or shutil.which("cc") + cls.fresh = cls.configure("fresh") + + @classmethod + def tearDownClass(cls): + cls.directory.cleanup() + + @classmethod + def configure(cls, name, *extra): + return subprocess.run(["cmake", *extra, "-S", str(cls.root / "source"), "-B", str(cls.root / name), "-G", "Ninja", + f"-DCMAKE_C_COMPILER={cls.compiler}"], check=True, capture_output=True, text=True).stdout + + def seed(self, cache, name): + seed = self.root / name + subprocess.run([sys.executable, str(TOOL), str(cache), str(seed)], check=True, capture_output=True) + return seed + + def test_seeded_configure_runs_no_check_and_gets_the_same_results(self): + self.assertRegex(self.fresh, r"Looking for|Performing Test") + output = self.configure("seeded", "-C", str(self.seed(self.root / "fresh" / "CMakeCache.txt", "seed.cmake"))) + self.assertNotRegex(output, r"Looking for|Performing Test") + fresh = configure_seed.checks(self.root / "fresh" / "CMakeCache.txt") + self.assertEqual(configure_seed.checks(self.root / "seeded" / "CMakeCache.txt"), fresh) + self.assertEqual(fresh["MINI_HAVE_STDIO_H"][0], "1") + self.assertEqual(fresh["MINI_FAILS"][0], "") + + def test_only_check_results_are_seeded(self): + found = configure_seed.checks(self.root / "fresh" / "CMakeCache.txt") + self.assertEqual({n for n in found if n.startswith("MINI_")}, + {"MINI_HAVE_STDIO_H", "MINI_HAVE_MISSING_H", "MINI_HAVE_PRINTF", "MINI_COMPILES", "MINI_FAILS"}) + self.assertNotIn("CMAKE_C_COMPILER", found) + + def test_values_with_cmake_syntax_round_trip(self): + cache = self.root / "odd" / "CMakeCache.txt" + cache.parent.mkdir() + value = r'a;b "c" \d ${e} $f' + cache.write_text(f"//Test MINI_ODD\nMINI_ODD:INTERNAL={value}\n", encoding="utf-8") + self.configure("odd-seeded", "-C", str(self.seed(cache, "odd.cmake"))) + seeded = (self.root / "odd-seeded" / "CMakeCache.txt").read_text(encoding="utf-8") + self.assertIn(f"MINI_ODD:INTERNAL={value}\n", seeded) + self.assertEqual(configure_seed.checks(self.root / "odd-seeded" / "CMakeCache.txt")["MINI_ODD"][0], value) + + def test_cache_without_checks_writes_an_empty_seed(self): + cache = self.root / "empty" / "CMakeCache.txt" + cache.parent.mkdir() + cache.write_text("//Path to a program.\nCMAKE_AR:FILEPATH=/usr/bin/ar\n", encoding="utf-8") + self.assertEqual(self.seed(cache, "empty.cmake").read_text(encoding="utf-8"), "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_select_tests.py b/tools/tests/test_select_tests.py new file mode 100644 index 0000000000..384e77742a --- /dev/null +++ b/tools/tests/test_select_tests.py @@ -0,0 +1,114 @@ +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +TOOL = Path(__file__).resolve().parents[1] / "select_tests.py" + +PROJECT = { + "CMakeLists.txt": "cmake_minimum_required(VERSION 3.20)\nproject(mini C)\nenable_testing()\n" + "add_subdirectory(alpha)\nadd_subdirectory(beta)\n" + "add_test(NAME script COMMAND ${CMAKE_COMMAND} -E true)\n", + "alpha/CMakeLists.txt": "add_executable(alpha_tests main.c)\nadd_test(NAME alpha COMMAND alpha_tests)\n" + "add_test(NAME alpha_data COMMAND alpha_tests ${CMAKE_SOURCE_DIR}/data/input.txt)\n", + "alpha/main.c": "#include \"alpha.h\"\nint main(void) { return ALPHA; }\n", + "alpha/alpha.h": "#define ALPHA 0\n", + "beta/CMakeLists.txt": "add_executable(beta_tests main.c)\nadd_executable(setup_tests setup.c)\n" + "add_test(NAME beta COMMAND beta_tests)\nadd_test(NAME beta_setup COMMAND setup_tests)\n" + "set_tests_properties(beta_setup PROPERTIES FIXTURES_SETUP beta_fixture)\n" + "set_tests_properties(beta PROPERTIES FIXTURES_REQUIRED beta_fixture)\n", + "beta/main.c": "int main(void) { return 0; }\n", + "beta/setup.c": "int main(void) { return 0; }\n", + "data/input.txt": "input\n", +} + + +@unittest.skipUnless(shutil.which("ninja") and (shutil.which("gcc") or shutil.which("cc")), "needs ninja and a C compiler") +class SelectTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.directory = tempfile.TemporaryDirectory() + (Path(cls.directory.name) / "real").mkdir() + (Path(cls.directory.name) / "link").symlink_to("real", target_is_directory=True) + root = Path(cls.directory.name) / "link" + cls.source, cls.build, cls.graph = root / "source", root / "build", root / "graph.json" + for name, content in PROJECT.items(): + (cls.source / name).parent.mkdir(parents=True, exist_ok=True) + (cls.source / name).write_text(content, encoding="utf-8") + compiler = shutil.which("gcc") or shutil.which("cc") + subprocess.run(["cmake", "-S", str(cls.source), "-B", str(cls.build), "-G", "Ninja", f"-DCMAKE_C_COMPILER={compiler}"], + check=True, capture_output=True) + subprocess.run(["cmake", "--build", str(cls.build)], check=True, capture_output=True) + cls.tool("dump", cls.build, cls.graph) + + @classmethod + def tearDownClass(cls): + cls.directory.cleanup() + + @staticmethod + def tool(*args): + return subprocess.run([sys.executable, str(TOOL), *map(str, args)], check=True, capture_output=True, text=True).stdout + + def select(self, *changed, graph=None): + listed = self.build / "changed.txt" + listed.write_text("".join(f"{c}\n" for c in changed), encoding="utf-8") + output = self.tool("select", self.build, graph or self.graph, listed) + tests = self.build / "select.tests" + if not tests.exists(): + self.assertFalse((self.build / "select.ninja").exists()) + return output, None + shown = subprocess.run(["ctest", "--test-dir", str(self.build), "-N", "-I", tests.read_text(encoding="utf-8").strip()], + check=True, capture_output=True, text=True).stdout + return output, set(re.findall(r"Test +#\d+: (\S+)", shown)) + + def test_header_selects_the_tests_of_its_executable(self): + self.assertEqual(self.select("alpha/alpha.h")[1], {"alpha", "alpha_data", "script"}) + + def test_argument_file_selects_only_the_test_that_reads_it(self): + self.assertEqual(self.select("data/input.txt")[1], {"alpha_data", "script"}) + + def test_fixture_setup_runs_with_its_test(self): + self.assertEqual(self.select("beta/main.c")[1], {"beta", "beta_setup", "script"}) + self.assertEqual(self.select("beta/setup.c")[1], {"beta_setup", "script"}) + + def test_cmake_file_selects_the_tests_it_registers(self): + self.assertEqual(self.select("beta/CMakeLists.txt")[1], {"beta", "beta_setup", "script"}) + + def test_file_outside_every_build_selects_its_directory(self): + self.assertEqual(self.select("alpha/notes.h")[1], {"alpha", "alpha_data", "script"}) + self.assertEqual(self.select("README.md")[1], {"script"}) + + def test_global_files_run_everything(self): + for changed in ("CMakeLists.txt", "cmake/Toolchain.cmake", ".github/workflows/build.yml", "3rdparty/x/y.c", + "tools/select_tests.py"): + with self.subTest(changed=changed): + output, selected = self.select("alpha/alpha.h", changed) + self.assertIsNone(selected) + self.assertIn(changed, output) + + def test_missing_graph_runs_everything(self): + self.assertIsNone(self.select("alpha/alpha.h", graph=self.build / "missing.json")[1]) + + def test_test_missing_from_the_graph_runs(self): + graph = json.loads(self.graph.read_text(encoding="utf-8")) + del graph["tests"]["beta"] + older = self.build / "older.json" + older.write_text(json.dumps(graph), encoding="utf-8") + self.assertEqual(self.select("alpha/alpha.h", graph=older)[1], {"alpha", "alpha_data", "beta", "beta_setup", "script"}) + + def test_selected_target_skips_unselected_executables(self): + self.select("alpha/main.c") + subprocess.run(["ninja", "-C", str(self.build), "-t", "clean"], check=True, capture_output=True) + subprocess.run(["ninja", "-C", str(self.build), "-f", "select.ninja", "selected"], check=True, capture_output=True) + built = {p.stem for p in self.build.rglob("*_tests*") if p.is_file() and os.access(p, os.X_OK)} + subprocess.run(["cmake", "--build", str(self.build)], check=True, capture_output=True) + self.assertEqual(built, {"alpha_tests"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_unwrap_self.py b/tools/tests/test_unwrap_self.py index 44e1897ccb..57fee3b981 100644 --- a/tools/tests/test_unwrap_self.py +++ b/tools/tests/test_unwrap_self.py @@ -13,6 +13,7 @@ ELF_HEADER_OFFSET = 0x1A0 PROGRAM_TABLE_OFFSET = 0x40 PT_LOAD = 1 PT_DYNAMIC = 2 +PT_TLS = 7 FIRST_REGION = 0x400 HEADER = b"\x7fELF\x02\x01\x01\x09" + b"\x00" * 56 @@ -117,6 +118,17 @@ class UnwrapTest(unittest.TestCase): self.assertEqual(dynamic[2], read_only[2] + (0x1040 - 0x1000)) self.assertEqual(dynamic[2] % 8, 0x1040 % 8) + def test_an_empty_segment_at_the_end_of_a_load_segment_is_placed_after_it(self): + segments = default_segments() + [(PT_TLS, 4, 0x1580, 0x1080, 0x0, 8)] + container = build_container(segments, default_records(), default_payloads()) + result, output = self.convert(container) + self.assertEqual(result.returncode, 0, result.stderr) + data, headers = read_program_headers(output) + read_only = [header for header in headers if header[0] == PT_LOAD][1] + tls = [header for header in headers if header[0] == PT_TLS][0] + self.assertEqual(tls[2], read_only[2] + 0x80) + self.assertEqual(tls[5], 0) + def test_declared_offsets_are_used_when_the_container_has_no_records(self): segments = [(PT_LOAD, 1, 0x800, 0x0, 0x40, 0x1000), (PT_LOAD, 4, 0x900, 0x1000, 0x40, 0x1000)] payloads = {0x800: pattern(3, 0x40), 0x900: pattern(5, 0x40)} diff --git a/tools/unwrap_self.py b/tools/unwrap_self.py index 05af4b508a..c0715feba0 100644 --- a/tools/unwrap_self.py +++ b/tools/unwrap_self.py @@ -127,10 +127,11 @@ def place_segments(data, headers, records): header["source"] = segment break - def load_for(vaddr): + def load_for(vaddr, size): found = None for header in loads: - if header["memory"] and header["vaddr"] <= vaddr < header["vaddr"] + header["memory"]: + end = header["vaddr"] + header["memory"] + if header["memory"] and header["vaddr"] <= vaddr and (vaddr < end or (size == 0 and vaddr == end)): found = header return found @@ -139,7 +140,7 @@ def place_segments(data, headers, records): if header["type"] == PT_LOAD: continue if header["vaddr"] != 0: - host = load_for(header["vaddr"]) + host = load_for(header["vaddr"], header["size"]) if host is None: raise Refused("segment %d vaddr %#x is in no PT_LOAD" % (header["index"], header["vaddr"])) header["host"] = host