fix(relinker): lower ANDN in Rosetta conversion

core/relinker/CMakeLists.txt

@@ -51,6 +51,7 @@ add_library(relinker_core STATIC
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp
@@ -83,6 +84,11 @@ target_compile_features(relinker_core PUBLIC cxx_std_20)
 set_target_properties(relinker_core PROPERTIES CXX_EXTENSIONS OFF)
 
 if(BUILD_TESTING)
+    add_test_executable(andn_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/AndnLoweringTests.cpp)
+    target_link_libraries(andn_lowering_tests PRIVATE relinker_core)
+    add_test(NAME andn_lowering COMMAND andn_lowering_tests)
+    set_tests_properties(andn_lowering PROPERTIES TIMEOUT 20)
+
     add_test_executable(relinker_conversion_tests ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/ConversionTests.cpp)
     target_link_libraries(relinker_conversion_tests PRIVATE relinker_core)
     add_test(NAME relinker_conversion COMMAND relinker_conversion_tests)
@@ -169,6 +175,7 @@ if(WIN32)
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp
     )
     target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows)
     if(BUILD_TESTING)
@@ -353,6 +360,7 @@ if(BUILD_TESTING)
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp

core/relinker/codegen/include/codegen/x86/AndnLowering.hpp

@@ -0,0 +1,22 @@
+#ifndef CODEGEN_X86_ANDNLOWERING_HPP
+#define CODEGEN_X86_ANDNLOWERING_HPP
+
+#include <codegen/x86/StubBodyBuilder.hpp>
+#include <optional>
+
+namespace Codegen {
+
+struct AndnOperands {
+    std::uint8_t Destination;
+    std::uint8_t InvertedSource;
+    std::uint8_t Source;
+    bool Wide;
+    std::optional<MemoryOperand> Memory;
+};
+
+[[nodiscard]] std::optional<AndnOperands> DecodeAndn(const std::uint8_t* data, std::size_t length);
+void EmitAndn(StubBodyBuilder& body, const AndnOperands& operands);
+
+}
+
+#endif

core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp

@@ -15,6 +15,7 @@
 #include <codegen/x86/ScalarCompareOperands.hpp>
 #include <codegen/x86/WideStoreLowering.hpp>
 #include <codegen/x86/WideStoreOperands.hpp>
+#include <codegen/x86/AndnLowering.hpp>
 #include <codegen/x86/X64InstructionDecoder.hpp>
 #include <codegen/x86/X64OpcodeConstants.hpp>
 #include <codegen/CodegenException.hpp>
@@ -217,6 +218,8 @@ Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchClzero(const DecodedInstructio
 std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::_matchRosetta(const DecodedInstruction& instr, std::span<const std::uint8_t> trailing) const {
     if (_target != Amd64OnlyTarget::Rosetta)
         return std::nullopt;
+    if (const auto andn = DecodeAndn(instr.Data, instr.Length))
+        return _trampoline("ANDN", instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitAndn(body, *andn); }));
     if (instr.IsRdseed()) {
         std::vector<std::uint8_t> replacement(instr.Data, instr.Data + instr.Length);
         replacement[instr.OpcodeOffset() + 2] &= static_cast<std::uint8_t>(~(1u << X64OpcodeConstants::ModRmRegShift));
@@ -261,6 +264,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c
         body.Raw(_rdpidReplacement(instr));
         return kRdpid.Name;
     }
+    if (const auto andn = _target == Amd64OnlyTarget::Rosetta ? DecodeAndn(instr.Data, instr.Length) : std::nullopt) {
+        EmitAndn(body, *andn);
+        return "ANDN";
+    }
     if (const auto wide = _target == Amd64OnlyTarget::Rosetta ? DecodeVexWideStore(instr.Data, instr.Length) : std::nullopt) {
         _wideStoreLowering.EmitOutOfLine(body, *wide);
         return kWideStore.Name;

core/relinker/codegen/src/x86/AndnLowering.cpp

@@ -0,0 +1,138 @@
+#include <codegen/x86/AndnLowering.hpp>
+#include <codegen/CodegenException.hpp>
+#include <algorithm>
+#include <array>
+#include <limits>
+
+namespace Codegen {
+
+namespace {
+
+constexpr std::int32_t kFrameSize = 144;
+constexpr std::uint8_t kStack = 4;
+
+bool _prefix(std::uint8_t value) {
+    return value == 0x2E || value == 0x36 || value == 0x3E || value == 0x26 || value == 0x64 || value == 0x65 || value == 0x67;
+}
+
+void _append32(std::vector<std::uint8_t>& bytes, std::int32_t value) {
+    for (unsigned shift = 0; shift < 32; shift += 8)
+        bytes.push_back(static_cast<std::uint8_t>(static_cast<std::uint32_t>(value) >> shift));
+}
+
+std::uint8_t _rex(bool wide, std::uint8_t reg, std::uint8_t rm) {
+    return static_cast<std::uint8_t>(0x40 | (wide ? 8 : 0) | ((reg & 8) >> 1) | ((rm & 8) >> 3));
+}
+
+void _register(StubBodyBuilder& body, std::uint8_t opcode, bool wide, std::uint8_t reg, std::uint8_t rm) {
+    const std::array bytes{_rex(wide, reg, rm), opcode, static_cast<std::uint8_t>(0xC0 | ((reg & 7) << 3) | (rm & 7))};
+    body.Raw(bytes);
+}
+
+void _stack(StubBodyBuilder& body, std::uint8_t opcode, bool wide, std::uint8_t reg, std::int32_t displacement) {
+    std::vector<std::uint8_t> bytes{_rex(wide, reg, kStack), opcode, static_cast<std::uint8_t>(0x84 | ((reg & 7) << 3)), 0x24};
+    _append32(bytes, displacement);
+    body.Raw(bytes);
+}
+
+std::uint8_t _scratch(const AndnOperands& operands) {
+    std::array<bool, 16> used{};
+    used[kStack] = used[operands.Destination] = used[operands.InvertedSource] = true;
+    if (!operands.Memory) {
+        used[operands.Source] = true;
+    } else {
+        const auto& memory = *operands.Memory;
+        if (memory.Rm == 4) {
+            const auto base = static_cast<std::uint8_t>((memory.Sib & 7) | ((memory.RexIndexBase & 1) << 3));
+            const auto index = static_cast<std::uint8_t>(((memory.Sib >> 3) & 7) | ((memory.RexIndexBase & 2) << 2));
+            if (memory.Mod != 0 || (memory.Sib & 7) != 5) used[base] = true;
+            if (index != kStack) used[index] = true;
+        } else if (!memory.RipRelative) {
+            used[memory.Rm | ((memory.RexIndexBase & 1) << 3)] = true;
+        }
+    }
+    for (const auto reg : {11, 10, 9, 8, 0, 1, 2, 3, 5, 6, 7, 12, 13, 14, 15})
+        if (!used[reg]) return static_cast<std::uint8_t>(reg);
+    throw CodegenException("ANDN has no scratch register");
+}
+
+void _memory(StubBodyBuilder& body, const AndnOperands& operands, std::uint8_t scratch) {
+    const auto& memory = *operands.Memory;
+    auto displacement = static_cast<std::int64_t>(memory.Displacement);
+    if (memory.StackBase) displacement += kFrameSize;
+    const bool address32 = std::find(memory.Prefixes.begin(), memory.Prefixes.end(), 0x67) != memory.Prefixes.end();
+    if (!address32 && displacement > std::numeric_limits<std::int32_t>::max())
+        throw CodegenException("ANDN stack displacement does not fit after spilling");
+    const auto mod = memory.StackBase ? 2 : memory.Mod;
+    auto bytes = memory.Prefixes;
+    bytes.push_back(static_cast<std::uint8_t>(_rex(operands.Wide, scratch, 0) | memory.RexIndexBase));
+    bytes.push_back(0x23);
+    bytes.push_back(static_cast<std::uint8_t>((mod << 6) | ((scratch & 7) << 3) | memory.Rm));
+    if (memory.Rm == 4) bytes.push_back(memory.Sib);
+    const auto offset = bytes.size();
+    if (mod == 1) bytes.push_back(static_cast<std::uint8_t>(displacement));
+    if (mod == 2 || (mod == 0 && (memory.RipRelative || (memory.Rm == 4 && (memory.Sib & 7) == 5))))
+        _append32(bytes, static_cast<std::int32_t>(displacement));
+    if (memory.RipRelative)
+        body.RipRelative(bytes, offset, static_cast<std::int64_t>(memory.InstructionLength) + memory.Displacement);
+    else
+        body.Raw(bytes);
+}
+
+}
+
+std::optional<AndnOperands> DecodeAndn(const std::uint8_t* data, const std::size_t length) {
+    std::size_t vex = 0;
+    while (vex < length && _prefix(data[vex])) ++vex;
+    if (vex + 5 > length || data[vex] != 0xC4 || (data[vex + 1] & 0x1F) != 2 || data[vex + 3] != 0xF2)
+        return std::nullopt;
+    if ((data[vex + 2] & 7) != 0)
+        throw CodegenException("ANDN requires VEX.L=0 and VEX.pp=0");
+    const auto extensions = static_cast<std::uint8_t>(~data[vex + 1]);
+    const auto modrm = data[vex + 4];
+    AndnOperands operands{
+        static_cast<std::uint8_t>(((modrm >> 3) & 7) | ((extensions & 0x80) >> 4)),
+        static_cast<std::uint8_t>((~data[vex + 2] >> 3) & 15),
+        static_cast<std::uint8_t>((modrm & 7) | ((extensions & 0x20) >> 2)),
+        (data[vex + 2] & 0x80) != 0, std::nullopt};
+    if ((modrm >> 6) == 3) {
+        if (vex + 5 != length) throw CodegenException("ANDN register operand has trailing bytes");
+    } else {
+        const auto rex = static_cast<std::uint8_t>(((extensions & 0x20) >> 5) | ((extensions & 0x40) >> 5));
+        operands.Memory = DecodeMemoryOperand(data, length, vex + 4, rex, {data, data + vex});
+        if (vex + 4 + operands.Memory->EncodedSize != length)
+            throw CodegenException("ANDN memory operand has trailing bytes");
+    }
+    return operands;
+}
+
+void EmitAndn(StubBodyBuilder& body, const AndnOperands& operands) {
+    const auto scratch = _scratch(operands);
+    _stack(body, 0x8D, true, kStack, -kFrameSize);
+    _stack(body, 0x89, true, scratch, 0);
+    if (!operands.Memory && operands.Source == kStack) {
+        _stack(body, 0x8D, true, scratch, kFrameSize);
+        _stack(body, 0x89, true, scratch, 8);
+    }
+    if (operands.InvertedSource == kStack)
+        _stack(body, 0x8D, true, scratch, kFrameSize);
+    else
+        _register(body, 0x8B, operands.Wide, scratch, operands.InvertedSource);
+    _register(body, 0xF7, operands.Wide, 2, scratch);
+    if (operands.Memory)
+        _memory(body, operands, scratch);
+    else if (operands.Source == kStack)
+        _stack(body, 0x23, operands.Wide, scratch, 8);
+    else
+        _register(body, 0x23, operands.Wide, scratch, operands.Source);
+    if (operands.Destination == kStack)
+        _stack(body, 0x89, operands.Wide, scratch, 8);
+    else
+        _register(body, 0x8B, operands.Wide, operands.Destination, scratch);
+    _stack(body, 0x8B, true, scratch, 0);
+    _stack(body, 0x8D, true, kStack, kFrameSize);
+    if (operands.Destination == kStack)
+        _stack(body, 0x8B, operands.Wide, kStack, 8 - kFrameSize);
+}
+
+}

core/relinker/codegen/tests/AndnLoweringTests.cpp

@@ -0,0 +1,245 @@
+#include <codegen/x86/AndnLowering.hpp>
+#include <codegen/x86/IAmd64OnlyInstructionMatcher.hpp>
+#include <codegen/CodegenException.hpp>
+#include <array>
+#include <cstring>
+#include <iostream>
+#include <limits>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__)
+#include <sys/mman.h>
+#endif
+
+namespace {
+
+using Bytes = std::vector<std::uint8_t>;
+
+void require(bool condition, const char* message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+Bytes instruction(bool wide, std::uint8_t dst, std::uint8_t first, std::uint8_t second) {
+    return {0xC4, static_cast<std::uint8_t>(0xE2 ^ ((dst & 8) << 4) ^ ((second & 8) << 2)),
+        static_cast<std::uint8_t>((wide ? 0x80 : 0) | ((~first & 15) << 3)), 0xF2,
+        static_cast<std::uint8_t>(0xC0 | ((dst & 7) << 3) | (second & 7))};
+}
+
+void decoding() {
+    for (bool wide : {false, true}) {
+        for (std::uint8_t dst = 0; dst < 16; ++dst) {
+            for (std::uint8_t first = 0; first < 16; ++first) {
+                for (std::uint8_t second = 0; second < 16; ++second) {
+                    const auto bytes = instruction(wide, dst, first, second);
+                    const auto operands = Codegen::DecodeAndn(bytes.data(), bytes.size());
+                    require(operands && operands->Wide == wide && operands->Destination == dst && operands->InvertedSource == first && operands->Source == second && !operands->Memory, "ANDN register decoding changed an operand");
+                }
+            }
+        }
+    }
+    const Bytes memory{0x64, 0x67, 0xC4, 0x02, 0xA8, 0xF2, 0x8C, 0xAC, 0x10, 0, 0, 0};
+    const auto operands = Codegen::DecodeAndn(memory.data(), memory.size());
+    require(operands && operands->Destination == 9 && operands->InvertedSource == 10 && operands->Memory && operands->Memory->RexIndexBase == 3 && operands->Memory->Prefixes == Bytes{0x64, 0x67}, "ANDN memory extensions or prefixes were lost");
+    for (const Bytes bytes : {Bytes{0x90}, Bytes{0xC4, 0xE2, 0xF0}, Bytes{0xC4, 0xE1, 0xF0, 0xF2, 0xC0}})
+        require(!Codegen::DecodeAndn(bytes.data(), bytes.size()), "Unrelated or truncated opcode matched ANDN");
+    for (const Bytes bytes : {Bytes{0xC4, 0xE2, 0xF4, 0xF2, 0xC0}, Bytes{0xC4, 0xE2, 0xF1, 0xF2, 0xC0}, Bytes{0xC4, 0xE2, 0xF0, 0xF2, 0x84}}) {
+        bool rejected = false;
+        try { (void)Codegen::DecodeAndn(bytes.data(), bytes.size()); }
+        catch (const Codegen::CodegenException&) { rejected = true; }
+        require(rejected, "Malformed ANDN was accepted");
+    }
+    const Bytes overflow{0xC4, 0xE2, 0xF0, 0xF2, 0x84, 0x24, 0xFF, 0xFF, 0xFF, 0x7F};
+    bool rejected = false;
+    try {
+        Codegen::StubBodyBuilder body;
+        Codegen::EmitAndn(body, *Codegen::DecodeAndn(overflow.data(), overflow.size()));
+    } catch (const Codegen::CodegenException&) { rejected = true; }
+    require(rejected, "An overflowing stack displacement was silently wrapped");
+}
+
+void matching() {
+    const auto bytes = instruction(true, 0, 1, 6);
+    const auto intel = Codegen::MakeAmd64OnlyInstructionMatcher();
+    const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta);
+    require(!intel->Match(bytes.data(), bytes.size()), "Intel mode rewrote ANDN");
+    const auto match = rosetta->Match(bytes.data(), bytes.size());
+    require(match && match->InstructionName == "ANDN" && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline && !match->Optional, "Rosetta mode did not require an ANDN trampoline");
+    const std::array<std::span<const std::uint8_t>, 2> sequence{bytes, bytes};
+    require(rosetta->MatchSequence(sequence, {}).has_value(), "An ANDN sequence could not move out of line");
+}
+
+#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__)
+
+struct State {
+    std::array<std::uint64_t, 16> Input{};
+    std::array<std::uint64_t, 16> Output{};
+    std::uint64_t Flags = 0;
+    std::array<std::uint64_t, 16> RedZone{};
+};
+
+class Execution {
+public:
+    Execution(const Bytes& site, std::uint64_t memoryValue) {
+        auto operands = Codegen::DecodeAndn(site.data(), site.size());
+        require(operands.has_value(), "Execution site was not ANDN");
+        for (auto reg : {3, 5, 12, 13, 14, 15}) push(static_cast<std::uint8_t>(reg));
+        memory(0x8D, 4, 4, -128);
+        rip({0x48, 0x89, 0x3D}, 0);
+        rip({0x48, 0x89, 0x25}, 8);
+        memory(0x89, 4, 7, 32);
+        bytes.insert(bytes.end(), {0x48, 0xB8});
+        append64(kSentinel);
+        for (int displacement = -128; displacement < 0; displacement += 8)
+            memory(0x89, 0, 4, displacement);
+        memory(0x8B, 0, 7, 16 * 8);
+        memory(0x89, 0, 4, 64);
+        for (std::uint8_t reg = 0; reg < 16; ++reg)
+            if (reg != 4 && reg != 7) memory(0x8B, reg, 7, reg * 8);
+        memory(0x8B, 7, 7, 7 * 8);
+        const auto siteOffset = bytes.size();
+        if (operands->Memory && operands->Memory->RipRelative) {
+            auto relocated = site;
+            const auto displacement = static_cast<std::int32_t>(4096 + 24 - (siteOffset + site.size()));
+            std::memcpy(relocated.data() + relocated.size() - 4, &displacement, sizeof(displacement));
+            operands = Codegen::DecodeAndn(relocated.data(), relocated.size());
+        }
+        Codegen::StubBodyBuilder builder;
+        Codegen::EmitAndn(builder, *operands);
+        auto body = builder.Finish();
+        const auto stubOffset = bytes.size();
+        Codegen::ApplyStubRelocations(body.Bytes, body.Relocations, siteOffset, stubOffset, 0);
+        bytes.insert(bytes.end(), body.Bytes.begin(), body.Bytes.begin() + static_cast<std::ptrdiff_t>(body.ReturnBranchOffset));
+        rip({0x48, 0x89, 0x25}, 16);
+        rip({0x48, 0x8B, 0x25}, 8);
+        memory(0x8D, 4, 4, -256);
+        for (std::uint8_t reg = 0; reg < 16; ++reg) if (reg != 4) push(reg);
+        bytes.push_back(0x9C);
+        rip({0x48, 0x8B, 0x3D}, 0);
+        memory(0x8B, 0, 4, 0);
+        memory(0x89, 0, 7, 32 * 8);
+        int offset = 8;
+        for (int reg = 15; reg >= 0; --reg) {
+            if (reg == 4) continue;
+            memory(0x8B, 0, 4, offset);
+            memory(0x89, 0, 7, (16 + reg) * 8);
+            offset += 8;
+        }
+        rip({0x48, 0x8B, 0x05}, 16);
+        memory(0x89, 0, 7, 20 * 8);
+        for (int slot = 0; slot < 16; ++slot) {
+            memory(0x8B, 0, 4, 256 + 128 - 128 + slot * 8);
+            memory(0x89, 0, 7, (33 + slot) * 8);
+        }
+        rip({0x48, 0x8B, 0x25}, 8);
+        memory(0x8D, 4, 4, 128);
+        for (auto reg : {15, 14, 13, 12, 5, 3}) pop(static_cast<std::uint8_t>(reg));
+        bytes.push_back(0xC3);
+        require(bytes.size() < 4096, "Execution wrapper exceeded its code page");
+        code = static_cast<std::uint8_t*>(mmap(nullptr, 8192, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
+        require(code != MAP_FAILED, "Cannot allocate execution test pages");
+        std::memcpy(code, bytes.data(), bytes.size());
+        std::memcpy(code + 4096 + 24, &memoryValue, sizeof(memoryValue));
+        require(mprotect(code, 4096, PROT_READ | PROT_EXEC) == 0, "Cannot make ANDN test code executable");
+    }
+
+    ~Execution() { if (code != MAP_FAILED) munmap(code, 8192); }
+
+    void Run(State& state) const { reinterpret_cast<void (*)(State*)>(code)(&state); }
+    const void* Memory() const { return code + 4096 + 24; }
+    static constexpr std::uint64_t kSentinel = 0xBADC0FFEE0DDF00Dull;
+
+private:
+    Bytes bytes;
+    std::uint8_t* code = reinterpret_cast<std::uint8_t*>(MAP_FAILED);
+
+    void append32(std::int32_t value) {
+        for (unsigned shift = 0; shift < 32; shift += 8) bytes.push_back(static_cast<std::uint8_t>(static_cast<std::uint32_t>(value) >> shift));
+    }
+    void append64(std::uint64_t value) {
+        for (unsigned shift = 0; shift < 64; shift += 8) bytes.push_back(static_cast<std::uint8_t>(value >> shift));
+    }
+    void rip(std::initializer_list<std::uint8_t> opcode, int slot) {
+        bytes.insert(bytes.end(), opcode);
+        append32(static_cast<std::int32_t>(4096 + slot - (bytes.size() + 4)));
+    }
+    void push(std::uint8_t reg) {
+        if (reg >= 8) bytes.push_back(0x41);
+        bytes.push_back(static_cast<std::uint8_t>(0x50 | (reg & 7)));
+    }
+    void pop(std::uint8_t reg) {
+        if (reg >= 8) bytes.push_back(0x41);
+        bytes.push_back(static_cast<std::uint8_t>(0x58 | (reg & 7)));
+    }
+    void memory(std::uint8_t opcode, std::uint8_t reg, std::uint8_t base, int displacement) {
+        bytes.insert(bytes.end(), {static_cast<std::uint8_t>(0x48 | ((reg & 8) >> 1) | ((base & 8) >> 3)), opcode,
+            static_cast<std::uint8_t>(0x80 | ((reg & 7) << 3) | (base & 7))});
+        if ((base & 7) == 4) bytes.push_back(0x24);
+        append32(displacement);
+    }
+};
+
+void check(const State& state, const Codegen::AndnOperands& operands, std::uint64_t second) {
+    const auto mask = operands.Wide ? ~std::uint64_t{0} : 0xFFFFFFFFull;
+    const auto expected = (~state.Input[operands.InvertedSource] & second) & mask;
+    for (std::size_t reg = 0; reg < 16; ++reg)
+        require(state.Output[reg] == (reg == operands.Destination ? expected : state.Input[reg]), "ANDN changed a source/scratch register or produced an incorrect result");
+    const auto flags = (expected == 0 ? 0x40u : 0u) | ((expected >> (operands.Wide ? 63 : 31)) != 0 ? 0x80u : 0u);
+    require((state.Flags & 0x8C1u) == flags, "ANDN did not preserve its defined flag semantics");
+    for (const auto value : state.RedZone) require(value == Execution::kSentinel, "ANDN overwrote the caller's red zone");
+}
+
+State initialState() {
+    State state;
+    for (std::size_t reg = 0; reg < 16; ++reg) state.Input[reg] = 0xABCD123456789000ull + reg * 0x1234567ull;
+    return state;
+}
+
+void execution() {
+    for (bool wide : {false, true}) {
+        for (std::uint8_t dst = 0; dst < 16; ++dst) {
+            for (const auto pair : {std::pair<std::uint8_t, std::uint8_t>{1, 6}, {dst, 10}, {11, dst}, {dst, dst}, {4, 9}, {9, 4}}) {
+                const auto site = instruction(wide, dst, pair.first, pair.second);
+                Execution executable(site, 0);
+                for (int variant = 0; variant < 3; ++variant) {
+                    auto state = initialState();
+                    if (variant == 1) state.Input.fill(0);
+                    if (variant == 2) state.Input.fill(~std::uint64_t{0});
+                    executable.Run(state);
+                    check(state, *Codegen::DecodeAndn(site.data(), site.size()), state.Input[pair.second]);
+                }
+            }
+        }
+        for (const Bytes tail : {Bytes{0x06}, Bytes{0x46, 0x10}, Bytes{0x84, 0x24, 0x40, 0, 0, 0}, Bytes{0x04, 0xCA}, Bytes{0x85, 0x10, 0, 0, 0}, Bytes{0x05, 0, 0, 0, 0}}) {
+            Bytes site{0xC4, 0xE2, static_cast<std::uint8_t>(wide ? 0xF0 : 0x70), 0xF2};
+            site.insert(site.end(), tail.begin(), tail.end());
+            constexpr std::uint64_t value = 0xFEDCBA9876543210ull;
+            Execution executable(site, value);
+            auto state = initialState();
+            state.Input[6] = reinterpret_cast<std::uint64_t>(executable.Memory()) - (tail[0] == 0x46 ? 16 : 0);
+            state.Input[2] = reinterpret_cast<std::uint64_t>(executable.Memory()) - 3 * 8;
+            state.Input[1] = 3;
+            state.Input[5] = reinterpret_cast<std::uint64_t>(executable.Memory()) - 16;
+            state.Output[0] = value;
+            executable.Run(state);
+            check(state, *Codegen::DecodeAndn(site.data(), site.size()), value);
+        }
+    }
+}
+#else
+void execution() { std::cout << "ANDN execution checks require a Linux/macOS x86-64 build\n"; }
+#endif
+
+}
+
+int main() {
+    try {
+        decoding();
+        matching();
+        execution();
+        std::cout << "ANDN lowering checks passed\n";
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/relinker/relinker/tests/test_to_rosetta.py

@@ -16,17 +16,17 @@ SITES = RDSEED + CLWB + b"\xc3"
 LOWERED = RDRAND + NOP4 + b"\xc3"
 
 
-def executable():
+def executable(code=SITES):
     image = fixture()
     struct.pack_into("<Q", image, 24, 0x10)
-    image[0x4010:0x4010 + len(SITES)] = SITES
+    image[0x4010:0x4010 + len(code)] = code
     return image
 
 
-def relink(relinker, directory, name, *options):
+def relink(relinker, directory, name, *options, code=SITES):
     source = Path(directory) / f"{name}.elf"
     output = Path(directory) / f"{name}.out"
-    source.write_bytes(bytes(executable()))
+    source.write_bytes(bytes(executable(code)))
     result = subprocess.run([str(relinker), "--skip-sce-module", *options, str(source), str(output)], capture_output=True, text=True, timeout=30)
     return result, output
 
@@ -52,6 +52,16 @@ def main():
             assert (LOWERED in data and SITES not in data) if lowered else (SITES in data and LOWERED not in data), f"{options} {'did not lower' if lowered else 'changed'} RDSEED and CLWB"
         result, _ = relink(relinker, directory, "windows", "--to-rosetta", "--windows")
         assert result.returncode != 0 and "--to-rosetta conflicts with --windows" in result.stdout + result.stderr, "--to-rosetta together with --windows was accepted"
+        andn = bytes.fromhex("c4e2f0f2c6c3")
+        for target in ((), ("--macos",)):
+            for conversion in ((), ("--to-intel",), ("--to-rosetta",)):
+                options = target + conversion
+                result, output = relink(relinker, directory, "andn" + "".join(options), *options, code=andn)
+                assert result.returncode == 0, (options, result.stdout, result.stderr)
+                lowered = conversion == ("--to-rosetta",)
+                assert (andn not in output.read_bytes()) == lowered, f"{options} {'did not lower' if lowered else 'changed'} ANDN"
+                if lowered:
+                    assert "ANDN" in result.stdout, result.stdout
     print("--to-rosetta integration tests passed")
 
 

docs/dev/debt/libkernel.md

@@ -153,7 +153,7 @@
 - [kern.proc.pathname](../../../core/libs/prx/libkernel/System/src/SystemConfiguration.cpp) (libkernel) returns `/app0/eboot.bin` for pid -1 and the calling process, following FreeBSD's `sysctl_kern_proc_pathname`; the exact pathname on retail consoles remains unverified.
 - [machdep.cacheline_size](../../../core/libs/prx/libkernel/System/src/SystemConfiguration.cpp) (libkernel) returns -1 with `ENOENT`, matching FreeBSD amd64 where this sysctl OID is not defined.
 - [hw.model](../../../core/libs/prx/libkernel/System/src/SystemConfiguration.cpp) (libkernel) reports the [host CPU's name](../../../core/libs/prx/libkernel/System/src/HostCpu.cpp), built as FreeBSD 11 builds `hw.model`: the CPUID brand string (leaves 0x80000002 to 0x80000004) without leading spaces ([identcpu.c](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/x86/x86/identcpu.c#L683-L691)). Trailing padding stays, as it does there, so an AMD host's name ends in spaces. The name a PS5 reports is unknown. A write returns -1 with `EPERM`. A host without an x86 CPUID brand string, such as an Apple Silicon Mac, throws. PPSA99203 only prints it (`[console] hw.model: %s`)
-- [Host CPU check](../../../core/libs/prx/libkernel/System/src/HostCpu.cpp) (libkernel) - when libkernel loads on a Linux or Windows x86-64 host, it logs which of the PS5 CPU's user-mode extensions CPUID reports and throws, naming the host CPU, if any extension is missing that LLVM selects from plain C/C++ for the PS5's Zen 2 (`-march=znver2`, [feature list](https://github.com/llvm/llvm-project/blob/llvmorg-19.1.7/llvm/lib/Target/X86/X86.td#L1459-L1513)): SSE to SSE4.2, SSSE3, POPCNT, LZCNT, BMI1, BMI2, MOVBE, CMPXCHG16B, LAHF/SAHF, AVX, AVX2, FMA, F16C, and the operating system's AVX state in XCR0 (#3719). SSE4A, CLZERO and SHA, which `--to-intel` converts, and AES, PCLMULQDQ, RDRAND, RDSEED, ADX, CLFLUSHOPT, CLWB, RDPID, XSAVEOPT, XSAVEC and PREFETCHW, which compilers emit only for intrinsics or explicit prefetches, are logged but not required: a title that executes one of them on a host without it still stops with an illegal instruction. macOS hosts are not checked, because Rosetta runs AVX2, FMA, BMI1/2, F16C and MOVBE without reporting them through CPUID
+- [Host CPU check](../../../core/libs/prx/libkernel/System/src/HostCpu.cpp) (libkernel) - when libkernel loads on a Linux or Windows x86-64 host, it logs which of the PS5 CPU's user-mode extensions CPUID reports and throws, naming the host CPU, if any extension is missing that LLVM selects from plain C/C++ for the PS5's Zen 2 (`-march=znver2`, [feature list](https://github.com/llvm/llvm-project/blob/llvmorg-19.1.7/llvm/lib/Target/X86/X86.td#L1459-L1513)): SSE to SSE4.2, SSSE3, POPCNT, LZCNT, BMI1, BMI2, MOVBE, CMPXCHG16B, LAHF/SAHF, AVX, AVX2, FMA, F16C, and the operating system's AVX state in XCR0 (#3719). SSE4A, CLZERO and SHA, which `--to-intel` converts, and AES, PCLMULQDQ, RDRAND, RDSEED, ADX, CLFLUSHOPT, CLWB, RDPID, XSAVEOPT, XSAVEC and PREFETCHW, which compilers emit only for intrinsics or explicit prefetches, are logged but not required: a title that executes one of them on a host without it still stops with an illegal instruction. macOS hosts are not checked: some Rosetta versions execute extensions without reporting them through CPUID, but support differs by host. ANDN and RDRAND fault on an M1 with macOS 15.3.1; see the [Rosetta conversion limits](relinker.md)
 - [machdep.tsc_freq](../../../core/libs/prx/libkernel/System/src/SystemConfiguration.cpp) (libkernel) reports the host TSC frequency that `sceKernelGetTscFrequency` measures, as a 64-bit value like FreeBSD 11's [`sysctl_machdep_tsc_freq`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/x86/x86/tsc.c#L658-L676). FreeBSD lets root write it to change the TSC timecounter's frequency; a write here returns -1 with `EPERM`. PPSA99203 reads it at start-up and measures the TSC itself when the value is 100 MHz or less
 - [getrlimit](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - `RLIMIT_CPU`, `RLIMIT_FSIZE`, `RLIMIT_CORE`, `RLIMIT_SBSIZE`, `RLIMIT_VMEM`/`AS`, `RLIMIT_NPTS`, `RLIMIT_SWAP`, `RLIMIT_KQUEUES` and `RLIMIT_UMTXP` report infinity as in FreeBSD's process-0 defaults, and `RLIMIT_DATA` mirrors the host limit. `RLIMIT_NOFILE` reports, as both soft and hard limit, the number of descriptors after which AnyPS5 itself fails with `EMFILE`: on Windows the C runtime's 8192 low-level descriptors ([`_setmaxstdio`](https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio)), which guest files, pipes and sockets all take, and on Linux the host's `RLIMIT_NOFILE`, whose soft limit libkernel raises to the hard limit at load. The console's limit is not known; XPSemu (PPSA97358) reads it in QEMU's `os_setup_limits` and raises the soft limit to the hard one when they differ. `RLIMIT_STACK`, `RLIMIT_RSS`, `RLIMIT_MEMLOCK` and `RLIMIT_NPROC` throw because no console values are established.
 - [setrlimit](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) validates the resource and input in FreeBSD order, maps negative limits to infinity, and lowers a soft limit above the hard limit. A request succeeds only when its normalized values match `getrlimit`; changing limits throws because guest limit state is not stored.

docs/dev/debt/relinker.md

@@ -16,6 +16,8 @@
 - In executables and guest modules with a `PT_SCE_DYNLIBDATA` segment, the [code analysis](../../../core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp) of the Windows and macOS outputs follows code only from the end of `PT_INTERP` to the end of the PLT after the last function of the unwind table, the end the pipeline's scans above use; a guest module with PLT relocations and no PLT after its last unwind function fails as the executable does. Code after that PLT is not analyzed: a TLS access in it is not rewritten, an entry point or pointer into it is ignored, and a function symbol or init/fini array entry pointing into it fails the conversion.
 - A relocation against a dynamic symbol entry that is all zero, like the null symbol, takes the value 0 plus its addend, as the gABI defines for `STN_UNDEF` ([pipeline](../../../core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp)). The Linux output imports such an entry as an unnamed weak symbol, which the host loader resolves to 0; the Windows and macOS outputs write the value into the slot ([relocation builder](../../../core/relinker/elfpatcher/src/windows/WindowsRelocationBuilder.cpp)). What the console's loader does with such an entry is not measured. OliSe Player (PPSA01153) has seven of them, and its code calls through those slots only when they are not 0 ([#4437](https://github.com/boykopovar/AnyPS5/issues/4437)).
 - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test.
-- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Rosetta runs AVX2, FMA, BMI1/2, F16C and MOVBE but reports none of them through CPUID, so code that checks CPUID takes its baseline path. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB; the same value reaches RDTSCP's ECX, which is not converted. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests; that the converted code runs under Rosetta was checked on an M5 Max only, outside this repository's CI.
+- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1. ANDN is lowered, but RDRAND and other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB; the same value reaches RDTSCP's ECX, which is not converted. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI.
 - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted).
 - `--trace-imports` ([entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp)) exists for Windows output only. Only imports bound through a PLT slot, or through a GOT slot whose symbol is a function, are traced; a function reached through any other relocation, or looked up at run time with `sceKernelDlsym`, is not, and neither are weak imports or the imports of modules loaded at run time
+
+- [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws; other BMI instructions are not lowered. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified.

docs/user/USAGE.md

@@ -60,7 +60,7 @@ All switches are disabled by default. `unused-filter` defaults to `0`; `--rpath`
 | `--trace-imports`             | Pass each function import through the libc.prx call tracer, see [tracing system library calls](#tracing-system-library-calls). Requires `--windows`; conflicts with `--lazy-binding`.                                                                                                                                                                                                                                                                                       |
 | `--linux-fixed-base <address>` | Load the Linux executable at this fixed address, given in decimal or hexadecimal (`0x`): every virtual address of the image is offset by it, so virtual address 0 is placed at the address, as the console places an executable. Requires a 64 KiB-aligned address from `0x10000` through `0x7fffffff0000` that is a multiple of every `PT_LOAD` alignment, and an image that ends in the user address space. The executable is written as `ET_EXEC`, without ASLR. Bundled modules keep their normal placement. Conflicts with `--windows`, `--macos` and `--lazy-binding`. |
 | `--to-intel`                  | Convert supported AMD-only instructions in the executable and bundled modules. Unsupported instructions or unreachable conversion stubs cause an error.                                                                                                                                                                                                                                                                                                                     |
-| `--to-rosetta`                | Convert as `--to-intel` does and also lower what Rosetta lacks: RDSEED becomes RDRAND, CLWB a NOP, and RDPID returns processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`.                                                    |
+| `--to-rosetta`                | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, RDSEED becomes RDRAND, CLWB a NOP, and RDPID returns processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`.                                                    |
 | `unused-filter=0`             | Keep all imported NID references.                                                                                                                                                                                                                                                                                                                                                                                                                                           |
 | `unused-filter=1`             | Filter unused non-PLT imports using control-flow and GOT access analysis, treating address-taken code as reachable; preserve PLT imports.                                                                                                                                                                                                                                                                                                                                   |
 | `unused-filter=2`             | Apply strict unused-import analysis and compact the PLT. Unsupported analysis cases cause an error.                                                                                                                                                                                                                                                                                                                                                                         |