fix(import-audit): walk host-to-host .prx dependencies of built libs

docs/user/USAGE.md

@@ -250,6 +250,8 @@ Each registry has a companion with the same prefix and the suffix `.needed.json`
 
 The report also lists needed libraries that have no file in `--libs` or `--modules`, including dependencies with zero imports. These do not increase the import or reference counts. Windows can skip a missing dependency when no import needs it; Linux still needs the file. The audit also lists imports exported only by a library other than the one they name; those resolve on Linux and can fail on Windows.
 
+The audit also walks the built libraries' own host dependencies: each `.prx` under `--libs` has its PE imports (or ELF `DT_NEEDED` entries) checked for `.prx` names with no file in `--libs` or `--modules`, following present libraries transitively. Gaps fail the audit and are reported as `Importer: <root> Chain: <root> -> ... -> <missing>` lines, since the loader fails on them even when every game import resolves.
+
 | Option | Effect |
 |--------|--------|
 | `--libs <dir>` | Built `.prx` directory. Required; repeat for several. |

tools/import_audit.py

@@ -220,7 +220,7 @@ def elf_imports(path, data):
     return symbols, needed
 
 
-def pe_exports(path, data):
+def pe_headers(path, data):
     header = unpack("<I", data, 0x3c, path, "PE header offset")[0]
     if unpack("<I", data, header, path, "PE signature")[0] != 0x4550:
         raise AuditError(f"{path}: invalid PE signature at 0x{header:x}")
@@ -233,12 +233,10 @@ def pe_exports(path, data):
     if optional_size < directory + 8:
         raise AuditError(f"{path}: PE optional header of {optional_size} bytes has no export directory")
     take(data, optional, optional_size, path, "PE optional header")
-    if unpack("<I", data, optional + directory - 4, path, "data directory count")[0] == 0:
+    count = unpack("<I", data, optional + directory - 4, path, "data directory count")[0]
+    if count == 0:
         raise AuditError(f"{path}: PE file declares no data directories")
     headers_size = unpack("<I", data, optional + 60, path, "header size")[0]
-    rva, size = unpack("<II", data, optional + directory, path, "export data directory")
-    if not rva or size < 40:
-        raise AuditError(f"{path}: PE file has no export table")
     sections = [unpack("<8sIIIIIIHHI", data, optional + optional_size + index * 40, path, f"PE section {index}") for index in range(section_count)]
 
     def to_offset(address, length):
@@ -256,6 +254,15 @@ def pe_exports(path, data):
             return raw_offset + relative, min(raw_offset + raw_size, len(data))
         raise AuditError(f"{path}: RVA 0x{address:x} is not inside any section")
 
+    return to_offset, optional + directory, count
+
+
+def pe_exports(path, data):
+    to_offset, directory, _ = pe_headers(path, data)
+    rva, size = unpack("<II", data, directory, path, "export data directory")
+    if not rva or size < 40:
+        raise AuditError(f"{path}: PE file has no export table")
+
     table = unpack("<IIHHIIIIIII", data, to_offset(rva, 40)[0], path, "export directory")
     function_count, name_count, functions_rva, names_rva, ordinals_rva = table[6:11]
     names = set()
@@ -287,6 +294,37 @@ def pe_exports(path, data):
     return names
 
 
+def pe_imports(path, data):
+    to_offset, directory, count = pe_headers(path, data)
+    if count < 2:
+        return set()
+    rva, size = unpack("<II", data, directory + 8, path, "import data directory")
+    if not rva:
+        return set()
+    if size < 20:
+        raise AuditError(f"{path}: PE import table of {size} bytes holds no import entry")
+    names = set()
+    for index in range(size // 20):
+        entry = unpack("<IIIII", data, to_offset(rva + index * 20, 20)[0], path, "import directory entry")
+        if entry == (0, 0, 0, 0, 0):
+            return names
+        if not entry[3]:
+            raise AuditError(f"{path}: PE import entry {index} names no library")
+        offset, limit = to_offset(entry[3], 1)
+        names.add(cstring(data, offset, limit, path, "import library name", "ASCII", "strict"))
+    raise AuditError(f"{path}: PE import table is not NUL-terminated")
+
+
+def read_host_imports(path, data=None):
+    if data is None:
+        data = path.read_bytes()
+    if data[:2] == b"MZ":
+        return pe_imports(path, data)
+    if data[:4] == b"\x7fELF":
+        return elf_imports(path, data)[1]
+    raise AuditError(f"{path}: neither ELF nor PE (first bytes {data[:4]!r})")
+
+
 def read_exports(path, data=None):
     if data is None:
         data = path.read_bytes()
@@ -327,6 +365,49 @@ def built_libraries(directories):
     return {name: sorted(files) for name, files in exports.items()}, libraries
 
 
+def built_host_dependencies(directories):
+    """Map each built .prx basename to the sorted .prx names it imports itself.
+
+    The loader resolves these host-to-host dependencies with LoadLibrary, so a
+    name with no file in --libs or --modules fails even when every game import
+    is implemented. System DLLs resolve through the OS and are not reported.
+    """
+    direct = {}
+    images = {}
+    for directory in directories:
+        for file in directory_files(directory, "*.prx"):
+            data = file.read_bytes()
+            digest = hashlib.sha256(data).digest()
+            if file.name in images:
+                previous, fingerprint = images[file.name]
+                if digest != fingerprint:
+                    raise AuditError(f"ambiguous built library {file.name}: {previous} and {file} have different contents")
+                continue
+            images[file.name] = (file, digest)
+            direct[file.name] = sorted({name for name in read_host_imports(file, data) if name.lower().endswith(".prx")})
+    return direct
+
+
+def audit_host_dependencies(direct, libraries, modules):
+    """Walk host imports transitively; list each missing library with its chain."""
+    available = set(libraries) | set(modules)
+    missing = []
+    for root in sorted(direct):
+        expanded = set()
+        stack = [(name, [root, name]) for name in reversed(direct[root])]
+        while stack:
+            name, chain = stack.pop()
+            if name not in available:
+                missing.append({"importer": root, "chain": chain})
+            elif name in direct and name not in expanded and name not in chain[1:-1]:
+                expanded.add(name)
+                for dependency in reversed(direct[name]):
+                    if dependency in chain:
+                        continue
+                    stack.append((dependency, chain + [dependency]))
+    return sorted(missing, key=lambda entry: (entry["importer"], entry["chain"]))
+
+
 def module_files(directory):
     for file in directory_files(directory, "*"):
         with file.open("rb") as handle:
@@ -538,7 +619,7 @@ def git_stamp(source=ROOT):
     return commit, dirty
 
 
-def summarize(records, references, missing):
+def summarize(records, references, missing, missing_hosts=()):
     counts = {kind: sum(1 for record in records if record["class"] == kind) for kind in CLASSES}
     by_references = {kind: sum(record["references"] for record in records if record["class"] == kind) for kind in CLASSES}
     return {
@@ -547,6 +628,7 @@ def summarize(records, references, missing):
         "unique_by_class": counts,
         "references_by_class": by_references,
         "missing_libraries": missing,
+        "missing_host_dependencies": [{"importer": entry["importer"], "chain": list(entry["chain"])} for entry in missing_hosts],
         "library_mismatch": sum(1 for record in records if record["library_mismatch"]),
     }
 
@@ -560,7 +642,7 @@ def label(record):
     return f"{text}  (imported by {record['module']})" if record.get("module") else text
 
 
-def render(summary, records, missing, stops):
+def render(summary, records, missing, stops, missing_hosts=()):
     lines = [f"{summary['references']} references, {summary['unique_imports']} unique imports"]
     for kind in CLASSES:
         lines.append(f"  {kind:<12}{summary['unique_by_class'][kind]:>6} imports  {summary['references_by_class'][kind]:>6} references")
@@ -587,6 +669,10 @@ def render(summary, records, missing, stops):
         lines.extend(f"  {library}: {count} imports" for library, count in missing.items())
         if any(count == 0 for count in missing.values()):
             lines.append("  Libraries with 0 imports are declared dependencies; Windows can skip them if no import needs them")
+    if missing_hosts:
+        lines.append("")
+        lines.append(f"Host dependencies with no file in --libs or --modules ({len(missing_hosts)})")
+        lines.extend(f"  Importer: {entry['importer']} Chain: {' -> '.join(entry['chain'])}" for entry in missing_hosts)
     if summary["library_mismatch"]:
         lines.append("")
         lines.append(f"{summary['library_mismatch']} imports are exported only by a library other than the one they name: they resolve on Linux and can fail on Windows")
@@ -618,11 +704,13 @@ def main(argv=None):
         imports = [entry for _, entries in registries for entry in entries]
         needed = sorted({library for path in dependency_registries for library in read_needed_registry(path)})
         exports, libraries = built_libraries(args.libs)
+        host_dependencies = built_host_dependencies(args.libs)
         module_imports, module_exports = read_modules(args.modules)
         records, missing = audit(imports, exports, source_stubs(args.source), libraries, set(module_imports), module_imports, module_exports, needed_libraries=needed)
+        missing_hosts = audit_host_dependencies(host_dependencies, libraries, set(module_imports))
         if args.names:
             attach_names(records, load_names(args.names))
-        summary = summarize(records, sum(record["references"] for record in records), missing)
+        summary = summarize(records, sum(record["references"] for record in records), missing, missing_hosts)
         stops = [{"registry": str(path), "stop": current_stop(entries, records)} for path, entries in registries]
         if args.json:
             commit, dirty = git_stamp(Path(args.source))
@@ -638,8 +726,8 @@ def main(argv=None):
     except OSError as error:
         print(f"FAIL: {error.strerror}: {error.filename}" if error.filename else f"FAIL: {error}", file=sys.stderr)
         return 2
-    print(render(summary, records, missing, stops))
-    return 1 if summary["unique_by_class"]["absent"] or missing or (args.strict_providers and summary["library_mismatch"]) else 0
+    print(render(summary, records, missing, stops, missing_hosts))
+    return 1 if summary["unique_by_class"]["absent"] or missing or missing_hosts or (args.strict_providers and summary["library_mismatch"]) else 0
 
 
 if __name__ == "__main__":

tools/tests/test_import_audit.py

@@ -136,6 +136,39 @@ def build_pe(names):
     return bytes(image) + raw + b"\xc3" * (16 * max(1, len(names)))
 
 
+def build_pe_with_imports(dlls, names=()):
+    idata_rva, edata_rva, text_rva, raw_offset = 0x1000, 0x2000, 0x3000, 0x200
+    directory, strings = b"", b""
+    for dll in dlls:
+        directory += struct.pack("<IIIII", 0, 0, 0, idata_rva + 20 * (len(dlls) + 1) + len(strings), 0)
+        strings += dll.encode() + b"\0"
+    idata = directory + bytes(20) + strings
+    exported, rvas = b"", []
+    start = 40 + 10 * len(names)
+    for name in names:
+        rvas.append(edata_rva + start + len(exported))
+        exported += name.encode() + b"\0"
+    edata = struct.pack("<IIHHIIIIIII", 0, 0, 0, 0, 0, 1, len(names), len(names), edata_rva + 40, edata_rva + 40 + 4 * len(names), edata_rva + 40 + 8 * len(names))
+    edata += b"".join(struct.pack("<I", text_rva + 16 * index) for index in range(len(names)))
+    edata += b"".join(struct.pack("<I", rva) for rva in rvas)
+    edata += b"".join(struct.pack("<H", index) for index in range(len(names))) + exported
+    text = b"\xc3" * (16 * max(1, len(names)))
+    image = bytearray(raw_offset)
+    image[0:2] = b"MZ"
+    image[0x3c:0x40] = struct.pack("<I", 0x40)
+    image[0x40:0x44] = b"PE\0\0"
+    image[0x44:0x58] = struct.pack("<HHIIIHH", 0x8664, 3, 0, 0, 0, 240, 0x2022)
+    image[0x58:0x5a] = struct.pack("<H", 0x20b)
+    image[0x58 + 60:0x58 + 64] = struct.pack("<I", raw_offset)
+    image[0x58 + 108:0x58 + 112] = struct.pack("<I", 16)
+    image[0x58 + 112:0x58 + 120] = struct.pack("<II", edata_rva, len(edata))
+    image[0x58 + 120:0x58 + 128] = struct.pack("<II", idata_rva, len(directory) + 20)
+    image[0x58 + 240:0x58 + 280] = struct.pack("<8sIIIIIIHHI", b".idata", len(idata), idata_rva, len(idata), raw_offset, 0, 0, 0, 0, 0x40000040)
+    image[0x58 + 280:0x58 + 320] = struct.pack("<8sIIIIIIHHI", b".edata", len(edata), edata_rva, len(edata), raw_offset + len(idata), 0, 0, 0, 0, 0x40000040)
+    image[0x58 + 320:0x58 + 360] = struct.pack("<8sIIIIIIHHI", b".text", len(text), text_rva, len(text), raw_offset + len(idata) + len(edata), 0, 0, 0, 0, 0x60000020)
+    return bytes(image) + idata + edata + text
+
+
 def write_source(root, library, text):
     directory = Path(root) / library
     directory.mkdir(parents=True, exist_ok=True)
@@ -1162,6 +1195,122 @@ class ImportAuditTests(unittest.TestCase):
         struct.pack_into("<II", image, 0x58 + 96, 0x1000, 56)
         self.assertEqual(import_audit.pe_exports(Path("lib.prx"), image), {"alpha"})
 
+    def test_pe_imports_reader(self):
+        path = Path("lib.prx")
+        self.assertEqual(import_audit.pe_imports(path, build_pe_with_imports(["KERNEL32.dll", "libSceIme.prx"], ["alpha"])),
+                         {"KERNEL32.dll", "libSceIme.prx"})
+        self.assertEqual(import_audit.pe_imports(path, build_pe_with_imports([], ["alpha"])), set())
+        self.assertEqual(import_audit.pe_imports(path, build_pe(["alpha"])), set())
+        image = build_pe_with_imports(["libSceIme.prx"], ["alpha"])
+        for cut in (0x3c, 0x60, 0x210):
+            with self.subTest(cut=cut):
+                with self.assertRaisesRegex(import_audit.AuditError, "lib.prx"):
+                    import_audit.pe_imports(path, image[:cut])
+
+    def test_pe_imports_without_an_import_directory(self):
+        image = bytearray(build_pe_with_imports(["libSceIme.prx"], ["alpha"]))
+        struct.pack_into("<I", image, 0x58 + 108, 1)
+        self.assertEqual(import_audit.pe_imports(Path("lib.prx"), image), set())
+        struct.pack_into("<I", image, 0x58 + 108, 16)
+        struct.pack_into("<II", image, 0x58 + 120, 0, 0)
+        self.assertEqual(import_audit.pe_imports(Path("lib.prx"), image), set())
+
+    def test_pe_import_table_must_terminate(self):
+        image = bytearray(build_pe_with_imports(["libSceIme.prx"], ["alpha"]))
+        struct.pack_into("<I", image, 0x58 + 124, 20)
+        with self.assertRaisesRegex(import_audit.AuditError, "NUL-terminated"):
+            import_audit.pe_imports(Path("lib.prx"), image)
+
+    def test_pe_import_entry_must_name_a_library(self):
+        image = bytearray(build_pe_with_imports(["libSceIme.prx"], ["alpha"]))
+        struct.pack_into("<II", image, 0x200 + 12, 0, 1)
+        with self.assertRaisesRegex(import_audit.AuditError, "names no library"):
+            import_audit.pe_imports(Path("lib.prx"), image)
+
+    def test_pe_import_name_must_be_inside_a_section(self):
+        image = bytearray(build_pe_with_imports(["libSceIme.prx"], ["alpha"]))
+        struct.pack_into("<I", image, 0x200 + 12, 0xffffff)
+        with self.assertRaisesRegex(import_audit.AuditError, "not inside any section"):
+            import_audit.pe_imports(Path("lib.prx"), image)
+
+    def test_host_dependency_gap_fails(self):
+        work = self.work
+        (work.libs / "libc.prx").write_bytes(build_pe_with_imports([], ["AAAAAAAAAAA"]))
+        (work.libs / "libSceVideoOut.prx").write_bytes(build_pe_with_imports(
+            ["KERNEL32.dll", "libc.prx", "libSceIme.prx", "libSceKeyboard.prx"], ["BBBBBBBBBBB"]))
+        registry = work.root / "game.json"
+        write_registry(registry, [("AAAAAAAAAAA", "libc.prx")])
+        result = work.root / "audit.json"
+        code, out, err = work.run(registry, "--json", str(result))
+        self.assertEqual((code, err), (1, ""))
+        data = json.loads(result.read_text())
+        self.assertEqual(data["missing_host_dependencies"], [
+            {"importer": "libSceVideoOut.prx", "chain": ["libSceVideoOut.prx", "libSceIme.prx"]},
+            {"importer": "libSceVideoOut.prx", "chain": ["libSceVideoOut.prx", "libSceKeyboard.prx"]},
+        ])
+        self.assertIn("Importer: libSceVideoOut.prx Chain: libSceVideoOut.prx -> libSceIme.prx", out)
+        self.assertIn("Importer: libSceVideoOut.prx Chain: libSceVideoOut.prx -> libSceKeyboard.prx", out)
+        self.assertNotIn("KERNEL32", out)
+
+    def test_transitive_host_dependency_reports_chain(self):
+        work = self.work
+        (work.libs / "libSceA.prx").write_bytes(build_pe_with_imports(["libSceB.prx"], ["AAAAAAAAAAA"]))
+        (work.libs / "libSceB.prx").write_bytes(build_pe_with_imports(["libSceMissing.prx"], ["BBBBBBBBBBB"]))
+        registry = work.root / "game.json"
+        write_registry(registry, [])
+        result = work.root / "audit.json"
+        code, out, err = work.run(registry, "--json", str(result))
+        self.assertEqual((code, err), (1, ""))
+        data = json.loads(result.read_text())
+        self.assertEqual(data["missing_host_dependencies"], [
+            {"importer": "libSceA.prx", "chain": ["libSceA.prx", "libSceB.prx", "libSceMissing.prx"]},
+            {"importer": "libSceB.prx", "chain": ["libSceB.prx", "libSceMissing.prx"]},
+        ])
+        self.assertIn("Importer: libSceA.prx Chain: libSceA.prx -> libSceB.prx -> libSceMissing.prx", out)
+
+    def test_host_dependency_satisfied_by_module(self):
+        work = self.work
+        (work.libs / "libSceA.prx").write_bytes(build_pe_with_imports(["libSceIme.prx"], ["AAAAAAAAAAA"]))
+        (work.modules / "libSceIme.prx").write_bytes(exporting())
+        registry = work.root / "game.json"
+        write_registry(registry, [])
+        code, out, err = work.run(registry)
+        self.assertEqual((code, err), (0, ""))
+        self.assertNotIn("Host dependencies", out)
+
+    def test_complete_host_dependencies_pass(self):
+        work = self.work
+        (work.libs / "libc.prx").write_bytes(build_pe_with_imports([], ["AAAAAAAAAAA"]))
+        (work.libs / "libSceVideoOut.prx").write_bytes(build_pe_with_imports(["libc.prx"], ["BBBBBBBBBBB"]))
+        registry = work.root / "game.json"
+        write_registry(registry, [])
+        code, out, err = work.run(registry)
+        self.assertEqual((code, err), (0, ""))
+        self.assertNotIn("Host dependencies", out)
+
+    def test_elf_host_dependencies_are_walked(self):
+        work = self.work
+        (work.libs / "libSceA.prx").write_bytes(build_elf(
+            [("AAAAAAAAAAA", 1, GLOBAL_FUNCTION)], ["libSceGhost.prx", "KERNEL32.dll"]))
+        registry = work.root / "game.json"
+        write_registry(registry, [])
+        result = work.root / "audit.json"
+        code, out, err = work.run(registry, "--json", str(result))
+        self.assertEqual((code, err), (1, ""))
+        data = json.loads(result.read_text())
+        self.assertEqual(data["missing_host_dependencies"], [
+            {"importer": "libSceA.prx", "chain": ["libSceA.prx", "libSceGhost.prx"]},
+        ])
+        self.assertNotIn("KERNEL32", out)
+
+    def test_conflicting_host_libraries_are_ambiguous(self):
+        first, second = self.work.libs, self.work.root / "other-libs"
+        second.mkdir()
+        (first / "libSceA.prx").write_bytes(build_pe_with_imports(["libSceB.prx"], ["AAAAAAAAAAA"]))
+        (second / "libSceA.prx").write_bytes(build_pe_with_imports([], ["AAAAAAAAAAA"]))
+        with self.assertRaisesRegex(import_audit.AuditError, "ambiguous"):
+            import_audit.built_host_dependencies([first, second])
+
 
     def test_elf_unnamed_defined_symbol_is_not_an_export(self):
         image = bytearray(exporting("alpha"))