diff --git a/qa/static/test_ipk.py b/qa/static/test_ipk.py index bbde278..a72fb43 100644 --- a/qa/static/test_ipk.py +++ b/qa/static/test_ipk.py @@ -567,21 +567,75 @@ def test_the_payload_keeps_libsodium_as_a_symlink(): % (lib, os.readlink(lib))) +# Scripts that build or ship something, and could therefore host a gate. +def _buildish_scripts(): + named = ["bin/common.sh", "bin/build-packages.sh", "bin/verify.sh", + "bin/patch.sh", "bin/pack.sh", "pkgs/lib.sh", + "tools/python/build.sh", "tools/python-packages/build.sh", + "tools/python-packages/build-libsodium.sh"] + found = [ROOT / n for n in named] + found += sorted(ROOT.glob("pkgs/*/build.sh")) + found += sorted(ROOT.glob("pkgs/3rdparty/*/build.sh")) + return [f for f in found if f.is_file()] + + +# The compiler wrapper check is NOT a second gate and these four files may +# keep it. It compiles a hello-world and reads THAT object's header, so what +# it judges is the toolchain, not anything the build produced -- a question +# qa/replica/test_abi.py structurally cannot answer, because a filesystem +# sweep cannot see a compiler. It also fails in a second, before a whole feed +# is built on a wrapper that quietly lost -mnan=2008. +# +# They are allowed the raw e_flags words and nothing else: an artefact gate +# needs to say nan2008/o32/mips32r2 (or call mips_abi_gate) to do its job, and +# the wrapper checks never do, so that vocabulary stays forbidden everywhere. +WRAPPER_CHECK_OK = { + "pkgs/lib.sh", + "tools/python/build.sh", + "tools/python-packages/build.sh", + "tools/python-packages/build-libsodium.sh", +} + +# "nan2008" and not "mips32r2": an artefact gate has to read the NaN encoding +# out of the header, and only a gate ever says that word -- whereas mips32r2 +# is ALSO the name of an ISA compiler flag, and pkgs/3rdparty/openssl/build.sh +# legitimately passes -mips32r2 to put the ISA back where the printer is. A +# guard word that collides with a build flag reports the flag, which is a +# false alarm and, once it is tuned out, a guard nobody reads. +GATE_WORDS = ("nan2008", "mips_abi_gate") + +# The raw e_flags words. Forbidden outside the four files allowed the wrapper +# check -- inside them a hex-only gate would still slip past, which is a known +# and accepted limit: two of those four are the superseded standalone trees +# under tools/ and nothing they build reaches a printer. +GATE_FLAGS = "0x7000140" + +# The decision the wrapper check makes, not the message it prints: a test +# pinned to the message goes green the moment someone edits the wording, and +# pkgs/lib.sh mentions both words again in its pkg_die string. +WRAPPER_CASE = "0x70001405|0x70001407)" + + def test_there_is_exactly_one_abi_gate_and_it_is_the_replica_one(): - """The ABI question is asked once, of the installed filesystem. + """The ABI question is asked of one thing, in one place: the filesystem + the kernel will actually load from. - It used to be asked in four places: mips_abi_gate in bin/common.sh, called - at the packaging boundary by bin/build-packages.sh and again by two - recipes that did not trust it, plus a narrower readelf|grep in - bin/verify.sh. Four implementations of one rule, each over a different - subset of the files -- and the largest binary we ship, HelixScreen, went - through a route (bin/patch.sh) that none of them read. + It used to be asked in six. mips_abi_gate in bin/common.sh, called at the + packaging boundary by bin/build-packages.sh and again by two recipes that + did not trust it; a narrower readelf|grep in bin/verify.sh over a single + .so; two tests in qa/replica/test_install.py reading one header a byte at + a time; and two more in tools/, which pinned the e_flags word EXACTLY and + were therefore strict enough to be wrong -- the low three bits are + NOREORDER/PIC/CPIC and vary between objects of identical ABI. - qa/replica/test_abi.py replaces all of it by sweeping the filesystem the - kernel will actually load from, which is the only place every route is - visible at once. This test exists so that a build-time gate cannot quietly - grow back beside it: two gates disagreeing about which files are exempt is - how the old set ended up with a hole in it. + Six implementations of one rule, each over a different subset of the + files, and between them they still could not see the largest binary we + ship: bin/patch.sh unpacks HelixScreen into /usr/prog and none of them + read that tree. + + This test is the ratchet. Two gates that disagree about which files are + exempt is how the first six ended up with a hole in them, so a seventh + must not be able to grow back quietly. """ gate = ROOT / "qa" / "replica" / "test_abi.py" assert gate.is_file(), ( @@ -592,23 +646,45 @@ def test_there_is_exactly_one_abi_gate_and_it_is_the_replica_one(): "qa/replica/test_abi.py no longer mentions %s, so it is not the " "gate this test thinks it is" % name) - shell = [ROOT / "bin" / "common.sh", ROOT / "bin" / "build-packages.sh", - ROOT / "bin" / "verify.sh", ROOT / "bin" / "patch.sh", - ROOT / "pkgs" / "lib.sh"] - shell += sorted(ROOT.glob("pkgs/*/build.sh")) - shell += sorted(ROOT.glob("pkgs/3rdparty/*/build.sh")) back = [] - for f in shell: - if not f.is_file(): - continue + for f in _buildish_scripts(): + rel = str(f.relative_to(ROOT)) + # Comments are stripped: a file is allowed to EXPLAIN where the gate + # went, and several of them now do. body = "\n".join(ln for ln in f.read_text().splitlines() if not ln.lstrip().startswith("#")) - if "nan2008" in body or "mips_abi_gate" in body: - back.append(str(f.relative_to(ROOT))) + hit = [w for w in GATE_WORDS if w in body] + # The hex words are the wrapper check's, and only in the four files + # that are allowed to make it. + if GATE_FLAGS in body and rel not in WRAPPER_CHECK_OK: + hit.append(GATE_FLAGS) + if hit: + back.append("%s (%s)" % (rel, ", ".join(hit))) assert not back, ( - "a build-time ABI gate is back in %s. There is one gate and it is " - "qa/replica/test_abi.py; a second one over a different subset of the " - "files is what the first set of four was." % ", ".join(back)) + "an ABI gate is back in the build path: %s. There is one gate and it " + "is qa/replica/test_abi.py, which reads the installed filesystem; a " + "second one over a different subset of the files is what the first " + "six were. The toolchain wrapper check is the one exception and is " + "allowed only in %s." + % ("; ".join(back), ", ".join(sorted(WRAPPER_CHECK_OK)))) + + +def test_the_toolchain_wrapper_is_still_checked_before_anything_is_built(): + """The check that survived the consolidation, kept honest from the side + that matters: it has to still be there. + + A wrapper that lost -mnan=2008 produces a tree that compiles, links and + passes every test on the build host, and is refused by the printer's + kernel at exec(). qa/replica/test_abi.py would eventually say so -- about + several hundred files at once, minutes later, in the replica lane. This + says it about the compiler, in a second, before the feed is built. + """ + body = (ROOT / "pkgs" / "lib.sh").read_text() + assert WRAPPER_CASE in body, ( + "pkgs/lib.sh no longer checks what its cross compiler emits, so a " + "wrapper that lost -mnan=2008 would build an entire feed of objects " + "the printer's kernel refuses, and nothing would say so until the " + "replica lane ran") def test_the_archives_are_built_by_upstream(): diff --git a/tools/python-packages/README.md b/tools/python-packages/README.md index 0850d9a..100d6de 100644 --- a/tools/python-packages/README.md +++ b/tools/python-packages/README.md @@ -19,9 +19,11 @@ with no gate and no Makefile target, and the questions it asked are settled. 18 sdists, 12 mipsel extension modules, from a 56-second clean build in one throwaway `debian:bookworm`. (This said 19 for a while; the list below is 18, and `versions.env` -- which is now the list that matters -- pins 18.) Every -`.so` is gated at `e_flags=0x70001407`, -`nan2008/o32/mips32r2` -- shared objects carry `EF_MIPS_PIC`, so they read -`…07` where an executable reads `…05`. +`.so` comes out `nan2008/o32/mips32r2` at `e_flags=0x70001407` -- shared +objects carry `EF_MIPS_PIC`, so they read `…07` where an executable reads +`…05`. This script no longer gates that itself; `qa/replica/test_abi.py` +reads every ELF on the installed filesystem, which is the one place the +question is asked now. Native: cffi, greenlet, lmdb, markupsafe, streaming-form-data, tornado speedups, pillow. Pure: jinja2, distro, inotify-simple, libnacl, dbus-next, @@ -144,7 +146,7 @@ purpose, so that a failure would be the interpreter and not an init script. ## libsodium: built into the prefix, and it costs anvil-env.sh nothing `build-libsodium.sh` cross-builds libsodium 1.0.20 with the Ingenic toolchain -(24s), gated at `0x70001407` nan2008/o32/mips32r2, SONAME `libsodium.so.26`, +(24s), producing `0x70001407` nan2008/o32/mips32r2, SONAME `libsodium.so.26`, needing only libpthread, libc and the loader. 406KB stripped, into `$MODDIR/lib`. diff --git a/tools/python-packages/build-libsodium.sh b/tools/python-packages/build-libsodium.sh index 66f07d5..333f578 100755 --- a/tools/python-packages/build-libsodium.sh +++ b/tools/python-packages/build-libsodium.sh @@ -110,24 +110,17 @@ STAGE=$W/stage-sodium rm -rf "$STAGE"; mkdir -p "$STAGE" make install DESTDIR="$STAGE" >> "$W/sodium-make.log" 2>&1 -# ---- the ABI gate, on the object that ships ------------------------------- -log "ABI gate" +# ---- what shipped, described rather than gated ----------------------------- +# The ABI ASSERTIONS that were here (e_flags == 0x70001407, and the Flags line +# decoding to nan2008/o32/mips32r2) are gone: qa/replica/test_abi.py reads +# every ELF on the installed filesystem, which is where this .so ends up and +# where four other partial copies of the same rule used to disagree. The +# readelf output stays because a build log that says what it produced is worth +# having -- it just no longer decides anything. +log "what the build produced" SO=$(readlink -f "$STAGE$PREFIX/lib/libsodium.so") file "$SO" mips-linux-gnu-readelf -h "$SO" | sed -n '/Class\|Data\|Type\|Flags/p' -F=$(eflags "$SO") -[ "$F" = "0x70001407" ] || { echo "!! e_flags $F, expected 0x70001407" >&2; exit 1; } -# The words that matter are decoded by readelf out of e_flags itself, on the -# header's Flags line -- `.MIPS.abiflags` (readelf -A) reports the ISA and the -# FP ABI but not the NaN encoding, so the header is where nan2008 is read. -FLAGLINE=$(mips-linux-gnu-readelf -h "$SO" | sed -n 's/.*Flags: *//p') -echo " decoded: $FLAGLINE" -for w in nan2008 o32 mips32r2; do - case ",$FLAGLINE," in - *"$w"*) ;; - *) echo "!! e_flags does not decode to $w: $FLAGLINE" >&2; exit 1 ;; - esac -done mips-linux-gnu-readelf -A "$SO" | sed -n '/ISA\|FP ABI/p' echo "-- SONAME / NEEDED" mips-linux-gnu-readelf -d "$SO" | sed -n '/SONAME\|NEEDED/p' diff --git a/tools/python-packages/build.sh b/tools/python-packages/build.sh index af0ccff..3e86e6b 100755 --- a/tools/python-packages/build.sh +++ b/tools/python-packages/build.sh @@ -380,23 +380,13 @@ find "$SP" -name '*.dist-info' -prune -exec rm -rf {} + 2>/dev/null || true rm -rf "$SP"/bin "$SP"/*.data 2>/dev/null || true # ============================================================= gates ======== -log "ABI gate on every .so in the tree" -BAD=0; N=0 -while IFS= read -r f; do - case "$(file -b "$f")" in *ELF*) ;; *) continue;; esac - hdr=$(mips-linux-gnu-readelf -h "$f" 2>/dev/null) - fl=$(echo "$hdr" | awk '/Flags:/{print $2}' | tr -d ,) - typ=$(echo "$hdr" | awk '/^ Type:/{print $2}') - want=0x70001405; [ "$typ" = "DYN" ] && want=0x70001407 - case "$hdr" in - *nan2008*o32*mips32r2*) ;; - *) echo "!! $f not nan2008/o32/mips32r2"; BAD=1; continue;; - esac - [ "$fl" = "$want" ] || { echo "!! $f type=$typ flags=$fl want=$want"; BAD=1; } - N=$((N+1)) -done < <(find "$SP" -type f -name '*.so') -[ $BAD -eq 0 ] || exit 1 -echo "$N mipsel extension modules, all nan2008/o32/mips32r2 DYN 0x70001407" +# NO ABI GATE HERE. This used to walk every .so in the site-packages tree and pin e_flags per ELF +# type -- 0x70001405 for an EXEC, 0x70001407 for a DYN. It was one of five +# implementations of that rule and the strictest of them, strict enough to be +# wrong: the low three bits are NOREORDER/PIC/CPIC and vary between objects of +# identical ABI, so an exact-word compare refuses files the kernel loads +# happily. qa/replica/test_abi.py asks the question once, over the installed +# filesystem, masking those bits off. echo echo "-- NEEDED of each extension module (nothing may want a /usr/prog soname) --" diff --git a/tools/python/README.md b/tools/python/README.md index 856caa7..8860898 100644 --- a/tools/python/README.md +++ b/tools/python/README.md @@ -57,7 +57,9 @@ binary file" -- and explains nothing. loader = /lib/ld-linux-mipsn8.so.1 rootfs glibc 2.33 Both words are correct and any ABI gate must accept both, or it fails on every -one of the 56 extension modules in the tree. `c_helper.so` reads `0x70001407` +one of the 56 extension modules in the tree. `qa/replica/test_abi.py` -- the +only ABI gate there is now -- masks the low three bits off for exactly this +reason. `c_helper.so` reads `0x70001407` too, for the same reason. **`-EL -mnan=2008` are mandatory in CFLAGS *and* in LDFLAGS.** This toolchain diff --git a/tools/python/build.sh b/tools/python/build.sh index 1789df5..3636527 100755 --- a/tools/python/build.sh +++ b/tools/python/build.sh @@ -328,29 +328,13 @@ rm -rf "$STAGE" make install DESTDIR="$STAGE" 2>&1 | tail -20 # ============================================ 4. gates + package ============ -log "ABI gate on every ELF in the stage" -# The measured target word is 0x70001405 for an EXECUTABLE. Shared objects -# additionally carry EF_MIPS_PIC (0x2) and so read 0x70001407 -- that is -# correct and unavoidable for a DYN, and klippy's own c_helper.so has it too. -# So gate on the ABI ATTRIBUTES, which is also what bin/patch.sh does -# (`readelf -h | grep -q nan2008`), and pin the whole word per ELF type. -BAD=0; N=0 -while IFS= read -r f; do - case "$(file -b "$f")" in *ELF*) ;; *) continue;; esac - hdr=$(mips-linux-gnu-readelf -h "$f" 2>/dev/null) - fl=$(echo "$hdr" | awk '/Flags:/{print $2}' | tr -d ,) - typ=$(echo "$hdr" | awk '/^ Type:/{print $2}') - want=0x70001405; [ "$typ" = "DYN" ] && want=0x70001407 - case "$hdr" in - *nan2008*o32*mips32r2*) ;; - *) echo "!! $f not nan2008/o32/mips32r2"; BAD=1; continue;; - esac - if [ "$fl" != "$want" ]; then - echo "!! $f type=$typ flags=$fl want=$want"; BAD=1 - fi - N=$((N+1)) -done < <(find "$STAGE" -type f) -[ $BAD -eq 0 ] && echo "$N ELF objects, all nan2008/o32/mips32r2 (EXEC 0x70001405, DYN 0x70001407)" || exit 1 +# NO ABI GATE HERE. This used to walk every ELF in the install stage and pin e_flags per ELF +# type -- 0x70001405 for an EXEC, 0x70001407 for a DYN. It was one of five +# implementations of that rule and the strictest of them, strict enough to be +# wrong: the low three bits are NOREORDER/PIC/CPIC and vary between objects of +# identical ABI, so an exact-word compare refuses files the kernel loads +# happily. qa/replica/test_abi.py asks the question once, over the installed +# filesystem, masking those bits off. # _sqlite3 is the reason this build exists. Its absence must be a hard error, # not a line in a 400-line make log that nobody reads.