diff --git a/.claude/skills/packages/SKILL.md b/.claude/skills/packages/SKILL.md index 869bec0..52d382b 100644 --- a/.claude/skills/packages/SKILL.md +++ b/.claude/skills/packages/SKILL.md @@ -73,6 +73,14 @@ pkg_end | `pkg_ship ...` | copy from staging into the package; globs are relative to the prefix | | `pkg_end` | seal the cache | +Three more exist only for python packages: + +| verb | what it does | +| --- | --- | +| `pkg_buildpython` | build (once, cached) an x86-64 CPython of the same version and export `$HOSTPY` | +| `pkg_pytarget` | make that interpreter's sysconfig answer for mipsel, using the headers `pkg_deps` unpacked | +| `pkg_pywheel ` | build one wheel and unpack it into the staging site-packages | + Exactly one source verb per recipe — `pkg_unpack` **or** `pkg_intree`. ## When `pkg_build` needs help @@ -87,6 +95,7 @@ Set these before calling it. Do not run a configure script yourself. | `PKG_INSTALL_TARGET` | `install` | `install_sw`, or `none` to place files yourself | | `PKG_MAKE_ARGS` | — | `LDLIBS=-lpthread` (s6) | | `PKG_STRIP_ARGS` | `--strip-unneeded` | `""` to strip executables fully | +| `PKG_PY_SETUP_ARGS` | — | drive `setup.py build_ext` instead of pip (pillow) | `pkg_build` returns non-zero rather than dying, so `if ! pkg_build ...` can retry with different flags. See `pkg/openssl/build.sh`. @@ -116,6 +125,23 @@ retry with different flags. See `pkg/openssl/build.sh`. - no compiler, just files — `pkg/moonraker` - from this repo, not a download — `pkg/anvil-core` - awkward build — `pkg/openssl`, `pkg/bzip2`, `pkg/zlib` +- a python package — `pkg/python-distro` (pure), `pkg/python-cffi` (native) + +## Adding a python package + +Four steps, and the first two are not in `pkg/`: + +1. `PYPKG__FILE` and `_SHA256` in `versions.env`, and the name on + `PYPKG_LIST`. The version is read out of the file name. +2. `./bin/fetch-assets.sh` to download it. +3. `pkg/python-/`, copied from `pkg/python-distro`. Name any other + package it imports in `PKG_DEPENDS`; add `PKG_ARCH=all` if it compiles + nothing. +4. `make packages PKG=python-`. + +The list in `versions.env` and the recipe directories are checked against each +other by `qa/static`, so a half-done addition fails there rather than on a +printer. ## Checking diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3042216..ece4bd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,9 +79,9 @@ jobs: # (see bin/fetch-assets.sh) -- so this job builds them for real rather # than stubbing around them the way it stubs Mainsail, HelixScreen and # the Klipper fork. Cached on versions.env: an unchanged pin never - # re-fetches the ~203MB Ingenic toolchain, and - # never recompiles either tree. work/.py313 carries its own version - # stamp; s6 is four recipes under work/pkg and carries pkg_stamp's. + # re-fetches the ~203MB Ingenic toolchain and never recompiles either + # tree. Both are recipes under work/pkg now -- four for s6, nineteen for + # CPython and its site-packages -- and every one carries pkg_stamp's key. # # work/pkg is every recipe under pkg/ at once -- one cache entry instead # of one per package, because pkg/lib.sh derives the output directory @@ -93,7 +93,6 @@ jobs: with: path: | vendor - work/.py313 work/pkg key: fixture-py-${{ hashFiles('versions.env') }} diff --git a/Makefile b/Makefile index d67e63f..a0558a4 100644 --- a/Makefile +++ b/Makefile @@ -357,19 +357,35 @@ test-nginx: image work/.s6-gate.tgz test-camera: image work/.s6-gate.tgz @$(RUNSIM) ./test/integration/printer-exec.py ./test/integration/printer/case-camera.sh sup.tgz=work/.s6-gate.tgz +# NINETEEN RECIPE OUTPUTS, MERGED, exactly as work/.s6-gate.tgz merges three. +# This used to be `tar -czf -C work/.py313 bin lib`: one directory, because +# bin/patch.sh cross-built the interpreter and its site-packages into one +# cache. CPython is pkg/python now and each third-party package is a +# pkg/python-* of its own, so what a printer sees is the union of their bin/ +# and lib/ -- which is what bin/patch.sh section 5c stages and what +# test/ffsim/gates.py packs for the suite. +work/.py-gate.tgz: FORCE + @rm -rf work/.py-gate && mkdir -p work/.py-gate + @[ -d work/pkg/python ] || { echo "!! work/pkg/python is missing -- run ./bin/patch.sh first" >&2; exit 1; } + @for t in work/pkg/python work/pkg/python-*; do \ + [ -d $$t ] || continue; \ + cp -a $$t/. work/.py-gate/; \ + done + @rm -f work/.py-gate/.version + @tar -czf $@ -C work/.py-gate bin lib + # The CPython 3.13 the build cross-compiles, on the printer's own kernel. -# Needs work/.py313, which bin/patch.sh fills -- the same relationship -# test-supervisor has to work/.s6. Nothing on the printer RUNS this -# interpreter yet, which is exactly why it has a gate of its own: see the -# header of case-python.sh. -test-python: image - @tar -czf work/.py-gate.tgz -C work/.py313 bin lib +# Needs the recipe outputs under work/pkg, which bin/patch.sh fills -- the same +# relationship test-supervisor has to work/pkg/s6. See the header of +# case-python.sh for why this has a gate of its own. +test-python: image work/.py-gate.tgz @$(RUNSIM) ./test/integration/printer-exec.py ./test/integration/printer/case-python.sh py.tgz=work/.py-gate.tgz # The real Moonraker, on the 3.13 we built, under the s6 we built, started by # init.d/S62moonraker. Three build outputs and no stand-ins, which is why it -# needs three tarballs where every other target needs one or none: work/.s6, -# work/.py313 + work/.sodium, and the pinned Moonraker sdist in vendor/. Two of +# needs three tarballs where every other target needs one or none: the s6 +# recipes, the python recipes + work/.sodium, and the pinned Moonraker sdist in +# vendor/. Two of # the three are assembled by test/ffsim/gates.py rather than by a `tar -czf` # here, so this runs the gate through a thin wrapper instead of calling # printer-exec.py -- see sim-moonraker313.py for that argument written out. diff --git a/bin/build-packages.sh b/bin/build-packages.sh index 303ecd6..46ae045 100755 --- a/bin/build-packages.sh +++ b/bin/build-packages.sh @@ -212,6 +212,14 @@ for r in "${RECIPES[@]}"; do # be unindented, and an unindented line is where the next stanza begins. emit() { _lay=$1; _nm=$2; _sect=$3; _dep=$4; _desc=$5 + # ONE LINE, WHATEVER THE RECIPE WROTE. A Depends list of thirteen + # packages does not fit on one line of a pkg.conf anybody wants to + # read, so anvil-moonraker's is wrapped -- and a raw newline inside + # a control field is a new stanza to some parsers and a folded + # continuation to others. Unquoted, so the shell splits on every + # run of whitespace and rejoins with single spaces. + # shellcheck disable=SC2086 + _dep=$(echo $_dep) { printf 'Package: %s\n' "$_nm" printf 'Version: %s-%s\n' "$PKG_VERSION" "$PKG_RELEASE" diff --git a/bin/common.sh b/bin/common.sh index 861ccee..2e0330e 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -153,10 +153,11 @@ export SKALIBS_TGZ S6_TGZ EXECLINE_TGZ S6RC_TGZ # interpreter cannot dlopen a glibc c_helper.so, and dlopen is exactly how # klippy loads it. See tools/python/README.md. # -# PY_BUILD is patch.sh's cache of the cross-built, trimmed tree -- the same -# thing S6_BUILD is for s6, and named here for the same reason: it is what -# decides whether anything has to be compiled at all, and both patch.sh and -# fetch-assets.sh need that answer. +# THERE IS NO PY_BUILD ANY MORE, and its absence is the point. It named +# work/.py313, patch.sh's private cache of a cross-built interpreter, and it +# went the way S6_BUILD went: CPython is pkg/python and its eighteen packages +# are pkg/python-*, so the cache is work/pkg/ and pkg_out derives that +# name from the recipe. Nothing has to be spelled here for a script to find it. PY_TGZ="${PY_TGZ:-$ROOT/vendor/Python-${PY_VERSION:-unpinned}.tgz}" OPENSSL_TGZ="${OPENSSL_TGZ:-$ROOT/vendor/openssl-${OPENSSL_VERSION:-unpinned}.tar.gz}" SQLITE_TGZ="${SQLITE_TGZ:-$ROOT/vendor/sqlite-autoconf-${SQLITE_VERSION:-unpinned}.tar.gz}" @@ -165,26 +166,31 @@ LIBFFI_TGZ="${LIBFFI_TGZ:-$ROOT/vendor/libffi-${LIBFFI_VERSION:-unpinned}.tar.gz XZ_TGZ="${XZ_TGZ:-$ROOT/vendor/xz-${XZ_VERSION:-unpinned}.tar.gz}" BZIP2_TGZ="${BZIP2_TGZ:-$ROOT/vendor/bzip2-${BZIP2_VERSION:-unpinned}.tar.gz}" EXPAT_TGZ="${EXPAT_TGZ:-$ROOT/vendor/expat-${EXPAT_VERSION:-unpinned}.tar.gz}" -PY_BUILD="${PY_BUILD:-$ROOT/work/.py313}" -# The cache key for work/.py313, spelled ONCE, here, for exactly the reason -# S6_STAMP had to be and then stopped needing to be -- see the note above. +# The ABI series CPython names its own directories and binaries after -- +# bin/python3.13, lib/python3.13/, config-3.13-mipsel-linux-gnu. It is spelled +# out rather than sed'd out of PY_VERSION because it is not a substring of it +# in any interesting sense, and a 3.14 bump has to be a deliberate edit. # -# It was spelled THREE times: bin/patch.sh section 5c wrote it, and -# bin/fetch-assets.sh derived it twice, in two different places, to decide -# whether the 203MB Ingenic toolchain had to come down. All three happened to -# agree, which is luck rather than design: the eight fields are the eight -# source pins of the interpreter and its static libraries, and the next person -# to add a ninth would have had to find all three. That is the same shape as -# the s6 stamp bug, caught before it inverted rather than after. +# HERE rather than in bin/patch.sh, where it used to live, because pkg/python +# and the eighteen pkg/python-* recipes all name these directories too. A +# constant that three scripts need is not patch.sh's local variable. +PY_MM="${PY_MM:-3.13}" +export PY_MM + +# THE CACHE KEY FOR CPYTHON IS GONE FROM THIS FILE TOO, and that is the end of +# a bug that took three tries to kill. PY_STAMP was "$PY_VERSION plus the seven +# library versions", written by bin/patch.sh and derived AGAIN, twice, in +# bin/fetch-assets.sh to decide whether the 203MB Ingenic toolchain had to come +# down. All three happened to agree, which is luck: the next person to add an +# eighth library would have had to find all three. # -# It goes away entirely when CPython becomes a recipe: pkg_stamp derives a key -# from the dependency graph, so "openssl, sqlite, zlib, libffi, xz, bzip2 and -# expat" stops being a list anybody maintains and becomes PKG_BUILD_DEPENDS. -PY_STAMP="$PY_VERSION $OPENSSL_VERSION $SQLITE_VERSION $ZLIB_VERSION $LIBFFI_VERSION $XZ_VERSION $BZIP2_VERSION $EXPAT_VERSION" -export PY_STAMP +# pkg_stamp derives a recipe's key from its dependency graph, so "openssl, +# sqlite, zlib, libffi, xz, bzip2 and expat" is PKG_BUILD_DEPENDS in +# pkg/python/pkg.conf and nothing computes it a second time. bin/fetch-assets.sh +# asks pkg_needs, which is the same code the recipes cache on. export PY_TGZ OPENSSL_TGZ SQLITE_TGZ ZLIB_TGZ LIBFFI_TGZ XZ_TGZ BZIP2_TGZ -export EXPAT_TGZ PY_BUILD +export EXPAT_TGZ # The third-party python packages that become the interpreter's # site-packages, and libsodium, which libnacl dlopens out of $MODDIR/lib. @@ -211,15 +217,21 @@ pypkg_var() { } # The sdist as bin/fetch-assets.sh leaves it in vendor/. pypkg_tgz() { printf '%s/vendor/%s' "$ROOT" "$(pypkg_var "$1" FILE)"; } -# The cache key for the whole package set: every file name (which carries the -# version) beside its hash, so a bumped pin AND a re-released upstream -# tarball both invalidate. Compared as a whole string, not hashed, so -# work/.py313/.pkg-version stays readable -- a diff of it is the changelog. -pypkg_stamp() { - local _p - for _p in $PYPKG_LIST $PYPKG_HOST_LIST; do - printf '%s %s\n' "$(pypkg_var "$_p" FILE)" "$(pypkg_var "$_p" SHA256)" - done +# The version, READ OUT OF THE PINNED FILE NAME rather than pinned a second +# time beside it. Every sdist on PyPI is -.tar.gz and the +# version is the part after the LAST dash, which is what makes this safe for +# the entries whose file name is not their list name: inotify_simple-1.3.5, +# streaming_form_data-1.19.1, pyserial-asyncio-0.6. +# +# A PYPKG__VERSION line per package would be eighteen more strings to +# keep in agreement with eighteen FILE lines, and the failure when they +# disagreed would be a package whose .ipk claims a version it does not +# contain. One pin, read two ways. +pypkg_version() { + local _f + _f="$(pypkg_var "$1" FILE)" + _f=${_f%.tar.gz}; _f=${_f%.tgz}; _f=${_f%.zip} + printf '%s' "${_f##*-}" } # The sources every recipe under pkg/ builds from. ZLIB_TGZ is NOT repeated # here: it is pinned thirty lines up for CPython, and pkg/zlib builds that same diff --git a/bin/fetch-assets.sh b/bin/fetch-assets.sh index e39b83e..956265e 100755 --- a/bin/fetch-assets.sh +++ b/bin/fetch-assets.sh @@ -93,21 +93,17 @@ fi # cross-built on every checkout. Gating this fetch on BUILD_KLIPPER=fork alone # -- the shape it had before 5c existed -- means a BUILD_KLIPPER=stock build # from a clean vendor/ reaches patch.sh's Python step with sources but no -# compiler and dies there instead of here. Checked against patch.sh's own -# work/.py313 stamp, so a checkout with a current cross-built interpreter -# never pulls this compiler either. +# compiler and dies there instead of here. # -# THE RECIPES UNDER pkg/ ARE THE THIRD CONSUMER, and pkg_needs is how they are -# asked. Every one of them cross-compiles with this toolchain, so a checkout -# with a stale work/pkg/zlib needs the compiler even when the interpreter and -# the Klipper fork are both current. Asking pkg/lib.sh rather than comparing a -# stamp string here is the point: the recipe and the fetcher then compute the -# cache key with the same code instead of two spellings that can drift -- which -# is exactly what went wrong with the s6 stamp below. +# TWO CONDITIONS, WHERE THERE USED TO BE FOUR. The interpreter and its eighteen +# site-packages are recipes under pkg/ now, and pkg_needs answers for every +# recipe at once -- so the two hand-written CPython stamp comparisons that sat +# here were the same question in a second spelling, and the second spelling is +# the one that drifts. Asking pkg/lib.sh means the fetcher computes the cache +# key with the code that writes it, which is exactly what went wrong with the +# s6 stamp below. if [ "$ALL" = 1 ] \ || { [ "${BUILD_KLIPPER:-0}" = "fork" ] && [ ! -d "${KLIPPER_FORK:-}/klippy" ]; } \ - || [ "$(cat "$PY_BUILD/.version" 2>/dev/null || true)" != "$PY_STAMP" ] \ - || [ "$(cat "$PY_BUILD/.pkg-version" 2>/dev/null || true)" != "$(pypkg_stamp)" ] \ || pkg_needs; then get "https://github.com/ballaswag/k1-discovery/releases/download/$MIPS_TOOLCHAIN_VERSION/$MIPS_TOOLCHAIN_FILE" \ "$MIPS_TOOLCHAIN_TGZ" "$MIPS_TOOLCHAIN_SHA256" @@ -264,28 +260,22 @@ fi say "cached opkg-utils $OPKG_UTILS_VERSION ($OPKG_UTILS_COMMIT)" # The Ingenic toolchain, on the condition that decides whether patch.sh has to -# compile at all. THREE stamps and not one, because three separate things are -# built with this one compiler and any of them going stale means the download -# is needed: the interpreter (work/.py313/.version), the packages that go into -# its site-packages (work/.py313/.pkg-version -- same cache directory, its own -# key, because a package bump forces a full interpreter rebuild; see the -# comment in bin/patch.sh) and every recipe under pkg/, which pkg_needs answers -# for all of them at once. +# compile at all. ONE question, asked of pkg_needs, because everything built +# with this compiler is a recipe under pkg/ now -- the interpreter, its +# eighteen site-packages, libsodium, the s6 family and the rest. # -# libsodium USED TO BE SPELLED OUT HERE, as a comparison against -# $SODIUM_VERSION. It is one of the recipes pkg_needs covers now, and the -# comparison had already gone subtly wrong: a recipe's cache key includes the -# toolchain and its dependencies, so a bare version string stopped being the -# thing written into work/pkg/libsodium/.version and the test would have been -# true forever. Asking the code that writes the stamp is the only spelling that -# cannot drift -- which is the same lesson as the s6 stamp above. +# IT USED TO BE THREE HAND-WRITTEN COMPARISONS and each of them had gone, or was +# going, wrong. libsodium's compared $SODIUM_VERSION against +# work/pkg/libsodium/.version, which stopped matching the moment a recipe's key +# grew to include its toolchain and its dependencies -- so the test was true +# forever and nobody noticed. The two CPython ones re-derived a stamp +# bin/patch.sh also derived, and agreed with it by luck. Asking the code that +# WRITES the stamp is the only spelling that cannot drift. # # Getting this wrong is not a slow build, it is a stopped one: patch.sh would # get 53MB of sdists and then halt for want of a compiler, halfway through, # rather than here where the fix is one command. if [ "$ALL" = 1 ] \ - || [ "$(cat "$PY_BUILD/.version" 2>/dev/null || true)" != "$PY_STAMP" ] \ - || [ "$(cat "$PY_BUILD/.pkg-version" 2>/dev/null || true)" != "$(pypkg_stamp)" ] \ || pkg_needs; then get "https://github.com/ballaswag/k1-discovery/releases/download/$MIPS_TOOLCHAIN_VERSION/$MIPS_TOOLCHAIN_FILE" \ "$MIPS_TOOLCHAIN_TGZ" "$MIPS_TOOLCHAIN_SHA256" diff --git a/bin/patch.sh b/bin/patch.sh index c993f11..89f6720 100755 --- a/bin/patch.sh +++ b/bin/patch.sh @@ -396,828 +396,118 @@ du -sh "$MOD_PAYLOAD/bin" | awk '{print " "$1"\tbin/"}' du -sh "$MOD_PAYLOAD/libexec" | awk '{print " "$1"\tlibexec/"}' # -------------------------------------------------- 5c. CPython 3.13 (shipped) -# A second Python for the printer, cross-compiled here from the sources pinned -# in versions.env with the SAME Ingenic glibc toolchain that builds c_helper.so -# above. Same shape as s6 in 5b -- pinned tarballs, compiled in place, cached -# on a version stamp, gated on the tree that ships -- and for the same reason: -# nothing binary is vendored, so anything the printer executes is built from a -# hash we pinned. +# A second Python for the printer. pkg/python builds the interpreter and the +# eighteen pkg/python-* recipes build what goes in its site-packages -- one +# recipe and one .ipk each. This section runs them and stages what they +# produce. Everything about HOW they are built lives with them now +# (pkg/python/build.sh, and pkg_buildpython / pkg_pytarget / pkg_pywheel in +# pkg/lib.sh); every reason WHY is in the pkg.conf beside each one. # # ############################################################################ -# # FF_PYTHON POINTS HERE NOW. payload/anvil-env.sh names this interpreter # -# # for Moonraker, ff-startup.py, ffscreen.py and ff_mcu_bringup.py. Every # -# # third-party C extension those need -- tornado, lmdb, cffi, greenlet, # -# # libnacl -- is cross-built into $MODDIR/lib/python3.13/site-packages by # -# # step 4 below, and Moonraker has been measured SERVING on this # -# # interpreter through the real boot path on the replica # -# # (test/integration/printer/case-moonraker313-s6.sh): S40s6's scandir, # -# # S62moonraker, readiness gating on :7125 actually listening, a kill -9 # -# # respawn, and a stop that stays stopped. # +# # FF_PYTHON POINTS HERE. payload/anvil-env.sh names this interpreter for # +# # Moonraker, ff-startup.py, ffscreen.py and ff_mcu_bringup.py, and every # +# # third-party C extension those need is one of the packages below. # +# # Moonraker has been measured SERVING on it through the real boot path on # +# # the replica (test/integration/printer/case-moonraker313-s6.sh): S40s6's # +# # scandir, S62moonraker, readiness gating on :7125 actually listening, a # +# # kill -9 respawn, and a stop that stays stopped. # # # # # # klippy is NOT among FF_PYTHON's callers and does not run on this # # # interpreter -- it is started separately, by FlashForge's own # # # /usr/prog/klipper/start.sh, hardcoded to 3.8.2 (see init.d/S70klipper). # # # klippy's numpy gap is therefore a separate, smaller item, not a # -# # precondition of this switch. # +# # precondition of this switch. # # ############################################################################ # -# WHY IT IS WORTH SHIPPING. FlashForge built 3.8.2 without _sqlite3, and -# that single omission is what pins MOONRAKER_VERSION to a 2023 commit: every -# Moonraker from v0.9.0 on keeps its database in sqlite. This interpreter has -# a working sqlite3 (measured on the replica, create/insert/select/reopen -- +# WHY IT IS WORTH SHIPPING. FlashForge built 3.8.2 without _sqlite3, and that +# single omission is what pins MOONRAKER_VERSION to a 2023 commit: every +# Moonraker from v0.9.0 on keeps its database in sqlite. This interpreter has a +# working sqlite3 (measured on the replica, create/insert/select/reopen -- # test/integration/printer/case-python.sh), which is what eventually unpins it. # -# WHERE IT LANDS. $MODDIR is a --prefix root -- that is the whole reason s6 was -# configured for it, and it is why the mod's tree has bin/, lib/, libexec/ and -# etc/ directly inside it rather than a directory per package. The interpreter -# is configured --prefix=$MODDIR like everything else, so `make install` puts -# bin/python3.13 beside bin/s6-svscan and lib/python3.13/ beside lib/. One -# prefix, one library path, one place to look. -# -# That path is COMPILED IN (sys.prefix, and the stdlib search that follows from -# it), so moving the directory on the printer breaks the interpreter exactly -# the way moving s6 breaks its waiting verbs -- it has to be rebuilt, not -# renamed. Which is also the argument for putting it here and not somewhere -# private: there is only one prefix to keep stable, not two. -# -# WHAT IS DELETED OUT OF bin/ BEFORE STAGING, and why it is not fussiness. -# `make install` also creates bin/python3 -- a symlink to python3.13 -- plus -# idle3, pydoc3 and the *-config scripts. anvil-env.sh PREPENDS $MODDIR/bin to -# PATH (s6-svscan execs s6-supervise by name off PATH, so it has to), so a -# bin/python3 of ours would sit AHEAD of FlashForge's on the PATH of every -# process that sources it. Anything that says `python3` rather than -# "$FF_PYTHON" -- a hand-typed ssh command, a subprocess in someone's -# component, a shebang -- would silently change interpreter. That is the -# accidental version of the switch the box above says must not happen yet, so -# the symlink does not ship. The name python3.13 is unambiguous and nothing on -# this printer answers to it. idle3 and pydoc3 go with it because idlelib and -# the config directory they need are trimmed away anyway. -# -# PY_MM is spelled out rather than derived from PY_VERSION with a sed, because -# it is not a substring of it in any interesting sense: it is the ABI series -# CPython names its own directories and binaries after, and a 3.14 bump has to -# be a deliberate edit of both lines. -PY_MM="3.13" -PY_PREFIX="$MODDIR" -# PY_HOST and PY_TOOLCHAIN_DIR come from bin/common.sh for the same reason -# MODDIR does -- pkg/libsodium/build.sh compiles with the same toolchain and -# must not carry its own spelling of where it is. -# The cache key is every version that goes into the tree, not just CPython's: -# a bumped OpenSSL with an unchanged PY_VERSION has to rebuild, and the failure -# if it does not is an interpreter linked against a library nobody can name. -# The SECOND cache key, for the third-party packages that go into this -# interpreter's site-packages (step 4 below): every sdist file name and hash -# from versions.env, as bin/common.sh assembles it. -# -# TWO STAMPS, ONE CACHE DIRECTORY, AND A REBUILD OF EVERYTHING IF EITHER -# MOVES. That is not sloppiness, it is the shape of the dependency. Building a -# package for this target needs three things that exist ONLY while this -# section is mid-flight: the untrimmed staged tree (the trim below deletes -# include/ and lib/python3.13/config-3.13-*, and INCLUDEPY points at the -# first), the static C libraries in work/.py-dep (pillow links zlib, cffi -# links libffi), and the x86-64 build-python that answers every sysconfig -# question for mipsel. Caching those separately means caching a second -# ~200MB tree and a second interpreter for the sake of a package bump, which -# buys a faster rare path by making the common path bigger. So a package pin -# that moves rebuilds CPython too: minutes, on a bump that happens rarely and -# deliberately, against a cache that is otherwise never touched. -PYPKG_STAMP="$(pypkg_stamp)" - -if [ "$(cat "$PY_BUILD/.version" 2>/dev/null || true)" != "$PY_STAMP" ] \ - || [ "$(cat "$PY_BUILD/.pkg-version" 2>/dev/null || true)" != "$PYPKG_STAMP" ]; then - if [ ! -x "$PY_TOOLCHAIN_DIR/bin/$PY_HOST-gcc" ]; then - [ -f "${MIPS_TOOLCHAIN_TGZ:-}" ] || { - echo " !! python needs (re)building and there is no toolchain:" >&2 - echo " $MIPS_TOOLCHAIN_TGZ is missing. Run ./bin/fetch-assets.sh." >&2 - exit 1; } - say "python: unpacking the Ingenic MIPS toolchain" - mkdir -p work/.mips-toolchain - tar -xzf "$MIPS_TOOLCHAIN_TGZ" -C work/.mips-toolchain - fi - for t in "${PY_TGZ:-}" "${OPENSSL_TGZ:-}" "${SQLITE_TGZ:-}" "${ZLIB_TGZ:-}" \ - "${LIBFFI_TGZ:-}" "${XZ_TGZ:-}" "${BZIP2_TGZ:-}" "${EXPAT_TGZ:-}"; do - [ -f "$t" ] || { echo " !! no python sources at '$t' -- run ./bin/fetch-assets.sh" >&2; exit 1; } - done - # And the package sdists, checked here rather than in step 4 -- twenty - # minutes into a CPython build is the wrong place to discover that - # vendor/ is one tarball short, and the fix is the same one command. - for p in $PYPKG_LIST $PYPKG_HOST_LIST; do - t="$(pypkg_tgz "$p")" - [ -f "$t" ] || { echo " !! no sdist for $p at '$t' -- run ./bin/fetch-assets.sh" >&2; exit 1; } - done - command -v gcc >/dev/null || { - echo " !! no host gcc. Cross-building CPython needs a build-python of" >&2 - echo " the SAME version, which is compiled here for x86-64 first." >&2 - echo " Run through 'make build' -- docker/Dockerfile.build has it." >&2 +# NINETEEN RECIPES RATHER THAN THE 800-LINE SECTION THAT USED TO BE HERE, and +# what that buys is not tidiness. It is that a package pin can move without +# rebuilding CPython. The old arrangement had one cache directory and two +# stamps and said so out loud: a bumped Pillow rebuilt the interpreter and all +# eighteen packages, because a wheel could only be cross-built during the few +# minutes when an untrimmed staging tree, a private sysroot of static libraries +# and a throwaway x86-64 build-python all happened to exist at once. Those +# three are now the feed's anvil-python-dev package, each recipe's own sysroot, +# and pkg_buildpython's shared cache -- none of which is a passing moment. So +# `make packages PKG=python-pillow` is a Pillow build, and nothing else. +say "python: CPython $PY_VERSION and $(echo $PYPKG_LIST | wc -w) packages" +bash pkg/python/build.sh +# PYPKG_LIST IS STILL THE LIST, and it is checked against the recipes rather +# than trusted: versions.env carries the pins and bin/fetch-assets.sh downloads +# from the same list, so an entry added there and nowhere else would otherwise +# be fetched, hashed and silently never built. +for p in $PYPKG_LIST; do + [ -d "pkg/python-$p" ] || { + echo " !! PYPKG_LIST names '$p' and there is no pkg/python-$p recipe" >&2 exit 1; } - say "python: cross-compiling CPython $PY_VERSION + 7 libraries for $PY_PREFIX" - say " (a few minutes, once -- work/.py313 caches it after that)" - rm -rf work/.py-src work/.py-dep work/.py-host work/.py-stage work/.py-xw "$PY_BUILD" - mkdir -p work/.py-src work/.py-dep work/.py-host work/.py-xw/bin - for t in "$PY_TGZ" "$OPENSSL_TGZ" "$SQLITE_TGZ" "$ZLIB_TGZ" \ - "$LIBFFI_TGZ" "$XZ_TGZ" "$BZIP2_TGZ" "$EXPAT_TGZ"; do - tar -xzf "$t" -C work/.py-src - done - # CPython twice, into two directories. The host build leaves its own - # Makefile, config.status, pyconfig.h and .o files behind, and a cross - # ./configure run on top of that inherits host answers -- the resulting - # interpreter is x86 in places and mipsel in others, and it fails at - # `make` in a way that reads like a compiler bug. Two trees, no sharing. - mkdir -p work/.py-src/cross - tar -xzf "$PY_TGZ" -C work/.py-src/cross - ( - # A subshell, as in 5b: the cross-compiler's CC/CFLAGS/PATH must not - # leak into anything patch.sh does afterwards -- and unlike s6 this - # section also exports LIBS and CONFIG_SITE, which would wreck a later - # ./configure in ways that are very hard to read. - set -e - SRC="$PWD/work/.py-src" - DEP="$PWD/work/.py-dep" # cross-built C libraries, STATIC. Not shipped. - HOSTPY="$PWD/work/.py-host" # the x86-64 build-python - STAGE="$PWD/work/.py-stage" # DESTDIR - TC="$PWD/$PY_TOOLCHAIN_DIR" - XW="$PWD/work/.py-xw" - # Absolute, because everything below runs from inside a source tree. - # A relative log path here means eight ./configure runs each writing a - # log into their own directory, which is then deleted with them -- - # i.e. no log at all on the one run where you want one. - LOG="$PWD/work" - JOBS=$(nproc 2>/dev/null || echo 4) + bash "pkg/python-$p/build.sh" +done - # ---------------------------------------------------------- wrappers - # THE SINGLE MOST IMPORTANT THING IN THIS SECTION. `-EL -mnan=2008` - # has to reach the COMPILE and the LINK of every object: this - # toolchain defaults to big-endian legacy-NaN, and the printer's - # kernel refuses anything that is not little-endian NAN2008 -- it - # says ENOEXEC and explains nothing. Passing them in CFLAGS is not - # enough, because several of the eight projects below do not forward - # CFLAGS to their link line. So the two flags are baked into the gcc - # driver itself by a wrapper on PATH, and no build system gets a vote. - for t in gcc g++ cpp; do - printf '#!/bin/sh\nexec %s/bin/%s-%s -EL -mnan=2008 "$@"\n' \ - "$TC" "$PY_HOST" "$t" > "$XW/bin/$PY_HOST-$t" - chmod +x "$XW/bin/$PY_HOST-$t" - done - # The binutils have no flags to bake in, so they are plain symlinks -- - # but they must be on the SAME PATH entry, or configure finds the - # host's ar/ranlib and produces archives the cross-linker cannot read. - for t in ar as ld nm objcopy objdump ranlib readelf strip strings size; do - ln -sf "$TC/bin/$PY_HOST-$t" "$XW/bin/$PY_HOST-$t" - done - export PATH="$XW/bin:$PATH" +# Staged into the SAME bin/ and lib/ as everything else in this prefix root, +# which is why these copy the CONTENTS of the directories and not the +# directories: s6's binaries are already in $MOD_PAYLOAD/bin and must stay +# there, and eighteen packages all merge into one site-packages. +cp -a "$(pkg_out python)/bin/." "$MOD_PAYLOAD/bin/" +mkdir -p "$MOD_PAYLOAD/lib" +cp -a "$(pkg_out python)/lib/." "$MOD_PAYLOAD/lib/" +for p in $PYPKG_LIST; do + cp -a "$(pkg_out "python-$p")/lib/." "$MOD_PAYLOAD/lib/" +done +chmod +x "$MOD_PAYLOAD/bin/python$PY_MM" - # Gate the wrapper before building 300MB on top of it. A wrapper that - # silently lost its flags produces a tree that builds perfectly and is - # ENOEXEC on the printer, and the ship gate below would then be the - # first thing to notice -- twenty minutes later. - echo 'int main(void){return 0;}' > "$SRC/abi.c" - "$PY_HOST-gcc" "$SRC/abi.c" -o "$SRC/abi.out" - abi=$("$PY_HOST-readelf" -h "$SRC/abi.out" | awk '/Flags:/{print $2}' | tr -d ,) - [ "$abi" = "0x70001405" ] || { - echo " !! the toolchain wrapper produces e_flags=$abi, want 0x70001405" >&2 - exit 1; } +# THE DEV HALF DOES NOT SHIP. work/pkg/python is the WHOLE build: the split +# into anvil-python and anvil-python-dev happens where the .ipk files are made, +# so the tree just copied from still carries the headers and the build +# configuration that only a build machine opens. Which files those are is +# pkg/python/pkg.conf's business and is read from there -- a second list here +# would be the same set of paths spelled twice, and the half that got forgotten +# would be 3MB of headers on every printer. +# +# A SUBSHELL because pkg_conf sets PKG_* for whatever it read last, and this +# file goes on to stage four more recipes after this one. +( pkg_conf python + # shellcheck disable=SC2086 + for g in $PKG_DEV_FILES; do rm -rf "$MOD_PAYLOAD"/$g; done ) - # ------------------------------------------------- 1. the build-python - # Cross-building CPython needs a build-python of the SAME version: - # the Makefile runs it to freeze modules, generate the deepfreeze - # sources and byte-compile the stdlib, and configure hard-errors when - # the version does not match. The image's python3.11 cannot stand in. - # x86-64, thrown away at the end of this section, never shipped. - # - # --with-ensurepip=install, where the CROSS build below has - # --without-ensurepip and must keep it. The two are not in tension: - # they are opposite answers to opposite questions. On the printer a - # pip would need a network and a compiler and has neither. HERE, pip - # is what builds the third-party packages in step 4 -- and it comes - # out of THIS tarball, whose sha256 is pinned, rather than off - # bootstrap.pypa.io the way the spike got it. That is the difference - # between a build machine that downloads and runs an unpinned - # get-pip.py and one that does not talk to anybody: everything below - # this line runs with --no-index against files bin/fetch-assets.sh - # already checked. - ( cd "$SRC/Python-$PY_VERSION" - ./configure --prefix="$HOSTPY" --with-ensurepip=install >"$LOG/.py-hostpy.log" 2>&1 - make -j"$JOBS" >>"$LOG/.py-hostpy.log" 2>&1 - make install >>"$LOG/.py-hostpy.log" 2>&1 ) - # zlib, asserted rather than assumed, because CPython's answer to a - # missing library is to record the module as absent and CARRY ON -- - # the same silence that hides a missing _sqlite3 in the cross build - # below, here on the host side. Every wheel is a zip and so is pip: - # ensurepip installs it out of a bundled .whl, and without zlib that - # fails with "can't decompress data" INSIDE a make install that - # reports success. The symptom then arrives three steps later as "No - # module named pip", nowhere near the cause. zlib1g-dev in - # docker/Dockerfile.build is the fix, and this is the sentence that - # says so. - "$HOSTPY/bin/python$PY_MM" -c 'import zlib' 2>/dev/null || { - echo " !! the build-python has no zlib module, so it cannot unpack" >&2 - echo " a single wheel. Install zlib1g-dev in the build image" >&2 - echo " (docker/Dockerfile.build has it) and delete work/.py313." >&2 - exit 1; } - # And pip itself, which --with-ensurepip=install above was for. Asked - # here and not in step 4 because `make install` runs ensurepip in a - # shell fragment that does not stop the install when it fails: without - # this line a broken bootstrap is reported by the FIRST PACKAGE to try - # to build, twenty lines of C library later. - "$HOSTPY/bin/python$PY_MM" -m pip --version >>"$LOG/.py-hostpy.log" 2>&1 || { - echo " !! the build-python has no pip, so nothing in step 4 can" >&2 - echo " be built. ensurepip failed inside 'make install' --" >&2 - echo " its traceback is in work/.py-hostpy.log." >&2 - exit 1; } - - # ------------------------------------------- 2. the C libraries, STATIC - # All seven go into $DEP as .a with -fPIC, and NONE of them ships. The - # interpreter and its extension modules link them in, so the tree on - # the printer has no .so of ours to find at runtime: no - # LD_LIBRARY_PATH to get right, no chance of picking up one of - # FlashForge's /usr/prog copies (which is a real hazard -- /usr/prog - # carries libffi.so.8 and the rootfs carries libffi.so.7), and nothing - # to version-skew. The cost is a few MB of duplicated libcrypto - # between _ssl.so and _hashlib.so, which is a good trade at 30MB. - export CC="$PY_HOST-gcc" CXX="$PY_HOST-g++" - export AR="$PY_HOST-ar" RANLIB="$PY_HOST-ranlib" STRIP="$PY_HOST-strip" - export CFLAGS="-O2 -fPIC -D_FILE_OFFSET_BITS=64" - export CPPFLAGS="-I$DEP/include" - export LDFLAGS="-L$DEP/lib" - # pkg-config must see ONLY our sysroot. Left alone it answers out of - # the build image's /usr/lib/pkgconfig, and configure then links an - # x86-64 .so into a mipsel interpreter -- or, more often, decides a - # module is buildable and finds out at `make` time that it is not. - export PKG_CONFIG_LIBDIR="$DEP/lib/pkgconfig" - export PKG_CONFIG_PATH="$DEP/lib/pkgconfig" - BUILDTRIPLE=x86_64-linux-gnu - - # zlib is a PACKAGE now, and this is the second CONSUMER of one build - # rather than a second build of one source. It used to be three lines - # of ./configure here and another three in pkg/opkg/build.sh: the same - # pinned tarball, the same flags, compiled twice, with neither copy - # able to see the other because each was a private detail of whatever - # needed it. pkg/zlib/build.sh is now the only place it is compiled, - # and pkg/libarchive names it as a build dependency the same way this - # does. See docs/notes/85-packaging.md. - # - # RUN WITH THE CROSS ENVIRONMENT REMOVED. Everything exported above - # points at $DEP -- CPPFLAGS, LDFLAGS, PKG_CONFIG_LIBDIR -- and the - # recipe has to configure against its own empty sysroot, not against a - # half-populated one belonging to the caller. `env -u` is what keeps - # this call from meaning something different depending on where in 5c - # it appears. - ( cd "$ROOT" \ - && env -u CC -u CXX -u AR -u RANLIB -u STRIP -u CFLAGS -u CPPFLAGS \ - -u LDFLAGS -u PKG_CONFIG_LIBDIR -u PKG_CONFIG_PATH \ - bash pkg/zlib/build.sh ) - mkdir -p "$DEP" - cp -a "$ZLIB_BUILD/." "$DEP/" - rm -f "$DEP/.version" - # The .pc file is the one thing that does not relocate by being copied: - # the package was configured --prefix=$MODDIR, so zlib.pc says - # prefix=/usr/data/anvil and pkg-config would answer with -I and -L - # paths that do not exist on this machine. CPython's configure asks - # pkg-config for zlib, so this is not cosmetic. pkg/lib.sh solves the - # same problem properly with PKG_CONFIG_SYSROOT_DIR for recipes that - # consume a package; 5c is not a recipe yet, so it does the one - # substitution by hand until phase 1 makes it one. - sed -i "s|^prefix=.*|prefix=$DEP|" "$DEP/lib/pkgconfig/zlib.pc" - - # OpenSSL, and two traps that were both hit for real: - # * `no-docs` only exists from 3.1. On 3.0.x it is an "Unsupported - # options" HARD ERROR, not a warning -- so it is not passed here. - # * the linux-mips32 target hardcodes -mips2 into its cflags, and - # this toolchain defaults to -mfp64, which gcc refuses below - # mips32r2 ("'-mgp32' and '-mfp64' can only be combined if the - # target supports the mfhc1 and mthc1 instructions"). User cflags - # land AFTER the target's, so -mips32r2 below puts the ISA back - # where the printer actually is. If a future OpenSSL orders them - # the other way round, linux-generic32 (portable C, no mips - # assembly) is the fallback -- and it is taken automatically rather - # than left as a note, because the failure is a wall of assembler - # errors that says nothing about ISA levels. - # * --openssldir is where the interpreter looks for CA certificates - # ON THE PRINTER. Nothing installs any there. `import ssl` works, - # but verifying a certificate chain does not until someone ships a - # ca-certificates bundle -- see tools/python/README.md. - ( cd "$SRC/openssl-$OPENSSL_VERSION" - ossl() { ./Configure "$@" --prefix="$DEP" --libdir=lib \ - --openssldir="$PY_PREFIX/ssl" no-shared no-tests \ - -fPIC -O2 -D_FILE_OFFSET_BITS=64 >/dev/null; } - ossl linux-mips32 -mips32r2 - if ! make -j"$JOBS" >"$LOG/.py-openssl.log" 2>&1; then - echo " .. openssl linux-mips32 failed; falling back to linux-generic32" - make distclean >/dev/null 2>&1 || true - ossl linux-generic32 no-asm - make -j"$JOBS" >"$LOG/.py-openssl.log" 2>&1 - fi - make install_sw >/dev/null ) - - for p in "libffi-$LIBFFI_VERSION --disable-docs" \ - "sqlite-autoconf-$SQLITE_VERSION --disable-readline --disable-editline" \ - "xz-$XZ_VERSION --disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo --disable-scripts --disable-doc --disable-nls" \ - "expat-$EXPAT_VERSION --without-docbook --without-examples --without-tests"; do - # Deliberately unquoted: each entry is a directory plus the flags - # that turn that project's command-line tools and documentation - # off. We want the library, never the programs. - # shellcheck disable=SC2086 - set -- $p - d=$1; shift - ( cd "$SRC/$d" - ./configure --host=$PY_HOST --build=$BUILDTRIPLE --prefix="$DEP" \ - --disable-shared --enable-static --with-pic "$@" >/dev/null - make -j"$JOBS" >/dev/null && make install >/dev/null ) - done - - # bzip2 has no configure and no install target worth using; drive its - # Makefile at the one target we want and place the two files by hand. - ( cd "$SRC/bzip2-$BZIP2_VERSION" - make -j"$JOBS" libbz2.a CC="$CC" AR="$AR" RANLIB="$RANLIB" \ - CFLAGS="-O2 -fPIC -D_FILE_OFFSET_BITS=64 -Wall -Winline" >/dev/null - install -m644 libbz2.a "$DEP/lib/" - install -m644 bzlib.h "$DEP/include/" ) - - # ------------------------------------------------ 3. CPython, cross - cd "$SRC/cross/Python-$PY_VERSION" - # Answers to the questions configure settles by COMPILING AND RUNNING - # a probe, which it cannot do when the target is mipsel and the - # builder is x86. Left unanswered these either stop configure or -- - # worse -- default to the conservative answer and produce a working - # interpreter with subtly wrong float and time behaviour. - cat > "$SRC/config.site" <<'EOF' -ac_cv_file__dev_ptmx=yes -ac_cv_file__dev_ptc=no -ac_cv_buggy_getaddrinfo=no -ac_cv_little_endian_double=yes -ac_cv_big_endian_double=no -ac_cv_mixed_endian_double=no -ac_cv_working_tzset=yes -ac_cv_have_long_long_format=yes -ac_cv_no_strict_aliasing=no -ac_cv_pthread_system_supported=yes -EOF - export CONFIG_SITE="$SRC/config.site" - export CFLAGS="-O2 -D_FILE_OFFSET_BITS=64" - # THE TRAP THAT COSTS A DAY IF YOU MISS IT, and the reason the gate - # below exists: - # -latomic 64-bit atomics on mips32 are out-of-line calls into - # libatomic, and CPython 3.13's _Py_atomic_* on 64-bit - # types needs them. libatomic.so.1 IS on the printer's - # rootfs -- measured -- so this one is a runtime dep, not - # a static link. - # -lm because libsqlite3 here is STATIC. A shared libsqlite3.so - # carries its own DT_NEEDED on libm; a libsqlite3.a does - # not. configure's `checking for sqlite3_bind_double in - # -lsqlite3` link probe then fails on undefined floor/log/ - # pow, and CPython records _sqlite3 as "missing" AND - # CARRIES ON -- a probe failure, not a compile failure, so - # nothing in 400 lines of build output says why the one - # module this whole section exists for is absent. - export LIBS="-latomic -lm" - # And the same link line stated outright, bypassing pkg-config, so - # that -lm cannot be reordered out from under the probe. - export LIBSQLITE3_CFLAGS="-I$DEP/include" - export LIBSQLITE3_LIBS="-L$DEP/lib -lsqlite3 -lm" - # --disable-shared: no libpython3.13.so to find at runtime, for the - # same reason the seven libraries above are static. An out-of-tree - # extension module was proven to build and import against this. - # --without-ensurepip: pip needs a network and a compiler; neither is - # on a printer, and a pip that half works is worse than none. - # --disable-test-modules: the CPython test suite is a third of the - # tree and none of it runs here. - ./configure \ - --host=$PY_HOST --build=$BUILDTRIPLE \ - --with-build-python="$HOSTPY/bin/python3.13" \ - --prefix="$PY_PREFIX" \ - --disable-shared \ - --without-ensurepip \ - --disable-test-modules \ - --with-openssl="$DEP" \ - --with-system-expat >"$LOG/.py-configure.log" 2>&1 - make -j"$JOBS" >"$LOG/.py-make.log" 2>&1 - make install DESTDIR="$STAGE" >>"$LOG/.py-make.log" 2>&1 - ) || { echo " !! the python cross-build failed. The most recently" >&2 - echo " written of work/.py-hostpy.log, work/.py-openssl.log," >&2 - echo " work/.py-configure.log and work/.py-make.log is where it" >&2 - echo " stopped; the source trees under work/.py-src are still" >&2 - echo " there, config.log included." >&2 - exit 1; } - - # ------------------------------------------- 4. the third-party packages - # What runs ON the interpreter: the 18 sdists pinned in versions.env, - # cross-built into the staged tree's own lib/python3.13/site-packages. - # - # WHY IT IS HERE, INSIDE 5c, RATHER THAN IN A SECTION OF ITS OWN. Because - # of what it needs, all of which exists only at this exact moment: - # - # * the UNTRIMMED stage. INCLUDEPY is $PY_PREFIX/include/python3.13 and - # every C extension compiles against it -- and the trim thirty lines - # below deletes include/ and lib/python3.13/config-3.13-* precisely - # because nothing on a PRINTER builds extension modules. - # * work/.py-dep, the static -fPIC C libraries. pillow links zlib out of - # it and cffi links libffi, and it is deleted with the rest of the - # scratch trees at the end of this section. - # * work/.py-host, the x86-64 build-python of the SAME 3.13.7. It is the - # thing that makes the cross work at all (see below), and it is also - # thrown away here. - # - # The alternative -- cache a second, untrimmed tree so the packages can be - # built later and separately -- was considered and declined: it means - # keeping ~200MB and a second interpreter warm to make a rare path faster, - # and it means two caches that can disagree about which interpreter the - # extension modules were compiled against. That disagreement is not - # theoretical: an .so built against 3.13 headers and loaded by a 3.14 - # interpreter is an import-time crash on the printer and nowhere else. - # One cache, two stamps, everything rebuilt together (see PYPKG_STAMP). - # - # THE CROSS TRICK, and there is no crossenv in it. The staged interpreter - # carries _sysconfigdata__linux_mipsel-linux-gnu.py, which records the - # cross CC, LDSHARED, EXT_SUFFIX (.cpython-313-mipsel-linux-gnu.so) and - # INCLUDEPY for the TARGET. Point _PYTHON_SYSCONFIGDATA_NAME at it from an - # x86-64 CPython of the same version and sysconfig -- and therefore - # setuptools' build_ext -- answers every question about mipsel while - # running on x86-64. The one thing the spike needed a container for was - # making INCLUDEPY resolve: it bind-mounted the stage at /usr/data/anvil. - # There is no daemon to ask for that here (the build lane deliberately has - # no docker socket), so instead a COPY of that module is written with - # every $PY_PREFIX path rewritten to where the stage actually is. It goes - # first on PYTHONPATH, is read only by the build, and never ships. - # - # NOTHING HERE TALKS TO A NETWORK. pip runs with --no-index against the - # sdists bin/fetch-assets.sh already checked the sha256 of, which is the - # strong form of the rule that --no-binary :all: below is the weak form - # of: pip cannot download a wheel it cannot reach. Both are kept, because - # the day someone adds an index URL for one awkward package the second one - # is what stops three x86-64 .so files from sailing into the tree -- which - # is not a hypothetical, it is what happened on the spike's first run. - say "python: cross-building $(echo $PYPKG_LIST | wc -w) third-party packages" - ( - set -e - DEP="$PWD/work/.py-dep" - STAGE="$PWD/work/.py-stage" - XW="$PWD/work/.py-xw" - LOG="$PWD/work" - HOSTPY="$PWD/work/.py-host/bin/python$PY_MM" - PKGSRC="$PWD/work/.py-pkgsrc" # unpacked sdists - WHEELS="$PWD/work/.py-wheels" # the wheels they produce - XSYS="$PWD/work/.py-xsysconfig" # the rewritten sysconfigdata - TREE="$STAGE$PY_PREFIX" # the untrimmed staged interpreter - SP="$TREE/lib/python$PY_MM/site-packages" - SYSCFG=_sysconfigdata__linux_mipsel-linux-gnu - rm -rf "$PKGSRC" "$WHEELS" "$XSYS" - mkdir -p "$PKGSRC" "$WHEELS" "$XSYS" "$SP" - # The same gcc wrappers step 3 built, still on disk: -EL -mnan=2008 - # baked into the driver so that setuptools -- which does NOT forward - # CFLAGS to its link lines -- cannot lose them on the way to ld. - export PATH="$XW/bin:$PATH" - - # -------------------------------------------------- the PEP 517 side - # Three backends into the build-python: setuptools (most of the list), - # flit_core (jinja2) and poetry_core (libnacl). All three sdists - # bootstrap themselves through backend-path, so this needs nothing - # that is not already in vendor/, and --no-index proves it. - # - # Every failure in this subshell is caught BY HAND, with an explicit - # `|| { ...; exit 1; }`, and the `set -e` at the top is not what is - # doing the work: a compound command on the left of `||` -- which is - # what this whole subshell is -- runs with errexit suppressed - # THROUGHOUT, so a bare failing command in here would be shrugged off - # and the build would carry on to produce a tree with one package - # quietly missing from it. - for p in setuptools $PYPKG_HOST_LIST; do - "$HOSTPY" -m pip install --quiet --no-index --no-cache-dir \ - --no-build-isolation --no-deps "$PWD/vendor/$(pypkg_var "$p" FILE)" \ - >>"$LOG/.py-pkg-backends.log" 2>&1 \ - || { echo " !! could not install the build backend $p --" \ - "work/.py-pkg-backends.log" >&2 - tail -15 "$LOG/.py-pkg-backends.log" | sed 's/^/ /' >&2 - exit 1; } - done - - # ------------------------------------------- sysconfig, for the TARGET - # The rewrite described above. It is a blanket substitution of - # $PY_PREFIX for the staging path over every string value rather than - # a list of the variables that matter, because the list of variables - # that matter is exactly the thing nobody gets right by hand: - # INCLUDEPY is the one that fails loudly, LIBDIR and LIBPL fail - # quietly, and a future setuptools is free to ask for another. - "$HOSTPY" - "$TREE/lib/python$PY_MM/$SYSCFG.py" "$XSYS/$SYSCFG.py" \ - "$PY_PREFIX" "$TREE" <<'PYEOF' || exit 1 -import sys -src, dst, prefix, stage = sys.argv[1:5] -ns = {} -exec(compile(open(src).read(), src, "exec"), ns) -out = {k: (v.replace(prefix, stage) if isinstance(v, str) else v) - for k, v in ns["build_time_vars"].items()} -with open(dst, "w") as fh: - fh.write("# Generated by bin/patch.sh: %s with the build-time prefix\n" - "# rewritten to the staging tree. Build-time only; never ships.\n" - "build_time_vars = %r\n" % (src, out)) -PYEOF - [ -s "$XSYS/$SYSCFG.py" ] || { - echo " !! could not rewrite $SYSCFG for the staging tree" >&2; exit 1; } - export _PYTHON_SYSCONFIGDATA_NAME="$SYSCFG" - # ONLY the rewritten module -- deliberately NOT the target's stdlib as - # well, which is what the spike put here. They are the same 3.13.7, so - # it did no harm, but it also means the host interpreter importing - # the TARGET's os.py and sysconfig.py by accident is one path-ordering - # mistake away, and the only thing that has to be importable is this. - export PYTHONPATH="$XSYS" - export PYTHONDONTWRITEBYTECODE=1 - export PIP_DISABLE_PIP_VERSION_CHECK=1 - export CC="$PY_HOST-gcc" CXX="$PY_HOST-g++" - export AR="$PY_HOST-ar" RANLIB="$PY_HOST-ranlib" STRIP="$PY_HOST-strip" - export LDSHARED="$PY_HOST-gcc -shared -L$DEP/lib" - export CFLAGS="-O2 -fPIC -D_FILE_OFFSET_BITS=64 -I$DEP/include -I$TREE/include/python$PY_MM" - export CXXFLAGS="$CFLAGS" - export LDFLAGS="-L$DEP/lib" - # pkg-config must see ONLY our sysroot, for the same reason as step 2: - # left alone it answers out of the build image's /usr/lib/pkgconfig and - # a package links an x86-64 library into a mipsel extension module. - export PKG_CONFIG_LIBDIR="$DEP/lib/pkgconfig" - export PKG_CONFIG_PATH="$DEP/lib/pkgconfig" - - # Gate the trick before building 18 packages on top of it. If - # _PYTHON_SYSCONFIGDATA_NAME has not taken, every question sysconfig - # is asked below is answered for x86-64 -- and the answer is a tree of - # host objects that builds perfectly and imports nowhere. - "$HOSTPY" - "$TREE/include/python$PY_MM" <<'PYEOF' || exit 1 -import sys, sysconfig -suffix = sysconfig.get_config_var("EXT_SUFFIX") -inc = sysconfig.get_config_var("INCLUDEPY") -print(" sysconfig answers for %s, headers %s" - % (suffix, "found" if __import__("os").path.isdir(inc) else "MISSING")) -if suffix != ".cpython-313-mipsel-linux-gnu.so": - raise SystemExit(" !! sysconfig is answering for the HOST (EXT_SUFFIX=%s)" - " -- _PYTHON_SYSCONFIGDATA_NAME did not take" % suffix) -if inc != sys.argv[1]: - raise SystemExit(" !! INCLUDEPY is %s, expected the staged %s" - % (inc, sys.argv[1])) -PYEOF - - # ------------------------------------------------------- the helpers - # pywheel [env=val ...] -- build one wheel into $WHEELS. - # - # --no-binary :all: is not caution and not decoration; see the top of - # this step and versions.env. --no-build-isolation is what keeps - # _PYTHON_SYSCONFIGDATA_NAME reaching setup.py: with isolation ON pip - # rewrites PYTHONPATH for the build subprocess and the module above - # stops being importable. - pywheel() { - local what=$1; shift - env "$@" "$HOSTPY" -m pip wheel --no-deps --no-build-isolation \ - --no-index --no-binary :all: --no-cache-dir \ - -w "$WHEELS" "$what" >"$LOG/.py-pkg-$(basename "$what").log" 2>&1 \ - || { echo " !! $(basename "$what") failed to build --" \ - "work/.py-pkg-$(basename "$what").log" >&2 - tail -25 "$LOG/.py-pkg-$(basename "$what").log" | sed 's/^/ /' >&2 - exit 1; } - } - # unpack -- untar its sdist and echo the source directory. - # Every one of these tarballs unpacks to its own basename; a release - # that did not would fail on the [ -d ] rather than build the wrong - # tree. - unpack() { - local f d - f="$(pypkg_var "$1" FILE)" - d="$PKGSRC/${f%.tar.gz}" - tar -xzf "$PWD/vendor/$f" -C "$PKGSRC" - [ -d "$d" ] || { echo " !! $f does not unpack to $(basename "$d")" >&2; exit 1; } - printf '%s' "$d" - } - - # ------------------------------------------------------- the packages - # PYPKG_LIST is the order, so the order lives in versions.env beside - # the pins. Four entries need more than "wheel the sdist", and each - # `case` arm says why. - for p in $PYPKG_LIST; do - SDIST="$PWD/vendor/$(pypkg_var "$p" FILE)" - case "$p" in - preprocess-cancellation) - # NOT BUILT: copied. This sdist declares no [build-system] and - # ships no setup.py, so there is no backend to call -- pip's - # legacy fallback would auto-discover a flat layout and emit a - # wheel named UNKNOWN-0.0.0 containing the one module, which - # is the right file arrived at by an accident that a future - # setuptools is free to change. The package IS one module, so - # the honest operation is a copy. Moonraker imports it in the - # metadata SUBPROCESS (gcode object cancellation), lazily, so - # a printer without it loses that feature and nothing else. - d=$(unpack "$p") - cp -f "$d/preprocess_cancellation.py" "$SP/" - ;; - greenlet) - # gcc 7.2 refuses greenlet 3.x's C++ designated initializers - # ("sorry, unimplemented: non-trivial designated initializers - # not supported") wherever they skip a field. The rule was - # probed, not assumed: in-order contiguous is fine, a TRAILING - # gap is fine, an interior or leading gap is refused. - # fill-designators.py writes the skipped fields back as - # explicit zeros, taking field order from the target's own - # headers and refusing to guess about a designator that is not - # in them. These are objects of static storage duration, so - # every field it inserts was already zero: it changes the - # spelling, not the program. 55 fields across two files. - d=$(unpack "$p") - # Deliberately unquoted: the find is a LIST of sources to - # patch, one argument each, and none of greenlet's file names - # has ever contained a space. - # shellcheck disable=SC2046 - "$HOSTPY" tools/python-packages/fill-designators.py \ - "$TREE/include/python$PY_MM" \ - $(find "$d/src/greenlet" -name '*.cpp' -o -name '*.hpp' | sort) \ - >"$LOG/.py-pkg-greenlet-designators.log" 2>&1 \ - || { echo " !! fill-designators.py failed --" \ - "work/.py-pkg-greenlet-designators.log" >&2 - tail -20 "$LOG/.py-pkg-greenlet-designators.log" | sed 's/^/ /' >&2 - exit 1; } - pywheel "$d" - ;; - lmdb) - # LMDB_FORCE_CPYTHON=1 is load-bearing. lmdb's setup.py has two - # backends: a real CPython extension and a cffi one that ships - # mdb.c and COMPILES IT AT IMPORT TIME. Left to itself here it - # picks cffi and produces a py3-none-any wheel with mdb.c - # inside -- i.e. a Moonraker that tries to invoke a compiler on - # a printer that has none. (Do not "fix" this with - # LMDB_FORCE_CFFI=0: setup.py tests the variable for PRESENCE, - # so the string "0" selects cffi.) - pywheel "$SDIST" LMDB_FORCE_CPYTHON=1 - ;; - pillow) - # ZLIB ONLY, and pip cannot drive it: the --disable-* flags are - # build_ext options and PEP 517 offers no way to pass them, so - # setup.py is called directly. Gcode thumbnails are base64 PNG, - # so zlib is the only codec on that path; jpeg, tiff, webp, - # jpeg2000, lcms, freetype and imagequant are libraries nobody - # has cross-built here, and pillow probes for them by trying to - # LINK against the HOST's copies unless told not to -- which is - # what --disable-platform-guessing is for. - d=$(unpack "$p") - ( cd "$d" && "$HOSTPY" setup.py build_ext \ - --disable-jpeg --disable-tiff --disable-webp --disable-jpeg2000 \ - --disable-imagequant --disable-lcms --disable-freetype \ - --disable-xcb --disable-platform-guessing --enable-zlib \ - -I"$DEP/include" -L"$DEP/lib" bdist_wheel ) \ - >"$LOG/.py-pkg-pillow.log" 2>&1 \ - || { echo " !! pillow failed to build -- work/.py-pkg-pillow.log" >&2 - tail -25 "$LOG/.py-pkg-pillow.log" | sed 's/^/ /' >&2 - exit 1; } - cp -a "$d"/dist/*.whl "$WHEELS/" \ - || { echo " !! pillow built no wheel -- work/.py-pkg-pillow.log" >&2 - exit 1; } - ;; - *) - pywheel "$SDIST" - ;; - esac - printf ' %-24s %s\n' "$p" "$(pypkg_var "$p" FILE)" - done - - # ----------------------------------------------------- into the tree - # Unzipped rather than pip-installed, because `pip install` insists on - # installing FOR the interpreter running it -- and that one is x86-64. - # A wheel is a zip whose layout is already the layout of - # site-packages, so unzipping it is the whole of what an install does - # here. - # - # .dist-info goes: nothing on the printer resolves a dependency, asks - # for a version or runs an entry point, and it is 200KB of metadata - # describing a build machine to nobody. The .data directories and the - # scripts in them go for a stronger reason -- their console-script - # shebangs name the BUILD-PYTHON's path, so shipping them would put - # files on the printer that reference /home/somebody/work/.py-host. - for w in "$WHEELS"/*.whl; do - "$HOSTPY" -m zipfile -e "$w" "$SP" \ - || { echo " !! could not unpack $(basename "$w") into site-packages" >&2 - exit 1; } - done - find "$SP" -name '__pycache__' -prune -exec rm -rf {} + 2>/dev/null || true - find "$SP" -maxdepth 1 -name '*.dist-info' -prune -exec rm -rf {} + 2>/dev/null || true - rm -rf "${SP:?}"/bin "${SP:?}"/*.data - # Stripped here rather than in the trim below, because the trim's - # find only knows about lib-dynload -- and these are the .so files - # that are NOT stdlib. 12 of them, ~1.5MB of symbols. - find "$SP" -name '*.so' -exec "$PY_HOST-strip" {} + 2>/dev/null || true - ) || { echo " !! the third-party package build failed. Its logs are" >&2 - echo " work/.py-pkg-*.log, one per package, and the unpacked" >&2 - echo " sources are still in work/.py-pkgsrc." >&2 - exit 1; } - - # --------------------------------------------------------------- trim - # 183MB staged, 30MB shipped. What goes: the test package and idlelib - # (nothing on a printer runs either), tkinter (there is no X11), the - # config-*/ directory, include/ and lib/pkgconfig (they exist to BUILD - # extension modules, which happens on a developer's machine and not here - # -- and lib/pkgconfig would otherwise put four .pc files describing that - # build into the prefix's SHARED lib/, next to s6's neighbours), every - # static archive, and every __pycache__ -- which is 12MB of .pyc for - # modules that will be imported once, if ever, and which the interpreter - # regenerates into /usr/data anyway if it ever wants them. - say "python: trimming the staged tree" - PY_TRIM="work/.py-stage$PY_PREFIX" - rm -rf "$PY_TRIM/lib/python$PY_MM/test" \ - "$PY_TRIM/lib/python$PY_MM/idlelib" \ - "$PY_TRIM/lib/python$PY_MM/tkinter" \ - "$PY_TRIM/lib/python$PY_MM/turtledemo" \ - "$PY_TRIM/lib/python$PY_MM/config-"* \ - "$PY_TRIM/include" "$PY_TRIM/share" "$PY_TRIM/lib/pkgconfig" - find "$PY_TRIM" -name '__pycache__' -prune -exec rm -rf {} + 2>/dev/null || true - find "$PY_TRIM" -name '*.a' -delete - # And out of bin/, everything except the one unambiguously-named binary -- - # see the comment at the top of this section. bin/python3 is the one that - # matters (it would shadow FlashForge's on the PATH anvil-env.sh exports); - # the rest are launchers for a package the trim above just deleted. - rm -f "$PY_TRIM/bin/python3" "$PY_TRIM/bin/idle3" "$PY_TRIM/bin/idle$PY_MM" \ - "$PY_TRIM/bin/pydoc3" "$PY_TRIM/bin/pydoc$PY_MM" \ - "$PY_TRIM/bin/python3-config" "$PY_TRIM/bin/python$PY_MM-config" - "$PY_TOOLCHAIN_DIR/bin/$PY_HOST-strip" "$PY_TRIM/bin/python$PY_MM" 2>/dev/null || true - find "$PY_TRIM/lib/python$PY_MM/lib-dynload" -name '*.so' \ - -exec "$PY_TOOLCHAIN_DIR/bin/$PY_HOST-strip" {} + 2>/dev/null || true - # Nothing must be left in bin/ but the interpreter: a future CPython that - # installs one more launcher would otherwise put an unreviewed name on the - # PATH of every mod process, which is precisely the accident the rm above - # is for. Cheap to assert, and it fails at build time rather than on a - # printer. - left=$(ls "$PY_TRIM/bin") - [ "$left" = "python$PY_MM" ] || { - echo " !! python: bin/ should hold python$PY_MM and nothing else, got: $left" >&2 - exit 1; } - - # The cache holds bin/ and lib/ -- exactly the shape work/.s6 holds for s6, - # and for the same reason: unpacking it at $MODDIR puts every file back - # where it was compiled to live. The .version stamp sits BESIDE them rather - # than inside, so it can never be mistaken for something that ships. - # - # site-packages rides along inside lib/python$PY_MM/ and needs no line of - # its own here, which is the whole reason step 4 installs into the staged - # tree rather than somewhere beside it: one copy, one cache, one tarball - # for the replica gates (test/ffsim/gates.py packs bin/ + lib/ out of this - # directory), and the ABI gate below already walks all of it. - mkdir -p "$PY_BUILD" - cp -a "$PY_TRIM/bin" "$PY_TRIM/lib" "$PY_BUILD/" - rm -rf work/.py-src work/.py-dep work/.py-host work/.py-stage work/.py-xw \ - work/.py-pkgsrc work/.py-wheels work/.py-xsysconfig - echo "$PY_STAMP" > "$PY_BUILD/.version" - # Written LAST and separately, so that an interrupted build leaves a cache - # that is stale rather than one that lies: both files have to be right - # before the next run is allowed to skip this section. - printf '%s\n' "$PYPKG_STAMP" > "$PY_BUILD/.pkg-version" -else - skip "python: work/.py313 already holds CPython $PY_VERSION and its packages" -fi - -# The gates, asked of the TREE THAT SHIPS rather than of the build -- so a -# cached tree from an older, wronger build is checked too, and so the answer -# comes from the bytes the printer will execute. -[ -x "$PY_BUILD/bin/python$PY_MM" ] \ - || { echo " !! python: no bin/python$PY_MM in $PY_BUILD" >&2; exit 1; } -# bin/ holds the interpreter and NOTHING ELSE -- re-asserted here and not only -# at build time, because this runs against the cache as well, and a cache -# built before that rule existed would otherwise stage a bin/python3 that -# shadows FlashForge's on PATH. See the top of this section. -PY_BINS=$(ls "$PY_BUILD/bin") -[ "$PY_BINS" = "python$PY_MM" ] \ - || { echo " !! python: $PY_BUILD/bin holds '$PY_BINS', expected only python$PY_MM." >&2 - echo " Delete work/.py313 and rebuild." >&2; exit 1; } -# _sqlite3 is THE reason this section exists, and its absence is silent: see -# the LIBS comment above. A hard failure here, at build time, is the only -# thing standing between a dropped -lm and a release that ships an -# interpreter with the one module it was built for missing. -PY_SQLITE=$(ls "$PY_BUILD/lib/python$PY_MM/lib-dynload/"_sqlite3*.so 2>/dev/null | head -n1) +# The gates, asked of the TREE THAT SHIPS rather than of a build -- so a cached +# recipe from an older, wronger build is checked too, and so the answer comes +# from the bytes the printer will execute. pkg/python/build.sh gates the +# interpreter's own stdlib modules when it runs; these run every time. +# +# Two claims about site-packages, because they fail differently. A +# site-packages with no EXTENSION MODULES in it is a build where every native +# package quietly fell back to something pure -- it imports on the build host +# and loses lmdb, cffi and greenlet on the printer. A site-packages missing a +# NAMED package is a list that changed without anyone noticing; the three named +# here are the three whose absence takes Moonraker or klippy down completely +# rather than costing a feature. +PY_SQLITE=$(ls "$MOD_PAYLOAD/lib/python$PY_MM/lib-dynload/"_sqlite3*.so 2>/dev/null | head -n1) [ -n "$PY_SQLITE" ] \ - || { echo " !! python: NO _sqlite3 MODULE. configure's link probe failed" >&2 - echo " silently -- check LIBS/LIBSQLITE3_LIBS and work/.py-configure.log" >&2 + || { echo " !! python: NO _sqlite3 MODULE -- the one module this" >&2 + echo " interpreter exists for. Delete work/pkg/python and rebuild." >&2 exit 1; } -# And the packages, asked of the cache for the same reason: this runs whether -# step 4 built them a minute ago or a previous build did. Two claims, because -# they fail differently. A site-packages with no EXTENSION MODULES in it is a -# build where every native package quietly fell back to something pure or was -# skipped -- it imports on the build host and loses lmdb, cffi and greenlet on -# the printer. A site-packages missing a NAMED package is a list that changed -# without anyone noticing; the three named here are the three whose absence -# takes Moonraker or klippy down completely rather than costing a feature. -PY_SP="$PY_BUILD/lib/python$PY_MM/site-packages" +PY_SP="$MOD_PAYLOAD/lib/python$PY_MM/site-packages" for m in lmdb tornado cffi; do [ -e "$PY_SP/$m" ] || [ -e "$PY_SP/$m.py" ] \ - || { echo " !! python: no '$m' in site-packages -- the package build" >&2 - echo " did not produce it. Delete work/.py313 and rebuild." >&2 + || { echo " !! python: no '$m' in site-packages -- pkg/python-$m" >&2 + echo " produced nothing. Delete work/pkg/python-$m and rebuild." >&2 exit 1; } done PY_EXT=$(find "$PY_SP" -name '*.so' | wc -l) [ "$PY_EXT" -ge 12 ] \ || { echo " !! python: only $PY_EXT extension modules in site-packages," >&2 echo " expected at least 12. A native package fell back to a" >&2 - echo " pure-python or py3-none-any build; check work/.py-pkg-*.log." >&2 + echo " pure-python build; check work/.pkg-python-*/wheel-*.log." >&2 exit 1; } - -# Staged into the SAME bin/ and lib/ as everything else in this prefix root, -# which is why this is a copy of the contents and not of the directories: s6's -# binaries are already in $MOD_PAYLOAD/bin and must stay there. -cp -a "$PY_BUILD/bin/." "$MOD_PAYLOAD/bin/" -mkdir -p "$MOD_PAYLOAD/lib" -cp -a "$PY_BUILD/lib/." "$MOD_PAYLOAD/lib/" -chmod +x "$MOD_PAYLOAD/bin/python$PY_MM" PY_ELF=$(mips_abi_gate "$MOD_PAYLOAD/bin/python$PY_MM" "$MOD_PAYLOAD/lib/python$PY_MM") || exit 1 say "python: $PY_ELF ELF objects staged, all nan2008/o32/mips32r2;" \ "$(basename "$PY_SQLITE") present; $PY_EXT extension modules in site-packages" -du -sh "$PY_BUILD/lib/python$PY_MM" | awk '{print " "$1"\tlib/python'"$PY_MM"'/"}' +du -sh "$MOD_PAYLOAD/lib/python$PY_MM" | awk '{print " "$1"\tlib/python'"$PY_MM"'/"}' du -sh "$PY_SP" | awk '{print " "$1"\tlib/python'"$PY_MM"'/site-packages/"}' # ---------------------------------------------------- 5d. libsodium (shipped) diff --git a/bin/verify.sh b/bin/verify.sh index 23be550..09bae12 100755 --- a/bin/verify.sh +++ b/bin/verify.sh @@ -165,7 +165,7 @@ if [ -f "$WORK/anvil.tar.xz" ]; then # interpreter can be present and perfectly runnable while _sqlite3 is # absent, because a dropped -lm makes configure's link probe fail and # CPython records the module as "missing" rather than stopping -- see the - # LIBS comment in patch.sh. sqlite3 is the entire reason this interpreter + # LIBS comment in pkg/python/build.sh. sqlite3 is the entire reason this interpreter # is built (it is what eventually unpins MOONRAKER_VERSION), so a tree # without it is a 30MB payload that bought nothing. Build bugs both: a # package that ships neither cannot be fixed on the printer. @@ -174,7 +174,7 @@ if [ -f "$WORK/anvil.tar.xz" ]; then || bad "no bin/python3.13 in the payload -- bin/patch.sh did not stage the cross-build" grep -q 'lib-dynload/_sqlite3' <<<"$LIST" \ && ok "python3.13 carries _sqlite3 (the module FlashForge's 3.8.2 has not got)" \ - || bad "python3.13 ships WITHOUT _sqlite3 -- the one module it exists for; check LIBS/LIBSQLITE3_LIBS in patch.sh" + || bad "python3.13 ships WITHOUT _sqlite3 -- the one module it exists for; check LIBS/LIBSQLITE3_LIBS in pkg/python/build.sh" # And the other half of the same decision: the interpreter goes into the # prefix root's bin/ like everything else, but WITHOUT the `python3` # symlink CPython installs beside it. $MODDIR/bin is prepended to PATH by @@ -204,10 +204,23 @@ if [ -f "$WORK/anvil.tar.xz" ]; then # itself cpython-313-x86_64-linux-gnu.so, so the name is the architecture. grep -q 'site-packages/lmdb/cpython.cpython-313-mipsel-linux-gnu\.so' <<<"$LIST" \ && ok "lmdb's mipsel CPython extension present (Moonraker's database at this pin)" \ - || bad "no mipsel lmdb extension in site-packages -- Moonraker cannot open its database; check bin/patch.sh section 5c step 4" + || bad "no mipsel lmdb extension in site-packages -- Moonraker cannot open its database; check work/.pkg-python-lmdb/wheel-lmdb.log" grep -q 'site-packages/_cffi_backend.cpython-313-mipsel-linux-gnu\.so' <<<"$LIST" \ && ok "cffi's mipsel extension present (klippy's route to c_helper.so)" \ || bad "no mipsel _cffi_backend in site-packages -- klippy could not load chelper on 3.13" + # THE DEV HALF MUST NOT BE HERE. anvil-python and anvil-python-dev are one + # build split in two, and only the first is something a printer installs: + # the headers, lib/pkgconfig and config-3.13-* exist so that the eighteen + # pkg/python-* recipes can compile extension modules on a BUILD machine. + # bin/patch.sh prunes them from the payload by reading PKG_DEV_FILES out of + # pkg/python/pkg.conf, and this is the check that the pruning happened -- + # it is three megabytes of files nothing on a printer can open, and it + # would ship silently, because a payload with extra files in it works. + if grep -qE '(^|/)(include/python3\.13/|lib/pkgconfig/|config-3\.13-)' <<<"$LIST"; then + bad "payload carries CPython's dev files (headers/pkgconfig/config-3.13-*) -- anvil-python-dev's half was not pruned" + else + ok "no CPython dev files in the payload (headers and build config stay in anvil-python-dev)" + fi # libsodium is the one library of ours that ships as a .so, because libnacl # dlopens it -- and the bare `libsodium.so` name is not decoration, it is # the exact name libnacl's path fallback constructs. A payload with the diff --git a/docs/how-it-works.md b/docs/how-it-works.md index c2a2a4b..ea18fa9 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -119,10 +119,11 @@ running the printer's own interpreter against the real image), and never invoked — `S62moonraker` starts the server itself. **FlashForge's python 3.8.2 is not what this runs on any more.** `FF_PYTHON` -in `anvil-env.sh` names a CPython 3.13 of our own, cross-built for mipsel -(`bin/patch.sh` section 5c) with every third-party C extension Moonraker -needs beside it in `$MODDIR/lib/python3.13/site-packages` and libsodium in -`$MODDIR/lib` — none of it borrowed from `/usr/prog`. Measured through the +in `anvil-env.sh` names a CPython 3.13 of our own, cross-built for mipsel by +`pkg/python`, with every third-party C extension Moonraker needs beside it in +`$MODDIR/lib/python3.13/site-packages` — eighteen packages, one `pkg/python-*` +recipe and one `.ipk` each — and libsodium in `$MODDIR/lib`; none of it +borrowed from `/usr/prog`. Measured through the real boot path (S40s6's scandir, `S62moonraker`, readiness gating on `:7125` actually listening, a `kill -9` respawn, a stop that stays stopped) in `test/integration/printer/case-moonraker313-s6.sh`. klippy is not part of diff --git a/docs/notes/80-s6-migration.md b/docs/notes/80-s6-migration.md index e0dde9a..30cb8fc 100644 --- a/docs/notes/80-s6-migration.md +++ b/docs/notes/80-s6-migration.md @@ -225,6 +225,14 @@ moonraker runs from `/usr/data` on the printer's own Python. ## Phase 6 -- own the Python environment (separate project) +**DONE, by 6b: CPython is built here.** `pkg/python` cross-compiles 3.13.7 +against the seven static libraries this feed already builds, and each of the +eighteen third-party packages is a `pkg/python-*` recipe with its own version +and its own `.ipk`. `anvil-env.sh` points `FF_PYTHON` at it. What is written +below is the scoping that produced that decision, kept because the premises it +corrects are still the ones somebody would reach for -- in particular why musl +was never an option here, which is the paragraph that decided phase 7 too. + Scoped, in the replica, before starting. The scoping changed the shape of this phase and corrected a premise, so read this before writing any of it. @@ -299,7 +307,11 @@ has to catch that class of failure. ## Phase 7 -- own Klipper -Only reachable after phase 6. Ship the klippy tree into `$MODDIR` instead of +**Now reachable: phase 6 is done.** The interpreter it needs is `pkg/python`, +built with the Ingenic glibc toolchain -- the requirement the paragraph above +calls fatal if got wrong, since klippy `dlopen`s a glibc `c_helper.so`. + +Ship the klippy tree into `$MODDIR` instead of handing it to FlashForge's `run.sh`; replace `klipperDaemon` with an s6 `run` script (the command line is quoted above); turn `S70klipper`'s retry loop into an s6 restart policy plus readiness. `checkEboard`, `libmcu-bare.bin` and diff --git a/docs/notes/85-packaging.md b/docs/notes/85-packaging.md index 463213e..ddf246b 100644 --- a/docs/notes/85-packaging.md +++ b/docs/notes/85-packaging.md @@ -319,21 +319,39 @@ on this. ## Phase 1 — a recipe per cross-build *(~3–5 days)* Turn the rest of `bin/patch.sh`'s builds into recipes, the way libsodium -became one. Roughly ten packages — +became one — klipper-fork toolchange-config - python3 python3-site-packages - openssl libffi sqlite xz expat bzip2 + python ✔ python-<18 packages> ✔ + openssl ✔ libffi ✔ sqlite ✔ xz ✔ expat ✔ bzip2 ✔ zlib ✔ libarchive ✔ libsodium ✔ opkg ✔ skalibs ✔ execline ✔ s6 ✔ s6-rc ✔ mainsail ✔ moonraker ✔ helixscreen ✔ anvil-core ✔ -Twelve of the nineteen are done. Section 5b — the last place where one script -built two libraries — is four recipes now (`skalibs`, `execline`, `s6`, -`s6-rc`), and moving them off the musl toolchain removed the second libc from -this tree entirely. What is left is CPython and the six static libraries under -it, and then Klipper, which `docs/notes/80-s6-migration.md` phase 7 makes -reachable only after CPython. +**Everything except Klipper is done: 38 recipes producing 40 packages.** +Section 5b — the last place where one script built two libraries — is four +recipes (`skalibs`, `execline`, `s6`, `s6-rc`), and moving them off the musl +toolchain removed the second libc from this tree entirely. Section 5c, which +was 825 lines, is `pkg/python` plus eighteen `pkg/python-*` and 115 lines of +staging. + +**What splitting CPython actually bought.** Not tidiness. The old section had +one cache and two stamps, and its own comment explained that a bumped Pillow +rebuilt the interpreter and all eighteen packages — because a wheel could only +be cross-built during the few minutes when an untrimmed staging tree, a private +sysroot of static libraries and a throwaway x86-64 build-python happened to +exist at once. Those are now, respectively: the feed's `anvil-python-dev` +package, each recipe's own sysroot, and `pkg_buildpython`'s shared cache. None +of them is a passing moment, so `make packages PKG=python-pillow` is a Pillow +build and nothing else. + +It also made eighteen invisible things visible. Each package now carries its +upstream version into the index, `anvil-moonraker` declares the thirteen it +imports, and a printer that does not want gcode thumbnails can leave the 500KB +of Pillow uninstalled — a decision that could not previously be expressed. + +Klipper is what remains, and `docs/notes/80-s6-migration.md` phase 7 makes it +reachable only after CPython, which is now behind us. `patch.sh` keeps staging the payload from those same trees, so the shipped tarball does not change. The work is mechanical; the risk is in the two builds @@ -356,8 +374,10 @@ real regression from an expected difference: What is checked instead: the staged `work/modpayload` **tree** — file list, modes and content hashes — is unchanged except for the components a change deliberately rebuilt. The `.ipk` files themselves ARE byte-reproducible and -that is checked directly: two cold `make packages` runs produce twelve -identical archives. +that is checked directly: two cold `make packages` runs produce identical +archives. Measured over all 40 with `work/pkg`, the shared build-python and +the unpacked toolchain all deleted in between — so the x86-64 interpreter that +generates every mipsel extension module is itself rebuilt rather than reused. ## Phase 2 — install packages on the printer *(~1–2 days)* @@ -408,11 +428,18 @@ filesystem, not packages of ours, and they stay in `patch.sh` until phase 7 of `docs/notes/80-s6-migration.md` moves Klipper under `$MODDIR` and leaves `firmwareExe` as the only file placed outside `/usr/data`. -**The blocker is CPython.** `patch.sh` section 5c is 800 lines and builds the -interpreter, seven static libraries and eighteen cross-built wheels; until +**The CPython blocker is gone.** It read: "section 5c is 800 lines and builds +the interpreter, seven static libraries and eighteen cross-built wheels; until that is a set of recipes there is no feed that contains everything the payload -needs, so the loop above would produce a tree with no Python in it. That is -the next chunk of work, and it is the last one before this becomes possible. +needs." It is a set of recipes now — 40 packages, and the loop above would +produce a tree with Python in it. Every recipe the payload needs exists. + +**What blocks it now is smaller and different.** `pkg/ipk-install` has to run +against a staging root rather than a live one, `bin/pack.sh` has to take its +tree from there, and section 10b's generated `.install-manifest` has to give +way to opkg's own `.list` files — which `pkg/ipk-install` already writes, and +already notes is what makes the hand-rolled manifest redundant. None of that +needs a compiler. **Gate:** a replica install driven by packages leaves the same tree the tarball leaves, and an upgrade removes exactly what the previous version diff --git a/pkg/lib.sh b/pkg/lib.sh index 253a023..de2f2e6 100644 --- a/pkg/lib.sh +++ b/pkg/lib.sh @@ -270,6 +270,7 @@ pkg_begin() { # only spells what is unusual about its project. See pkg_build. PKG_CONFIGURE='./configure'; PKG_CONFIGURE_AUTO=1 PKG_MAKE_TARGET=''; PKG_INSTALL_TARGET='install'; PKG_MAKE_ARGS='' + PKG_PY_SETUP_ARGS='' return 0 } @@ -358,6 +359,123 @@ pkg_toolchain() { pkg_say "$PKG_ID: toolchain ready ($PKG_HOST, e_flags=$_abi)" } +# ---------------------------------------------------------- pkg_buildpython +# +# pkg_buildpython +# +# Provide an x86-64 CPython of exactly $PY_VERSION and export $HOSTPY. +# +# IT SITS BESIDE pkg_toolchain BECAUSE IT IS THE SAME KIND OF THING: a +# compiler for the build machine, not a file for the printer. pkg_toolchain +# unpacks one and this one compiles it, which is an implementation detail of +# the same sentence -- nothing either of them produces is ever shipped, and no +# recipe should be spelling out how to get one. +# +# WHY THE VERSIONS MUST MATCH EXACTLY. Cross-compiling CPython needs a +# build-python OF THE SAME VERSION: the Makefile runs it to freeze modules, +# generate the deepfreeze sources and byte-compile the stdlib, and configure +# hard-errors when the version does not match. The build image's own python3 +# cannot stand in. And the eighteen pkg/python-* recipes need it for a second +# reason -- it is the interpreter that runs pip and setuptools while +# _PYTHON_SYSCONFIGDATA_NAME makes them answer for mipsel (see pkg_pytarget), +# and an extension module compiled against 3.13 headers by a 3.14 setuptools +# is an import-time crash on the printer and nowhere else. +# +# ONE CACHE, SHARED BY NINETEEN RECIPES. It lives at work/.py-host rather than +# inside $PKG_WORK, so pkg_end does not delete it and the second recipe to ask +# gets it for the price of reading a stamp. That stamp is $PY_VERSION plus the +# PEP 517 backend sdists installed into it, because a backend bump has to +# rebuild the thing the backends live in. +# +# --with-ensurepip=install, where the CROSS build has --without-ensurepip and +# must keep it. The two are opposite answers to opposite questions: on a +# printer pip would need a network and a compiler and has neither, while HERE +# pip is what builds every wheel -- and it comes out of the pinned tarball +# rather than off bootstrap.pypa.io, which is the difference between a build +# machine that downloads and runs an unpinned get-pip.py and one that does not +# talk to anybody. +pkg_buildpython() { + PKG_HOSTPY_ROOT="$PWD/work/.py-host" + HOSTPY="$PKG_HOSTPY_ROOT/bin/python$PY_MM" + export HOSTPY + + _hpstamp="$PY_VERSION" + for _p in setuptools $PYPKG_HOST_LIST; do + _hpstamp="$_hpstamp $(pypkg_var "$_p" FILE)" + done + if [ "$(cat "$PKG_HOSTPY_ROOT/.version" 2>/dev/null || true)" = "$_hpstamp" ]; then + pkg_skip "$PKG_ID: build-python $PY_VERSION is already at work/.py-host" + return 0 + fi + + [ -f "${PY_TGZ:-}" ] || pkg_die \ + "$PKG_ID needs the CPython source and '${PY_TGZ:-}' is missing. Run ./bin/fetch-assets.sh." + command -v gcc >/dev/null || pkg_die \ + "$PKG_ID needs a host gcc to build the build-python. Run through 'make packages'." + + pkg_say "$PKG_ID: building the x86-64 build-python $PY_VERSION (once, then cached)" + rm -rf "$PKG_HOSTPY_ROOT" work/.py-hostsrc + mkdir -p work/.py-hostsrc + tar -xf "$PY_TGZ" -C work/.py-hostsrc \ + || pkg_die "$PKG_ID: could not untar $PY_TGZ" + _hplog="$PKG_LOG/buildpython.log" + ( + set -e + # THE CROSS ENVIRONMENT IS REMOVED, NOT AVOIDED BY CALLING ORDER. A + # recipe is free to call pkg_toolchain and pkg_deps before this, and + # both export CC, CFLAGS and a sysroot that describe the PRINTER. A + # configure run inheriting those builds an x86-64 interpreter with + # mipsel flags and fails somewhere unrecognisable. Unsetting them here + # means this verb means the same thing wherever it appears in a recipe. + unset CC CXX AR RANLIB STRIP NM OBJCOPY OBJDUMP LD + unset CFLAGS CXXFLAGS CPPFLAGS LDFLAGS LIBS CONFIG_SITE + unset PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG_SYSROOT_DIR + unset _PYTHON_SYSCONFIGDATA_NAME PYTHONPATH + cd "work/.py-hostsrc/Python-$PY_VERSION" + ./configure --prefix="$PKG_HOSTPY_ROOT" --with-ensurepip=install + make -j"$(nproc 2>/dev/null || echo 4)" + make install + ) > "$_hplog" 2>&1 || pkg_die "$PKG_ID: the build-python failed to build -- see $_hplog" + + # zlib, ASSERTED RATHER THAN ASSUMED, because CPython's answer to a missing + # library is to record the module as absent and carry on. Every wheel is a + # zip and so is pip: ensurepip installs it out of a bundled .whl, and + # without zlib that fails with "can't decompress data" INSIDE a make + # install that reports success. The symptom then arrives three steps later + # as "No module named pip", nowhere near the cause. zlib1g-dev in + # docker/Dockerfile.build is the fix, and this is the sentence that says so. + "$HOSTPY" -c 'import zlib' 2>/dev/null || pkg_die \ + "the build-python has no zlib module, so it cannot unpack a single wheel. Install zlib1g-dev in docker/Dockerfile.build and delete work/.py-host." + "$HOSTPY" -m pip --version >> "$_hplog" 2>&1 || pkg_die \ + "the build-python has no pip -- ensurepip failed inside 'make install'; its traceback is in $_hplog" + + # The PEP 517 backends, into the build-python rather than into an isolated + # environment per wheel: --no-build-isolation is what keeps + # _PYTHON_SYSCONFIGDATA_NAME reaching setup.py, and with isolation off the + # backends have to already be here. All three bootstrap themselves through + # backend-path, so --no-index proves nothing outside vendor/ is needed. + for _p in setuptools $PYPKG_HOST_LIST; do + _sd=$(pypkg_tgz "$_p") + [ -f "$_sd" ] || pkg_die \ + "no sdist for the build backend $_p at '$_sd' -- run ./bin/fetch-assets.sh" + ( + set -e + unset CC CXX AR RANLIB STRIP NM OBJCOPY OBJDUMP LD + unset CFLAGS CXXFLAGS CPPFLAGS LDFLAGS LIBS CONFIG_SITE + unset _PYTHON_SYSCONFIGDATA_NAME PYTHONPATH + "$HOSTPY" -m pip install --quiet --no-index --no-cache-dir \ + --no-build-isolation --no-deps "$_sd" + ) >> "$_hplog" 2>&1 || pkg_die \ + "$PKG_ID: could not install the build backend $_p -- see $_hplog" + done + + rm -rf work/.py-hostsrc + # Written LAST, so an interrupted build leaves a cache that is stale rather + # than one that lies. + echo "$_hpstamp" > "$PKG_HOSTPY_ROOT/.version" + pkg_say "$PKG_ID: build-python ready ($("$HOSTPY" -V 2>&1))" +} + # ----------------------------------------------------------------- pkg_deps # # Fill the recipe's sysroot from the feed: every recipe named in @@ -581,6 +699,215 @@ pkg_build() { pkg_say "$PKG_ID: built $_tag" } +# ------------------------------------------------------------- pkg_pytarget +# +# pkg_pytarget +# +# Point the build-python's sysconfig at the TARGET interpreter in the sysroot, +# so that pip and setuptools -- running on x86-64 -- answer every question +# about mipsel. Call it after pkg_toolchain, pkg_deps and pkg_buildpython. +# +# THE CROSS TRICK, AND THERE IS NO crossenv IN IT. anvil-python ships +# lib/python3.13/_sysconfigdata__linux_mipsel-linux-gnu.py, which records the +# cross CC, LDSHARED, EXT_SUFFIX (.cpython-313-mipsel-linux-gnu.so) and +# INCLUDEPY for the TARGET. Point _PYTHON_SYSCONFIGDATA_NAME at it from an +# x86-64 CPython of the same version and sysconfig -- and therefore +# setuptools' build_ext -- builds for mipsel while running on x86-64. +# +# WHY THE MODULE IS REWRITTEN RATHER THAN USED AS IT SHIPS. Every path in it +# is /usr/data/anvil/..., which is where those files live ON THE PRINTER and +# nowhere on this machine; INCLUDEPY in particular has to name a directory +# that exists or every C extension fails to find Python.h. The spike solved +# that with a container and a bind mount. Here the whole module is rewritten +# with $MODDIR replaced by the sysroot that pkg_deps just filled -- a blanket +# substitution over every string value rather than a list of the variables +# that matter, because the list of variables that matter is exactly the thing +# nobody gets right by hand: INCLUDEPY fails loudly, LIBDIR and LIBPL fail +# quietly, and a future setuptools is free to ask for another. +# +# It goes first on PYTHONPATH, is read only by the build, and never ships. +# +# THE PURE-PYTHON RECIPES CALL THIS TOO, and there is nothing for it to +# cross-compile in them. That is deliberate: sysconfig is what a wheel build +# asks for its tags and its paths whether or not there is any C in the package, +# so running the same five verbs in all eighteen recipes means none of them has +# to decide which kind it is -- and a package that grows a C accelerator in +# some future release does not need its recipe rewritten to notice. +pkg_pytarget() { + [ -n "${HOSTPY:-}" ] || pkg_die \ + "$PKG_ID: pkg_pytarget needs the build-python -- call pkg_buildpython first" + [ -n "${PKG_HOST:-}" ] || pkg_die \ + "$PKG_ID: pkg_pytarget needs the cross toolchain -- call pkg_toolchain first" + + PKG_PYROOT="$PKG_SYSROOT$MODDIR" + PKG_PYINC="$PKG_PYROOT/include/python$PY_MM" + _sc="_sysconfigdata__linux_mipsel-linux-gnu" + _scsrc="$PKG_PYROOT/lib/python$PY_MM/$_sc.py" + [ -f "$_scsrc" ] || pkg_die \ + "$PKG_ID: no $_sc in the sysroot -- is 'python' in PKG_BUILD_DEPENDS?" + [ -d "$PKG_PYINC" ] || pkg_die \ + "$PKG_ID: no target headers at $PKG_PYINC -- anvil-python-dev is what carries them" + + _xsys="$PWD/$PKG_WORK/xsysconfig" + rm -rf "$_xsys"; mkdir -p "$_xsys" + "$HOSTPY" - "$_scsrc" "$_xsys/$_sc.py" "$MODDIR" "$PKG_PYROOT" <<'PYEOF' \ + || pkg_die "$PKG_ID: could not rewrite $_sc for the sysroot" +import sys +src, dst, prefix, sysroot = sys.argv[1:5] +ns = {} +exec(compile(open(src).read(), src, "exec"), ns) +out = {k: (v.replace(prefix, sysroot) if isinstance(v, str) else v) + for k, v in ns["build_time_vars"].items()} +with open(dst, "w") as fh: + fh.write("# Generated by pkg/lib.sh: %s with the build-time prefix rewritten\n" + "# to this recipe's sysroot. Build-time only; never ships.\n" + "build_time_vars = %r\n" % (src, out)) +PYEOF + + export _PYTHON_SYSCONFIGDATA_NAME="$_sc" + # ONLY the rewritten module -- deliberately NOT the target's stdlib as + # well, which is what the spike put here. They are the same CPython, so it + # did no harm, but it also means the host interpreter importing the + # TARGET's os.py by accident is one path-ordering mistake away, and the + # only thing that has to be importable is this. + export PYTHONPATH="$_xsys" + export PYTHONDONTWRITEBYTECODE=1 + export PIP_DISABLE_PIP_VERSION_CHECK=1 + # setuptools does NOT forward CFLAGS to its link lines, which is why the + # -EL -mnan=2008 that decide this printer's ABI are baked into the gcc + # wrapper pkg_toolchain wrote and not passed here. What is passed here is + # only what a compile needs to find headers. + export LDSHARED="$PKG_HOST-gcc -shared -L$PKG_PYROOT/lib" + export CFLAGS="-O2 -fPIC -D_FILE_OFFSET_BITS=64 -I$PKG_PYROOT/include -I$PKG_PYINC" + export CXXFLAGS="$CFLAGS" + + # Where every wheel is unpacked: the recipe's own staging tree, at the + # path site-packages has on the printer. pkg_ship reads from here. + PKG_PYSP="$PWD/$PKG_WORK/stage$MODDIR/lib/python$PY_MM/site-packages" + mkdir -p "$PKG_PYSP" + + # GATED BEFORE ANYTHING IS BUILT ON IT. If _PYTHON_SYSCONFIGDATA_NAME has + # not taken, every question sysconfig is asked below is answered for + # x86-64 -- and the answer is a tree of host objects that builds + # perfectly, passes every test here and imports nowhere. + "$HOSTPY" - "$PKG_PYINC" ".cpython-$(printf '%s' "$PY_MM" | tr -d .)-mipsel-linux-gnu.so" <<'PYEOF' \ + || pkg_die "$PKG_ID: the sysconfig cross trick did not take" +import os, sys, sysconfig +inc_want, suffix_want = sys.argv[1:3] +suffix = sysconfig.get_config_var("EXT_SUFFIX") +inc = sysconfig.get_config_var("INCLUDEPY") +if suffix != suffix_want: + raise SystemExit(" !! sysconfig is answering for the HOST (EXT_SUFFIX=%s," + " wanted %s) -- _PYTHON_SYSCONFIGDATA_NAME did not take" + % (suffix, suffix_want)) +if inc != inc_want: + raise SystemExit(" !! INCLUDEPY is %s, expected %s" % (inc, inc_want)) +if not os.path.isdir(inc): + raise SystemExit(" !! INCLUDEPY %s does not exist" % inc) +PYEOF + pkg_say "$PKG_ID: sysconfig answers for mipsel, headers at $PKG_PYINC" +} + +# ---------------------------------------------------------------- pkg_pysrc +# +# pkg_pysrc +# +# Where this package's sdist unpacked to. Every sdist on PyPI unpacks to a +# directory named after the file with .tar.gz removed, and pkg_unpack put it +# under $PKG_WORK/src -- so this is that path, spelled once instead of in +# eighteen recipes. +pkg_pysrc() { + _f=$(pypkg_var "$1" FILE) + printf '%s/src/%s' "$PKG_WORK" "${_f%.tar.gz}" +} + +# -------------------------------------------------------------- pkg_pywheel +# +# pkg_pywheel [VAR=VAL ...] +# +# Build one wheel for the target out of the source pkg_unpack just extracted, +# and unpack it into the staging site-packages. Trailing VAR=VAL pairs are +# environment for that build alone (lmdb needs one). +# +# IT TAKES THE PACKAGE NAME AND NOT A PATH so that every recipe reads the same +# way and none of them spells $PKG_WORK. It builds from the UNPACKED +# DIRECTORY rather than from the sdist for the same reason: greenlet has to +# patch its sources before they compile and pillow has to be driven through +# setup.py, and if those two took a directory while the other sixteen took a +# tarball then two thirds of the recipes would differ in shape for no reason a +# reader could see. pip builds a directory exactly as it builds an sdist. +# +# NOTHING HERE TALKS TO A NETWORK. pip runs --no-index against a tree that came +# out of an archive bin/fetch-assets.sh already checked the sha256 of, which is +# the strong form of the rule that --no-binary :all: is the weak form of: pip +# cannot download a wheel it cannot reach. Both are kept, because the day +# someone adds an index URL for one awkward package the second one is what +# stops three x86-64 .so files from sailing into the tree -- which is not +# hypothetical, it is what happened on the spike's first run. +# +# PKG_PY_SETUP_ARGS drive setup.py build_ext with these options and then +# bdist_wheel, instead of calling pip. PEP 517 offers no +# way to pass build_ext options, and pillow's --disable-* +# are build_ext options. +# +# UNZIPPED RATHER THAN pip-installed, because `pip install` insists on +# installing FOR the interpreter running it -- and that one is x86-64. A wheel +# is a zip whose layout is already the layout of site-packages, so unzipping it +# is the whole of what an install does here. +# +# .dist-info GOES: nothing on the printer resolves a dependency, asks for a +# version or runs an entry point, and it is metadata describing a build machine +# to nobody. The .data directories and the scripts in them go for a stronger +# reason -- their console-script shebangs name the BUILD-PYTHON's path, so +# shipping them would put files on the printer that reference work/.py-host. +pkg_pywheel() { + _pw=$1; shift + _wdir=$(pkg_pysrc "$_pw") + [ -d "$_wdir" ] || pkg_die \ + "$PKG_ID: $(pypkg_var "$_pw" FILE) did not unpack to $(basename "$_wdir")" + [ -n "${PKG_PYSP:-}" ] || pkg_die \ + "$PKG_ID: pkg_pywheel needs the target environment -- call pkg_pytarget first" + _wheels="$PWD/$PKG_WORK/wheels" + rm -rf "$_wheels"; mkdir -p "$_wheels" + _wlog="$PKG_LOG/wheel-$_pw.log" + + if [ -n "${PKG_PY_SETUP_ARGS:-}" ]; then + ( + set -e + cd "$_wdir" + # Deliberately unquoted: PKG_PY_SETUP_ARGS is a list of options. + # shellcheck disable=SC2086 + env "$@" "$HOSTPY" setup.py build_ext $PKG_PY_SETUP_ARGS bdist_wheel + ) > "$_wlog" 2>&1 || { + printf ' !! %s: %s failed to build -- see %s\n' "$PKG_ID" "$_pw" "$_wlog" >&2 + tail -25 "$_wlog" | sed 's/^/ /' >&2 + return 1; } + cp -a "$_wdir"/dist/*.whl "$_wheels/" \ + || pkg_die "$PKG_ID: $_pw's setup.py produced no wheel -- see $_wlog" + else + env "$@" "$HOSTPY" -m pip wheel --no-deps --no-build-isolation \ + --no-index --no-binary :all: --no-cache-dir -w "$_wheels" "$_wdir" \ + > "$_wlog" 2>&1 || { + printf ' !! %s: %s failed to build -- see %s\n' "$PKG_ID" "$_pw" "$_wlog" >&2 + tail -25 "$_wlog" | sed 's/^/ /' >&2 + return 1; } + fi + + _nw=0 + for _w in "$_wheels"/*.whl; do + [ -f "$_w" ] || continue + "$HOSTPY" -m zipfile -e "$_w" "$PKG_PYSP" \ + || pkg_die "$PKG_ID: could not unpack $(basename "$_w") into site-packages" + pkg_say "$PKG_ID: $(basename "$_w")" + _nw=$((_nw + 1)) + done + [ "$_nw" -gt 0 ] || pkg_die "$PKG_ID: no wheel came out of $_pw -- see $_wlog" + + find "$PKG_PYSP" -name '__pycache__' -prune -exec rm -rf {} + 2>/dev/null || true + find "$PKG_PYSP" -maxdepth 1 -name '*.dist-info' -prune -exec rm -rf {} + 2>/dev/null || true + rm -rf "${PKG_PYSP:?}"/bin "${PKG_PYSP:?}"/*.data +} + # --------------------------------------------------------------- pkg_ship # # pkg_ship [...] diff --git a/pkg/moonraker/pkg.conf b/pkg/moonraker/pkg.conf index 5e04095..ec1268c 100644 --- a/pkg/moonraker/pkg.conf +++ b/pkg/moonraker/pkg.conf @@ -23,15 +23,24 @@ PKG_SECTION=net PKG_EXCLUDE=".version" -# NO Depends YET, AND THAT IS TEMPORARY -- worth writing down so it is fixed -# rather than forgotten. Moonraker runs on the CPython 3.13 this repo builds -# (payload/anvil-env.sh points FF_PYTHON at it) and reaches libsodium through -# libnacl's ctypes fallback. Both of those are packages: anvil-python and -# anvil-libsodium. The first does not exist yet -- CPython is the next chunk of -# the migration -- and naming a dependency the feed cannot satisfy is an -# install that refuses itself. This gains `Depends: anvil-python, -# anvil-libsodium` in the same change that adds the python recipes. -PKG_DEPENDS="" +# WHAT MOONRAKER ACTUALLY NEEDS, now that the feed can satisfy it. It runs on +# the CPython 3.13 this repo builds (payload/anvil-env.sh points FF_PYTHON at +# it) and imports thirteen third-party packages, each of which is its own .ipk. +# +# THE LIST IS moonraker-requirements.txt MINUS WHAT MOONRAKER DOES NOT IMPORT. +# Not on it: anvil-python-pillow, which only the metadata SUBPROCESS uses, so a +# printer without it serves fine and shows no gcode thumbnails; and +# anvil-python-preprocess-cancellation, imported lazily in that same +# subprocess. Both are Recommends in spirit -- opkg has no such field, and +# putting them in Depends would make a thumbnail library non-optional. +# anvil-libsodium is not here either: libnacl declares it, which is where a +# ctypes dependency belongs. +PKG_DEPENDS="anvil-python, anvil-python-tornado, anvil-python-lmdb, + anvil-python-streaming-form-data, anvil-python-distro, + anvil-python-inotify-simple, anvil-python-libnacl, + anvil-python-dbus-next, anvil-python-jinja2, + anvil-python-pyserial, anvil-python-pyserial-asyncio, + anvil-python-smart-open, anvil-python-setuptools" PKG_BUILD_DEPENDS="" PKG_WHEN='[ "${BUILD_MOONRAKER:-1}" = "1" ]' diff --git a/pkg/python-cffi/build.sh b/pkg/python-cffi/build.sh new file mode 100755 index 0000000..3023514 --- /dev/null +++ b/pkg/python-cffi/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# cffi -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-cffi || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz cffi)" +pkg_pywheel cffi +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-cffi/pkg.conf b/pkg/python-cffi/pkg.conf new file mode 100644 index 0000000..6c5cf1d --- /dev/null +++ b/pkg/python-cffi/pkg.conf @@ -0,0 +1,20 @@ +# anvil-python-cffi -- for the CPython this feed builds. +# +# klippy's chelper, and the package that justifies the whole glibc decision: +# this is what dlopens FlashForge's own c_helper.so, measured. It links the +# static -fPIC libffi this feed already builds for the interpreter's _ctypes. +# +# THE PIN IS IN versions.env, as PYPKG_CFFI_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-cffi +PKG_VERSION="$(pypkg_version cffi)" +PKG_RELEASE=1 +PKG_SECTION=python + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python, anvil-python-pycparser" +PKG_BUILD_DEPENDS="python libffi" + +PKG_DESCRIPTION="cffi -- C foreign function interface. What klippy's chelper loads FlashForge's c_helper.so through." diff --git a/pkg/python-dbus-next/build.sh b/pkg/python-dbus-next/build.sh new file mode 100755 index 0000000..27aa3d1 --- /dev/null +++ b/pkg/python-dbus-next/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# dbus_next -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-dbus-next || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz dbus-next)" +pkg_pywheel dbus-next +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-dbus-next/pkg.conf b/pkg/python-dbus-next/pkg.conf new file mode 100644 index 0000000..169238b --- /dev/null +++ b/pkg/python-dbus-next/pkg.conf @@ -0,0 +1,22 @@ +# anvil-python-dbus-next -- for the CPython this feed builds. +# +# Moonraker's dbus_manager component. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_DBUS_NEXT_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-dbus-next +PKG_VERSION="$(pypkg_version dbus-next)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="dbus_next -- a pure-python DBus library. Moonraker's dbus_manager uses it." diff --git a/pkg/python-distro/build.sh b/pkg/python-distro/build.sh new file mode 100755 index 0000000..16d83ed --- /dev/null +++ b/pkg/python-distro/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# distro -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-distro || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz distro)" +pkg_pywheel distro +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-distro/pkg.conf b/pkg/python-distro/pkg.conf new file mode 100644 index 0000000..ddbaf43 --- /dev/null +++ b/pkg/python-distro/pkg.conf @@ -0,0 +1,22 @@ +# anvil-python-distro -- for the CPython this feed builds. +# +# Moonraker reads it to identify the OS it is running on. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_DISTRO_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-distro +PKG_VERSION="$(pypkg_version distro)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="distro -- OS release identification. Pure python; Moonraker imports it at startup." diff --git a/pkg/python-greenlet/build.sh b/pkg/python-greenlet/build.sh new file mode 100755 index 0000000..2174ee7 --- /dev/null +++ b/pkg/python-greenlet/build.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# greenlet -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-greenlet || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz greenlet)" + +# gcc 7.2 refuses greenlet 3.x's C++ designated initializers wherever they skip +# a field. The rule was probed, not assumed: in-order contiguous is fine, a +# TRAILING gap is fine, an interior or leading gap is refused. +# fill-designators.py writes the skipped fields back as explicit zeros, taking +# field order from the TARGET's own headers and refusing to guess about a +# designator that is not in them. These are objects of static storage duration, +# so every field it inserts was already zero: it changes the spelling, not the +# program. 55 fields across two files. +_g=$(pkg_pysrc greenlet) +# Deliberately unquoted: the find is a LIST of sources to patch, one argument +# each, and none of greenlet's file names has ever contained a space. +# shellcheck disable=SC2046 +"$HOSTPY" tools/python-packages/fill-designators.py \ + "$PKG_PYINC" \ + $(find "$_g/src/greenlet" -name '*.cpp' -o -name '*.hpp' | sort) \ + > "$PKG_LOG/fill-designators.log" 2>&1 \ + || pkg_die "greenlet: fill-designators.py failed -- see $PKG_WORK/fill-designators.log" + +pkg_pywheel greenlet +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-greenlet/pkg.conf b/pkg/python-greenlet/pkg.conf new file mode 100644 index 0000000..dfc94da --- /dev/null +++ b/pkg/python-greenlet/pkg.conf @@ -0,0 +1,25 @@ +# anvil-python-greenlet -- for the CPython this feed builds. +# +# klippy's reactor. THE MIPS STACK-SWITCHING ASSEMBLY WAS NEVER THE PROBLEM; +# C++ designated initializers were. greenlet 3.x writes its PyTypeObjects as +# designated initializers, which gcc 7.2 -- the only compiler that produces +# this printer's ABI -- accepts only in declaration order with no gaps, and +# refuses anything else with "sorry, unimplemented: non-trivial designated +# initializers not supported". build.sh rewrites the skipped fields as explicit +# zeros first. Bumping this pin means re-running that, and it fails loudly +# rather than quietly. +# +# THE PIN IS IN versions.env, as PYPKG_GREENLET_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-greenlet +PKG_VERSION="$(pypkg_version greenlet)" +PKG_RELEASE=1 +PKG_SECTION=python + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="greenlet -- lightweight coroutines, klippy's reactor. Its C++ sources are rewritten for gcc 7.2 before they are compiled." diff --git a/pkg/python-inotify-simple/build.sh b/pkg/python-inotify-simple/build.sh new file mode 100755 index 0000000..fbcf187 --- /dev/null +++ b/pkg/python-inotify-simple/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# inotify_simple -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-inotify-simple || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz inotify-simple)" +pkg_pywheel inotify-simple +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-inotify-simple/pkg.conf b/pkg/python-inotify-simple/pkg.conf new file mode 100644 index 0000000..4e29187 --- /dev/null +++ b/pkg/python-inotify-simple/pkg.conf @@ -0,0 +1,22 @@ +# anvil-python-inotify-simple -- for the CPython this feed builds. +# +# Moonraker's file_manager watches the gcode directory through it. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_INOTIFY_SIMPLE_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-inotify-simple +PKG_VERSION="$(pypkg_version inotify-simple)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="inotify_simple -- a thin wrapper over inotify. Pure python; Moonraker's file manager watches the gcode directory with it." diff --git a/pkg/python-jinja2/build.sh b/pkg/python-jinja2/build.sh new file mode 100755 index 0000000..86f5270 --- /dev/null +++ b/pkg/python-jinja2/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Jinja2 -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-jinja2 || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz jinja2)" +pkg_pywheel jinja2 +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-jinja2/pkg.conf b/pkg/python-jinja2/pkg.conf new file mode 100644 index 0000000..d967971 --- /dev/null +++ b/pkg/python-jinja2/pkg.conf @@ -0,0 +1,22 @@ +# anvil-python-jinja2 -- for the CPython this feed builds. +# +# Template rendering for Moonraker and for klippy's gcode macros. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_JINJA2_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-jinja2 +PKG_VERSION="$(pypkg_version jinja2)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python, anvil-python-markupsafe" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="Jinja2 -- the template engine Moonraker and klippy's gcode macros render through." diff --git a/pkg/python-libnacl/build.sh b/pkg/python-libnacl/build.sh new file mode 100755 index 0000000..0a6f9a7 --- /dev/null +++ b/pkg/python-libnacl/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# libnacl -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-libnacl || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz libnacl)" +pkg_pywheel libnacl +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-libnacl/pkg.conf b/pkg/python-libnacl/pkg.conf new file mode 100644 index 0000000..40388d5 --- /dev/null +++ b/pkg/python-libnacl/pkg.conf @@ -0,0 +1,26 @@ +# anvil-python-libnacl -- for the CPython this feed builds. +# +# IT REACHES libsodium THROUGH ctypes, which is why anvil-libsodium is a +# runtime dependency and not a build one: there is nothing to link, and the +# .so has to be findable at import time or Moonraker's ed25519 signing fails. +# libnacl constructs the library name with a dlopen fallback, so the SONAME +# symlinks anvil-libsodium ships are load-bearing. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_LIBNACL_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-libnacl +PKG_VERSION="$(pypkg_version libnacl)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python, anvil-libsodium" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="libnacl -- ctypes bindings for libsodium. Pure python; the library it binds is anvil-libsodium." diff --git a/pkg/python-lmdb/build.sh b/pkg/python-lmdb/build.sh new file mode 100755 index 0000000..1bd3f5e --- /dev/null +++ b/pkg/python-lmdb/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# lmdb -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-lmdb || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz lmdb)" + +# LMDB_FORCE_CPYTHON=1 IS LOAD-BEARING. lmdb's setup.py has two backends: a +# real CPython extension and a cffi one that ships mdb.c and COMPILES IT AT +# IMPORT TIME. Left to itself here it picks cffi and produces a py3-none-any +# wheel with mdb.c inside -- i.e. a Moonraker that tries to invoke a compiler +# on a printer that has none. Do not "fix" this with LMDB_FORCE_CFFI=0: +# setup.py tests the variable for PRESENCE, so the string "0" selects cffi. +pkg_pywheel lmdb LMDB_FORCE_CPYTHON=1 + +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-lmdb/pkg.conf b/pkg/python-lmdb/pkg.conf new file mode 100644 index 0000000..9dceda8 --- /dev/null +++ b/pkg/python-lmdb/pkg.conf @@ -0,0 +1,20 @@ +# anvil-python-lmdb -- for the CPython this feed builds. +# +# Moonraker's database AT THE PINNED COMMIT, and the package that disappears +# the moment MOONRAKER_VERSION moves past 80c7620 (lmdb -> sqlite), which is +# the whole reason for having built an interpreter with _sqlite3. +# +# THE PIN IS IN versions.env, as PYPKG_LMDB_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-lmdb +PKG_VERSION="$(pypkg_version lmdb)" +PKG_RELEASE=1 +PKG_SECTION=python + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="lmdb -- the memory-mapped database Moonraker keeps its state in, built as a real CPython extension rather than the cffi backend that would compile at import time." diff --git a/pkg/python-markupsafe/build.sh b/pkg/python-markupsafe/build.sh new file mode 100755 index 0000000..6acc09a --- /dev/null +++ b/pkg/python-markupsafe/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# MarkupSafe -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-markupsafe || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz markupsafe)" +pkg_pywheel markupsafe +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-markupsafe/pkg.conf b/pkg/python-markupsafe/pkg.conf new file mode 100644 index 0000000..20679a8 --- /dev/null +++ b/pkg/python-markupsafe/pkg.conf @@ -0,0 +1,20 @@ +# anvil-python-markupsafe -- for the CPython this feed builds. +# +# jinja2's escape accelerator, on the path of every gcode macro klippy +# renders. A pure-python fallback (_native.py) ships inside the package if the +# extension ever fights. +# +# THE PIN IS IN versions.env, as PYPKG_MARKUPSAFE_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-markupsafe +PKG_VERSION="$(pypkg_version markupsafe)" +PKG_RELEASE=1 +PKG_SECTION=python + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="MarkupSafe -- jinja2's escaping accelerator, with its C speedup built for mipsel." diff --git a/pkg/python-pillow/build.sh b/pkg/python-pillow/build.sh new file mode 100755 index 0000000..8d33c47 --- /dev/null +++ b/pkg/python-pillow/build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Pillow with zlib and nothing else -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-pillow || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz pillow)" + +# PIP CANNOT DRIVE THIS ONE: the --disable-* flags are build_ext options and +# PEP 517 offers no way to pass them, so PKG_PY_SETUP_ARGS calls setup.py +# directly. --disable-platform-guessing is the important one -- without it +# pillow probes for jpeg, tiff and the rest by trying to LINK against the build +# machine's copies. +PKG_PY_SETUP_ARGS="--disable-jpeg --disable-tiff --disable-webp + --disable-jpeg2000 --disable-imagequant --disable-lcms --disable-freetype + --disable-xcb --disable-platform-guessing --enable-zlib + -I$PKG_SYSROOT$MODDIR/include -L$PKG_SYSROOT$MODDIR/lib" +pkg_pywheel pillow + +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-pillow/pkg.conf b/pkg/python-pillow/pkg.conf new file mode 100644 index 0000000..4be7993 --- /dev/null +++ b/pkg/python-pillow/pkg.conf @@ -0,0 +1,26 @@ +# anvil-python-pillow -- for the CPython this feed builds. +# +# ZLIB ONLY, AND THAT IS THE WHOLE CONFIGURATION. Moonraker's only use of +# pillow is components/file_manager/metadata.py, which runs as a SUBPROCESS to +# pull thumbnails out of gcode -- and gcode thumbnails are base64 PNG, so zlib +# is the only codec on that path. jpeg, tiff, webp, jpeg2000, lcms, freetype +# and imagequant are libraries nobody has cross-built here, and pillow probes +# for them by trying to LINK against the HOST's copies unless told not to. +# +# Not on Moonraker's startup path: without this package Moonraker runs and +# Mainsail shows no thumbnails. +# +# THE PIN IS IN versions.env, as PYPKG_PILLOW_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-pillow +PKG_VERSION="$(pypkg_version pillow)" +PKG_RELEASE=1 +PKG_SECTION=python + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python zlib" + +PKG_DESCRIPTION="Pillow with zlib and nothing else -- enough to read the base64 PNG thumbnails out of gcode, which is all Moonraker asks of it." diff --git a/pkg/python-preprocess-cancellation/build.sh b/pkg/python-preprocess-cancellation/build.sh new file mode 100755 index 0000000..b72598d --- /dev/null +++ b/pkg/python-preprocess-cancellation/build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# preprocess_cancellation -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-preprocess-cancellation || exit 0 +pkg_unpack "$(pypkg_tgz preprocess-cancellation)" + +# COPIED, NOT BUILT. This sdist declares no [build-system] and ships no +# setup.py, so there is no backend to call -- pip's legacy fallback would +# auto-discover a flat layout and emit a wheel named UNKNOWN-0.0.0 containing +# the one module, which is the right file arrived at by an accident a future +# setuptools is free to change. The package IS one module, so the honest +# operation is a copy. (The same release also declares version 0.0.0 in its own +# metadata, so a version specifier would be refused by pip even if there were +# something to build.) +pkg_stage "$(pkg_pysrc preprocess-cancellation)/preprocess_cancellation.py" \ + "lib/python$PY_MM/site-packages/preprocess_cancellation.py" + +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-preprocess-cancellation/pkg.conf b/pkg/python-preprocess-cancellation/pkg.conf new file mode 100644 index 0000000..bd9057f --- /dev/null +++ b/pkg/python-preprocess-cancellation/pkg.conf @@ -0,0 +1,25 @@ +# anvil-python-preprocess-cancellation -- for the CPython this feed builds. +# +# NO BUILD DEPENDENCY ON python, AND THAT IS THE HONEST ANSWER rather than a +# missing line: this package is one .py file copied into site-packages. It +# compiles nothing, so it needs no interpreter to build against -- only one to +# run on, which is what Depends says. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_PREPROCESS_CANCELLATION_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-preprocess-cancellation +PKG_VERSION="$(pypkg_version preprocess-cancellation)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="" + +PKG_DESCRIPTION="preprocess_cancellation -- gcode object cancellation. One module, copied rather than built; Moonraker imports it lazily in the metadata subprocess." diff --git a/pkg/python-pycparser/build.sh b/pkg/python-pycparser/build.sh new file mode 100755 index 0000000..8ab4bcf --- /dev/null +++ b/pkg/python-pycparser/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# pycparser -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-pycparser || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz pycparser)" +pkg_pywheel pycparser +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-pycparser/pkg.conf b/pkg/python-pycparser/pkg.conf new file mode 100644 index 0000000..7bfdef4 --- /dev/null +++ b/pkg/python-pycparser/pkg.conf @@ -0,0 +1,22 @@ +# anvil-python-pycparser -- for the CPython this feed builds. +# +# cffi's only runtime dependency; it parses the C declarations cffi is given. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_PYCPARSER_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-pycparser +PKG_VERSION="$(pypkg_version pycparser)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="pycparser -- a C parser in pure python. cffi's only runtime dependency." diff --git a/pkg/python-pyserial-asyncio/build.sh b/pkg/python-pyserial-asyncio/build.sh new file mode 100755 index 0000000..99f39f5 --- /dev/null +++ b/pkg/python-pyserial-asyncio/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# pyserial-asyncio -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-pyserial-asyncio || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz pyserial-asyncio)" +pkg_pywheel pyserial-asyncio +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-pyserial-asyncio/pkg.conf b/pkg/python-pyserial-asyncio/pkg.conf new file mode 100644 index 0000000..11a3ec7 --- /dev/null +++ b/pkg/python-pyserial-asyncio/pkg.conf @@ -0,0 +1,22 @@ +# anvil-python-pyserial-asyncio -- for the CPython this feed builds. +# +# It is a transport layer over pyserial and imports it at module scope. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_PYSERIAL_ASYNCIO_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-pyserial-asyncio +PKG_VERSION="$(pypkg_version pyserial-asyncio)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python, anvil-python-pyserial" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="pyserial-asyncio -- asyncio transport over pyserial." diff --git a/pkg/python-pyserial/build.sh b/pkg/python-pyserial/build.sh new file mode 100755 index 0000000..331ce81 --- /dev/null +++ b/pkg/python-pyserial/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# pyserial -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-pyserial || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz pyserial)" +pkg_pywheel pyserial +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-pyserial/pkg.conf b/pkg/python-pyserial/pkg.conf new file mode 100644 index 0000000..a7df493 --- /dev/null +++ b/pkg/python-pyserial/pkg.conf @@ -0,0 +1,22 @@ +# anvil-python-pyserial -- for the CPython this feed builds. +# +# Serial transport for anything that talks to the MCU from python. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_PYSERIAL_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-pyserial +PKG_VERSION="$(pypkg_version pyserial)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="pyserial -- serial port access. Pure python." diff --git a/pkg/python-setuptools/build.sh b/pkg/python-setuptools/build.sh new file mode 100755 index 0000000..3268663 --- /dev/null +++ b/pkg/python-setuptools/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# setuptools and pkg_resources. Nothing in this feed imports them at runtime; they are here so that lmdb never reaches for its compile-at-import fallback. -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-setuptools || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz setuptools)" +pkg_pywheel setuptools +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-setuptools/pkg.conf b/pkg/python-setuptools/pkg.conf new file mode 100644 index 0000000..f35ce28 --- /dev/null +++ b/pkg/python-setuptools/pkg.conf @@ -0,0 +1,34 @@ +# anvil-python-setuptools -- for the CPython this feed builds. +# +# A RUNTIME DEPENDENCY OF NOTHING IN THIS FEED, and shipped anyway. It is the +# fix for a MEASURED failure: a tree without pkg_resources made lmdb fall back +# to its cffi path and try to invoke mips-linux-gnu-gcc ON THE PRINTER at +# Moonraker startup, and there is no compiler on a printer. Installing lmdb as +# a wheel rather than as an egg removes that path as well, so this is now belt +# and braces -- and 4.9MB of it, which makes it the one obvious package to +# drop if the data partition ever gets tight. A separate package is what makes +# that a decision somebody can take. +# +# The SAME sdist is also the host-side build backend for most of this feed: +# pkg_buildpython installs it into the throwaway x86-64 interpreter. One pin, +# both jobs, so the setuptools that generates the shipped .so files is the +# setuptools whose hash is checked in versions.env. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_SETUPTOOLS_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-setuptools +PKG_VERSION="$(pypkg_version setuptools)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="setuptools and pkg_resources. Nothing in this feed imports them at runtime; they are here so that lmdb never reaches for its compile-at-import fallback." diff --git a/pkg/python-smart-open/build.sh b/pkg/python-smart-open/build.sh new file mode 100755 index 0000000..cf45b62 --- /dev/null +++ b/pkg/python-smart-open/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# smart_open -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-smart-open || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz smart-open)" +pkg_pywheel smart-open +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-smart-open/pkg.conf b/pkg/python-smart-open/pkg.conf new file mode 100644 index 0000000..5090389 --- /dev/null +++ b/pkg/python-smart-open/pkg.conf @@ -0,0 +1,25 @@ +# anvil-python-smart-open -- for the CPython this feed builds. +# +# NOT OPTIONAL, WHATEVER THE NAME SUGGESTS, and it is not Moonraker that +# wants it: streaming-form-data's targets.py imports it at MODULE SCOPE, so +# `import streaming_form_data` raises ModuleNotFoundError without it. Measured +# on the replica, where every other package imported and that one did not. +# +# ARCHITECTURE all. Nothing in this package is compiled; it is python source +# and it runs on any interpreter that can import it. +# +# THE PIN IS IN versions.env, as PYPKG_SMART_OPEN_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-smart-open +PKG_VERSION="$(pypkg_version smart-open)" +PKG_RELEASE=1 +PKG_SECTION=python +PKG_ARCH=all + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="smart_open -- streamed file opening. Present because streaming_form_data imports it at module scope." diff --git a/pkg/python-streaming-form-data/build.sh b/pkg/python-streaming-form-data/build.sh new file mode 100755 index 0000000..7d2fb09 --- /dev/null +++ b/pkg/python-streaming-form-data/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# streaming_form_data -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-streaming-form-data || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz streaming-form-data)" +pkg_pywheel streaming-form-data +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-streaming-form-data/pkg.conf b/pkg/python-streaming-form-data/pkg.conf new file mode 100644 index 0000000..f30db4e --- /dev/null +++ b/pkg/python-streaming-form-data/pkg.conf @@ -0,0 +1,21 @@ +# anvil-python-streaming-form-data -- for the CPython this feed builds. +# +# Moonraker's upload parser, Cython-generated C. The pin is NOT the 1.13.0 +# moonraker-requirements.txt asks for: that release ships a pregenerated +# _parser.c calling the four-argument _PyLong_AsByteArray, which grew a fifth +# parameter in 3.13, so every integer conversion in the file fails to compile. +# +# THE PIN IS IN versions.env, as PYPKG_STREAMING_FORM_DATA_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-streaming-form-data +PKG_VERSION="$(pypkg_version streaming-form-data)" +PKG_RELEASE=1 +PKG_SECTION=python + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python, anvil-python-smart-open" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="streaming_form_data -- Moonraker's multipart upload parser, with its Cython-generated C built for mipsel." diff --git a/pkg/python-tornado/build.sh b/pkg/python-tornado/build.sh new file mode 100755 index 0000000..4027454 --- /dev/null +++ b/pkg/python-tornado/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Tornado -- built for the printer's CPython, into its site-packages. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python-tornado || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_pytarget +pkg_unpack "$(pypkg_tgz tornado)" +pkg_pywheel tornado +pkg_ship "lib/python$PY_MM/site-packages" +pkg_end diff --git a/pkg/python-tornado/pkg.conf b/pkg/python-tornado/pkg.conf new file mode 100644 index 0000000..3a32024 --- /dev/null +++ b/pkg/python-tornado/pkg.conf @@ -0,0 +1,20 @@ +# anvil-python-tornado -- for the CPython this feed builds. +# +# Moonraker's web server. Its C speedup is optional and its PUBLISHED WHEEL +# carries an x86-64 copy of it, which is exactly what pkg_pywheel's +# --no-binary :all: is for. +# +# THE PIN IS IN versions.env, as PYPKG_TORNADO_FILE, beside its sha256 and +# beside the other seventeen. PKG_VERSION is READ OUT OF that file name rather +# than written again here, so there is one string to bump and no way for the +# two to disagree. +PKG_NAME=anvil-python-tornado +PKG_VERSION="$(pypkg_version tornado)" +PKG_RELEASE=1 +PKG_SECTION=python + +PKG_EXCLUDE=".version" +PKG_DEPENDS="anvil-python" +PKG_BUILD_DEPENDS="python" + +PKG_DESCRIPTION="Tornado -- the web framework Moonraker serves on, with its C speedup built for mipsel." diff --git a/pkg/python/build.sh b/pkg/python/build.sh new file mode 100755 index 0000000..9e4709b --- /dev/null +++ b/pkg/python/build.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# CPython, cross-compiled for the printer against the seven libraries this +# feed already builds. +# +# THE SEVEN LIBRARIES ARE STATIC AND NONE OF THEM SHIPS WITH THIS PACKAGE. +# They come out of the feed as -dev packages -- headers and .a files -- and are +# linked into the interpreter and its extension modules, so the tree on the +# printer has no .so of ours to find at runtime: no LD_LIBRARY_PATH to get +# right, no chance of picking up one of FlashForge's /usr/prog copies (a real +# hazard -- /usr/prog carries libffi.so.8 and the rootfs carries libffi.so.7), +# and nothing to version-skew. The cost is a few MB of duplicated libcrypto +# between _ssl.so and _hashlib.so, which is a good trade at 30MB. +set -euo pipefail +. "$(dirname "$0")/../../bin/common.sh" +. pkg/lib.sh + +pkg_begin python || exit 0 +pkg_toolchain +pkg_deps +pkg_buildpython +pkg_unpack "$PY_TGZ" + +_sys="$PKG_SYSROOT$MODDIR" + +# Answers to the questions configure settles by COMPILING AND RUNNING a probe, +# which it cannot do when the target is mipsel and the builder is x86. Left +# unanswered these either stop configure or -- worse -- default to the +# conservative answer and produce a working interpreter with subtly wrong float +# and time behaviour. +cat > "$PKG_WORK/src/config.site" <<'EOF' +ac_cv_file__dev_ptmx=yes +ac_cv_file__dev_ptc=no +ac_cv_buggy_getaddrinfo=no +ac_cv_little_endian_double=yes +ac_cv_big_endian_double=no +ac_cv_mixed_endian_double=no +ac_cv_working_tzset=yes +ac_cv_have_long_long_format=yes +ac_cv_no_strict_aliasing=no +ac_cv_pthread_system_supported=yes +EOF +export CONFIG_SITE="$PWD/$PKG_WORK/src/config.site" +export CFLAGS="-O2 -D_FILE_OFFSET_BITS=64" + +# THE TRAP THAT COSTS A DAY IF YOU MISS IT, and the reason the gates at the +# bottom of this file exist: +# -latomic 64-bit atomics on mips32 are out-of-line calls into libatomic, +# and CPython 3.13's _Py_atomic_* on 64-bit types needs them. +# libatomic.so.1 IS on the printer's rootfs -- measured -- so this +# one is a runtime dependency, not a static link. +# -lm because libsqlite3 here is STATIC. A shared libsqlite3.so carries +# its own DT_NEEDED on libm; a libsqlite3.a does not. configure's +# `checking for sqlite3_bind_double in -lsqlite3` link probe then +# fails on undefined floor/log/pow, and CPython records _sqlite3 as +# "missing" AND CARRIES ON -- a probe failure, not a compile +# failure, so nothing in 400 lines of build output says why the one +# module this whole package exists for is absent. +export LIBS="-latomic -lm" +# And the same link line stated outright, bypassing pkg-config, so that -lm +# cannot be reordered out from under the probe. +export LIBSQLITE3_CFLAGS="-I$_sys/include" +export LIBSQLITE3_LIBS="-L$_sys/lib -lsqlite3 -lm" + +# --disable-shared: no libpython3.13.so to find at runtime, for the same reason +# the seven libraries are static. Extension modules resolve their Python +# symbols out of the interpreter's own dynamic symbol table, which CPython +# links with -export-dynamic precisely so that they can. +# --without-ensurepip: pip needs a network and a compiler; neither is on a +# printer, and a pip that half works is worse than none. The BUILD-python has +# the opposite answer for the opposite reason -- see pkg_buildpython. +# --disable-test-modules: the CPython test suite is a third of the tree and +# none of it runs here. +pkg_build "Python-$PY_VERSION" \ + --build=x86_64-linux-gnu \ + --with-build-python="$HOSTPY" \ + --disable-shared \ + --without-ensurepip \ + --disable-test-modules \ + --with-openssl="$_sys" \ + --with-system-expat + +# ------------------------------------------------------------------ the trim +# +# What goes, and none of it is fussiness: idlelib and turtledemo (nothing on a +# printer runs either), tkinter (there is no X11), share/ (man pages), every +# __pycache__ -- which is 12MB of .pyc for modules that will be imported once, +# if ever, and which the interpreter regenerates into /usr/data anyway if it +# ever wants them -- and libpython3.13.a, 35MB of static archive that exists to +# link a python that is not this one. +# +# It is done HERE rather than by PKG_EXCLUDE because these are paths, not +# names: PKG_EXCLUDE is a `find -name` sweep, and `-name test` over a CPython +# tree removes a dozen legitimate submodules along with the one directory +# meant. +_st="$PKG_WORK/stage$MODDIR" +rm -rf "$_st/lib/python$PY_MM/idlelib" \ + "$_st/lib/python$PY_MM/tkinter" \ + "$_st/lib/python$PY_MM/turtledemo" \ + "$_st/share" +rm -f "$_st/lib/libpython$PY_MM.a" \ + "$_st/lib/python$PY_MM/config-$PY_MM-"*/"libpython$PY_MM.a" +find "$_st" -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true + +# ------------------------------------------------------------------ the ship +# +# ONE BINARY OUT OF bin/, BY NAME. `make install` also leaves python3, idle3, +# pydoc3, python3-config and python3.13-config there; naming what ships is what +# keeps bin/python3 -- which would shadow FlashForge's interpreter on the PATH +# anvil-env.sh exports -- from ever reaching a printer. It is a stronger rule +# than the `rm` bin/patch.sh used to do, because a future CPython that installs +# one more launcher is silently excluded rather than silently included. +# +# site-packages rides along inside lib/python3.13/ and needs no line of its +# own: `make install` creates it empty, this package owns it, and the eighteen +# pkg/python-* packages drop their files into it. +pkg_ship "bin/python$PY_MM" "lib/python$PY_MM" "include" "lib/pkgconfig" + +# ----------------------------------------------------------------- the gates +# +# ONE PER BUILD DEPENDENCY, ASKED BY MODULE NAME. CPython does not fail when it +# cannot link a library: configure records the module as unavailable, `make` +# prints a line among four hundred others, and the install succeeds. So every +# one of the seven libraries in PKG_BUILD_DEPENDS is checked here by the module +# it was added for. Without this the failure mode is a package that installs +# cleanly and raises ImportError on a printer. +_dyn="$PKG_OUT/lib/python$PY_MM/lib-dynload" +for _m in _ssl _hashlib _sqlite3 zlib _ctypes _lzma _bz2 pyexpat; do + ls "$_dyn/$_m".*.so >/dev/null 2>&1 \ + || pkg_die "python: NO $_m MODULE. Its library link probe failed silently -- see $PKG_WORK/Python-$PY_VERSION-configure.log" +done +[ -x "$PKG_OUT/bin/python$PY_MM" ] \ + || pkg_die "python: no bin/python$PY_MM came out of the install" + +pkg_say "python: $(ls "$_dyn" | wc -l) stdlib extension modules, $(du -sh "$PKG_OUT/lib/python$PY_MM" | cut -f1) of library" +pkg_end diff --git a/pkg/python/pkg.conf b/pkg/python/pkg.conf new file mode 100644 index 0000000..98c09c4 --- /dev/null +++ b/pkg/python/pkg.conf @@ -0,0 +1,60 @@ +# anvil-python -- CPython, cross-compiled for the printer. +# +# WHY THERE IS A SECOND PYTHON ON THIS MACHINE AT ALL. FlashForge built its +# 3.8.2 without _sqlite3, and that single omission is what pins +# MOONRAKER_VERSION to a 2023 commit: every Moonraker from v0.9.0 on keeps its +# database in sqlite. This interpreter has a working sqlite3, which is what +# eventually unpins it. +# +# THE PREFIX IS COMPILED IN, exactly as it is for s6. sys.prefix and the stdlib +# search that follows from it are baked in at configure time, so moving the +# directory on the printer breaks the interpreter -- it has to be rebuilt, not +# renamed. That is also the argument for putting it in $MODDIR with everything +# else rather than somewhere private: there is one prefix to keep stable, not +# two. +# +# ONLY bin/python3.13 SHIPS, and the name matters. payload/anvil-env.sh +# PREPENDS $MODDIR/bin to PATH (s6-svscan execs s6-supervise by name off PATH, +# so it has to), so a bin/python3 of ours would sit AHEAD of FlashForge's for +# every process that sources it, and anything saying `python3` rather than +# "$FF_PYTHON" would silently change interpreter. `make install` creates that +# symlink, plus idle3, pydoc3 and the *-config scripts; build.sh names the one +# binary it ships and the rest are simply never copied out of the staging tree. +PKG_NAME=anvil-python +PKG_VERSION="$PY_VERSION" +PKG_RELEASE=1 +PKG_SECTION=lang + +PKG_EXCLUDE=".version" + +# NO Depends, AND THAT IS A MEASURED CLAIM RATHER THAN AN OVERSIGHT. Every C +# library this interpreter uses is linked in STATICALLY -- see build.sh -- so +# the only shared objects it needs are the printer's own libc, libm, libdl, +# libpthread and libatomic.so.1, all of which are on the stock rootfs. Nothing +# in this feed has to be installed first. +PKG_DEPENDS="" + +# SEVEN LIBRARIES, AND EACH ONE IS EXACTLY ONE stdlib MODULE. openssl is _ssl +# and _hashlib, sqlite is _sqlite3, zlib is zlib, libffi is _ctypes, xz is +# _lzma, bzip2 is _bz2, expat is pyexpat. build.sh gates all seven by name +# after the build, because CPython's answer to a library it cannot link is to +# record the module as absent and CARRY ON -- a silent success that produces an +# interpreter missing the one module this package exists for. +PKG_BUILD_DEPENDS="openssl sqlite zlib libffi xz bzip2 expat" + +# THE HEADERS ARE THE POINT OF THE DEV SPLIT HERE, more than anywhere else in +# this feed. The eighteen pkg/python-* recipes cross-compile C extensions +# against them, and before this package existed they reached them by building +# inside bin/patch.sh at the one moment an untrimmed staging tree happened to +# be on disk. A -dev package is that moment made durable: the headers are in +# the feed, a wheel recipe names `python` as a build dependency and gets them, +# and a printer -- which builds no extension modules -- installs none of it. +# +# config-3.13-* comes along because sysconfig's LIBPL points into it, and a +# sysconfig answering with a directory that does not exist is the kind of thing +# a future setuptools notices. libpython3.13.a is deleted in build.sh rather +# than shipped inside it: 35MB of static archive that nothing links. +PKG_DEV_FILES="include lib/pkgconfig lib/python$PY_MM/config-$PY_MM-*" +PKG_DEV_DESCRIPTION="CPython headers and build configuration for mipsel. This is what the pkg/python-* recipes compile their C extensions against; nothing installs it on a printer." + +PKG_DESCRIPTION="CPython $PY_VERSION for the printer, with sqlite3, ssl, lzma, bz2, ctypes and expat linked in statically. Installs as bin/python3.13 -- never as python3, which would shadow the stock interpreter on PATH." diff --git a/qa/static/test_ipk.py b/qa/static/test_ipk.py index 2a9b785..564a9d7 100644 --- a/qa/static/test_ipk.py +++ b/qa/static/test_ipk.py @@ -448,7 +448,7 @@ def test_the_package_and_the_payload_share_one_build(): # decided each of these ships both ways -- a list read off patch.sh would # agree with patch.sh by construction and assert nothing. staged = ("libsodium", "mainsail", "moonraker", "helixscreen", - "skalibs", "execline", "s6", "s6-rc", "anvil-core") + "skalibs", "execline", "s6", "s6-rc", "anvil-core", "python") for recipe in staged: assert "bash pkg/%s/build.sh" % recipe in patch, ( "bin/patch.sh does not run pkg/%s/build.sh -- the payload's copy " @@ -855,8 +855,9 @@ def test_no_cache_stamp_is_spelled_in_two_places(): assert '"$SKALIBS_VERSION $S6_VERSION' not in text, ( "bin/%s spells the s6 stamp out by hand" % name) assert '"$PY_VERSION $OPENSSL_VERSION' not in text, ( - "bin/%s spells the CPython stamp out instead of using $PY_STAMP " - "from bin/common.sh" % name) + "bin/%s spells the CPython stamp out by hand -- CPython is " + "pkg/python and pkg_stamp derives its key from PKG_BUILD_DEPENDS" + % name) # The fetcher must ask the recipes rather than compare a string it wrote # itself. This is what makes the whole class impossible for anything under @@ -865,3 +866,100 @@ def test_no_cache_stamp_is_spelled_in_two_places(): assert "pkg_needs" in fetch, ( "bin/fetch-assets.sh no longer asks pkg_needs -- it is back to " "deciding whether a recipe is stale by a rule of its own") + +def test_the_python_package_list_and_the_recipes_agree(): + """PYPKG_LIST and pkg/python-* are one list, and it is checked. + + versions.env carries the pin for each third-party python package and + bin/fetch-assets.sh downloads from that list, while what actually BUILDS + each one is a recipe directory. Those are two spellings of one set, and the + failure when they disagree is quiet in both directions: an entry added to + versions.env alone is fetched, hashed and never built, and a recipe added + alone has no source to build from. + + bin/patch.sh checks the first direction at build time, one entry at a time. + This checks both, all at once, without building anything. + """ + listed = set(_sh('printf "%s" "$PYPKG_LIST"').split()) + recipes = {d.name[len("python-"):] for d in (ROOT / "pkg").glob("python-*") + if d.is_dir()} + assert listed == recipes, ( + "PYPKG_LIST and the pkg/python-* recipes disagree.\n" + " in versions.env only: %s\n" + " in pkg/ only: %s" + % (sorted(listed - recipes) or "none", sorted(recipes - listed) or "none")) + + +def test_a_python_package_does_not_pin_its_version_twice(): + """The version comes out of the pinned file name, never written again. + + Every sdist is -.tar.gz and versions.env pins the FILE, so + the version is already there. A PKG_VERSION written out by hand in the + pkg.conf would be a second copy of a string nobody would think to update + together with the first -- and the result is a .ipk that claims a version + it does not contain, which nothing downstream can detect. + """ + for conf in sorted((ROOT / "pkg").glob("python-*/pkg.conf")): + name = conf.parent.name[len("python-"):] + text = conf.read_text() + want = 'PKG_VERSION="$(pypkg_version %s)"' % name + assert want in text, ( + "%s does not read its version from the pin: expected\n %s\n" + "so that versions.env stays the only place the version is written" + % (conf.relative_to(ROOT), want)) + + +def test_bin_patch_builds_every_python_package(): + """The eighteen recipes are run by patch.sh, in a loop over the same list. + + The other staged recipes appear in bin/patch.sh by name, and + test_the_package_and_the_payload_share_one_build checks each of those. The + python packages are too many for that and are driven from PYPKG_LIST + instead -- so what is checked here is that the loop exists and reaches the + recipe directory, rather than eighteen literal lines. + """ + patch = (ROOT / "bin" / "patch.sh").read_text() + assert 'for p in $PYPKG_LIST; do' in patch, ( + "bin/patch.sh no longer iterates PYPKG_LIST -- the python packages " + "are either not built or built from a list of their own") + assert 'bash "pkg/python-$p/build.sh"' in patch, ( + "bin/patch.sh iterates PYPKG_LIST but does not run pkg/python-$p/" + "build.sh, so the payload's copy and the packaged one are built by " + "different code") + +def test_every_declared_dependency_is_a_package_this_feed_builds(): + """A Depends the feed cannot satisfy is an install that refuses itself. + + opkg resolves Depends before it unpacks anything, so a package naming one + that does not exist does not install PARTIALLY -- it does not install at + all, and the error names the missing dependency rather than the recipe that + asked for it. Nothing in a build catches this: bin/build-packages.sh writes + whatever PKG_DEPENDS says into the control file, opkg-make-index copies it + into the index, and the first thing to notice is a printer. + + Checked from the pkg.conf files rather than from a built feed, so it runs + on a bare checkout. Names ending -dev are matched against the recipes that + set PKG_DEV_FILES, because that is what makes the second archive exist. + + Dependencies on the STOCK ROOTFS are a different thing and are deliberately + absent everywhere: anvil-python needs libatomic.so.1 and does not say so, + because opkg has no idea what FlashForge installed and would refuse a + package for want of a library that is already there. + """ + provided = set() + for conf in RECIPES: + recipe = conf.parent.name + name = _conf(recipe, "PKG_NAME") + provided.add(name) + if _conf(recipe, "PKG_DEV_FILES"): + provided.add(name + "-dev") + + for conf in RECIPES: + recipe = conf.parent.name + depends = _conf(recipe, "PKG_DEPENDS") + for dep in [d.strip().split()[0] for d in depends.split(",") if d.strip()]: + assert dep in provided, ( + "pkg/%s depends on '%s' and no recipe under pkg/ produces it. " + "opkg refuses the whole install rather than part of it, so " + "this is a package that cannot be installed at all." + % (recipe, dep)) diff --git a/test/ffsim/gates.py b/test/ffsim/gates.py index d7d233f..25d7931 100644 --- a/test/ffsim/gates.py +++ b/test/ffsim/gates.py @@ -95,14 +95,14 @@ def mcu_bringup(config, on_output=None): FF_PYTHON is our own cross-built CPython 3.13 now (payload/anvil-env.sh), not FlashForge's 3.8.2, so the interpreter under test is a build output -- - work/.py313 -- handed over as py.tgz exactly as case-python.sh receives + work/pkg/python* -- handed over as py.tgz exactly as case-python.sh receives one. Skips rather than degrading: there is no fallback interpreter worth testing this against, since nothing on the printer still launches ff_mcu_bringup.py on FlashForge's 3.8.2. """ tree = _python_tarball(config) if not tree: - raise Skip("nothing in work/.py313 -- run ./bin/patch.sh first") + raise Skip("nothing in work/pkg/python -- run ./bin/patch.sh first") replica = Replica.start(config, want_output=on_output) replica.run_case(_case(config, "case-mcu-bringup.sh"), packages={"py.tgz": tree}, on_output=on_output) @@ -112,13 +112,13 @@ def boot_screen(config, on_output=None): """Does the first-boot screen draw, on FF_PYTHON and its fb0? Hand-packed pixels and an interpreter FF_PYTHON resolves to -- since the - switch, our own cross-built CPython 3.13 (work/.py313), handed over as + switch, our own cross-built CPython 3.13 (work/pkg/python*), handed over as py.tgz. Skips rather than degrading, for the same reason as mcu_bringup above: FlashForge's 3.8.2 is not what draws this screen any more. """ tree = _python_tarball(config) if not tree: - raise Skip("nothing in work/.py313 -- run ./bin/patch.sh first") + raise Skip("nothing in work/pkg/python -- run ./bin/patch.sh first") replica = Replica.start(config, want_output=on_output) replica.run_case(_case(config, "case-boot-screen.sh"), packages={"py.tgz": tree}, on_output=on_output) @@ -167,7 +167,7 @@ def moonraker(config, on_output=None): """ prefix = _prefix_tarball(config) if not prefix: - raise Skip("nothing in work/.py313 or work/.sodium -- run ./bin/patch.sh first") + raise Skip("nothing in work/pkg/python or work/.sodium -- run ./bin/patch.sh first") packages = {"pref.tgz": prefix} s6 = _s6_tarball(config) if s6: @@ -256,7 +256,7 @@ def moonraker313_s6(config, on_output=None): raise Skip("nothing in work/pkg/s6 -- run ./bin/patch.sh first") prefix = _prefix_tarball(config) if not prefix: - raise Skip("nothing in work/.py313 or work/.sodium -- run ./bin/patch.sh first") + raise Skip("nothing in work/pkg/python or work/.sodium -- run ./bin/patch.sh first") tree = _moonraker_tarball(config) if not tree: raise Skip("no Moonraker tarball in vendor/ -- run ./bin/fetch-assets.sh first") @@ -271,28 +271,28 @@ def _prefix_tarball(config): """The mod's own /usr/data/anvil prefix: interpreter, stdlib, packages and libsodium, in one tarball because it is one prefix. - _python_tarball below packs work/.py313 alone, which is right for - case-python.sh -- that case is about the interpreter and deliberately has - no third-party anything in it. A Moonraker needs the other half too, and - bin/patch.sh puts both halves under the same $MODDIR: section 5c writes - bin/python3.13 and lib/python3.13 (stdlib AND site-packages), section 5d - writes lib/libsodium.so*. Two caches, one destination, so one tarball -- - which also means the case unpacks it exactly once and every file lands - where it was compiled to expect itself. + bin/patch.sh puts every one of these under the same $MODDIR: section 5c + writes bin/python3.13 and lib/python3.13 (stdlib AND site-packages) out of + pkg/python and the eighteen pkg/python-* recipes, and section 5d writes + lib/libsodium.so* out of pkg/libsodium. Many recipe outputs, one + destination, so one tarball -- which also means the case unpacks it exactly + once and every file lands where it was compiled to expect itself. - Returns None when either cache is missing, which for this gate is a Skip: + Returns None when either half is missing, which for this gate is a Skip: see the docstring above for why there is no useful fallback. """ - py = config.root / "work" / ".py313" + trees = _python_trees(config) sodium = config.root / "work" / ".sodium" - if not (py / "bin" / "python3.13").is_file(): + if not (trees[0] / "bin" / "python3.13").is_file(): return None if not list(sodium.glob("lib/libsodium.so*")): return None out = config.root / "work" / ".pref-gate.tgz" with tarfile.open(str(out), "w:gz") as tar: - for sub in ("bin", "lib"): - tar.add(str(py / sub), arcname=sub) + for tree in trees: + for sub in ("bin", "lib"): + if (tree / sub).is_dir(): + tar.add(str(tree / sub), arcname=sub) for so in sorted(sodium.glob("lib/libsodium.so*")): tar.add(str(so), arcname="lib/" + so.name) return str(out) @@ -367,25 +367,59 @@ def supervisor(config, on_output=None): packages={"sup.tgz": s6}, on_output=on_output) +def _python_trees(config): + """Every recipe output that makes up the printer's python prefix. + + NINETEEN TREES, NOT ONE, for the same reason _s6_tarball reads three. This + used to be work/.py313, the single directory bin/patch.sh cross-built the + interpreter and its site-packages into together. CPython is pkg/python now + and each of the eighteen third-party packages is a pkg/python-* of its own, + so what a printer sees is the union of their bin/ and lib/ -- which is + exactly what bin/patch.sh section 5c stages, in this order. + + The interpreter comes first so that a half-built checkout fails on the + thing the caller actually needs rather than on a package that depends on it. + """ + pkg = config.root / "work" / "pkg" + return [pkg / "python"] + sorted(d for d in pkg.glob("python-*") if d.is_dir()) + + def _python_tarball(config): """The cross-built CPython 3.13 tree, packed the way a printer sees it. - bin/patch.sh leaves it in work/.py313 as bin/ + lib/ -- the same shape - work/.s6 holds for s6, because it is the same prefix: the interpreter is - configured --prefix=/usr/data/anvil and its stdlib lives in - lib/python3.13/. So a tarball made from here unpacks straight into $MODDIR - and every file lands where it was compiled to expect itself. Returns None - when nothing has built it yet; the same shape as _s6_tarball above, and - for the same reason: whether that is a Skip or a fallback is the caller's - question. + The interpreter is configured --prefix=/usr/data/anvil and its stdlib lives + in lib/python3.13/, so a tarball of the merged bin/ + lib/ unpacks straight + into $MODDIR and every file lands where it was compiled to expect itself. + Returns None when nothing has built it yet; the same shape as _s6_tarball + above, and for the same reason: whether that is a Skip or a fallback is the + caller's question. + + THE DEV HALF RIDES ALONG and is deliberately not filtered out. work/pkg/python + holds the whole build -- headers, lib/pkgconfig and config-3.13-* included -- + because the split into anvil-python and anvil-python-dev happens where the + .ipk files are made. Which paths those are is pkg/python/pkg.conf's business, + and repeating the list here would be a second spelling that goes stale + silently. This is a test fixture unpacked into a simulator, not something a + printer installs: 3MB of headers it will never open costs nothing, and the + package boundary is gated where it is made. """ - built = config.root / "work" / ".py313" - if not (built / "bin" / "python3.13").is_file(): + trees = _python_trees(config) + if not (trees[0] / "bin" / "python3.13").is_file(): return None + staged = config.root / "work" / ".py-gate" + if staged.is_dir(): + shutil.rmtree(str(staged)) + for tree in trees: + for sub in ("bin", "lib"): + src = tree / sub + if src.is_dir(): + shutil.copytree(str(src), str(staged / sub), dirs_exist_ok=True, + symlinks=True) out = config.root / "work" / ".py-gate.tgz" with tarfile.open(str(out), "w:gz") as tar: for sub in ("bin", "lib"): - tar.add(str(built / sub), arcname=sub) + if (staged / sub).is_dir(): + tar.add(str(staged / sub), arcname=sub) return str(out) @@ -408,7 +442,7 @@ def python(config, on_output=None): """ tree = _python_tarball(config) if not tree: - raise Skip("nothing in work/.py313 -- run ./bin/patch.sh first") + raise Skip("nothing in work/pkg/python -- run ./bin/patch.sh first") replica = Replica.start(config, want_output=on_output) replica.run_case(_case(config, "case-python.sh"), packages={"py.tgz": tree}, on_output=on_output) diff --git a/test/integration/printer/case-boot-screen.sh b/test/integration/printer/case-boot-screen.sh index 09189e0..1c13c83 100755 --- a/test/integration/printer/case-boot-screen.sh +++ b/test/integration/printer/case-boot-screen.sh @@ -43,7 +43,7 @@ H=800 BPP=32 mkdir -p $MOD/bin -# The interpreter FF_PYTHON resolves to -- a BUILD OUTPUT (work/.py313), not +# The interpreter FF_PYTHON resolves to -- a BUILD OUTPUT (work/pkg/python), not # part of /tmp/payload, handed over as py.tgz. gates.py's boot_screen() Skips # the whole case when nothing has built it, so reaching here without one is a # harness bug, not an absent feature. diff --git a/test/integration/printer/case-mcu-bringup.sh b/test/integration/printer/case-mcu-bringup.sh index 3b4ba3e..659d93f 100755 --- a/test/integration/printer/case-mcu-bringup.sh +++ b/test/integration/printer/case-mcu-bringup.sh @@ -32,7 +32,7 @@ PY=$MOD/bin/python3.13 mkdir -p $MOD/bin # The interpreter FF_PYTHON resolves to. Not part of /tmp/payload -- it is a -# BUILD OUTPUT (bin/patch.sh section 5c, cached in work/.py313), so gates.py +# BUILD OUTPUT (bin/patch.sh section 5c, cached under work/pkg), so gates.py # hands it over as py.tgz the same way case-python.sh receives one, and this # is a hard failure rather than a Skip if it is missing: gates.py's # mcu_bringup() already Skips the whole case when nothing has built it, so diff --git a/test/integration/printer/case-moonraker313-s6.sh b/test/integration/printer/case-moonraker313-s6.sh index 44a1671..8b7f04e 100755 --- a/test/integration/printer/case-moonraker313-s6.sh +++ b/test/integration/printer/case-moonraker313-s6.sh @@ -63,7 +63,7 @@ # in payload/ -- they are build outputs, exactly as work/.s6 is: # # sup.tgz work/.s6 bin/ + libexec/, the cross-built s6 -# pref.tgz work/.py313 + bin/python3.13, lib/python3.13 (stdlib +# pref.tgz work/pkg/python* + bin/python3.13, lib/python3.13 (stdlib # work/.sodium AND site-packages) and libsodium.so*. # One tarball because it is one prefix: # bin/patch.sh sections 5c and 5d put both diff --git a/test/integration/sim-boot-screen.py b/test/integration/sim-boot-screen.py index 715864c..f6f1cec 100755 --- a/test/integration/sim-boot-screen.py +++ b/test/integration/sim-boot-screen.py @@ -42,7 +42,7 @@ def run(): config = Config.load() tree = _python_tarball(config) if not tree: - raise SystemExit("nothing in work/.py313 -- run ./bin/patch.sh first") + raise SystemExit("nothing in work/pkg/python -- run ./bin/patch.sh first") replica = Replica.start(config, want_output=_echo) # The frames come back on stdout as base64, so this body is the payload -- # not just a log. It is deliberately not echoed. diff --git a/tools/python-packages/README.md b/tools/python-packages/README.md index 29bf281..9f3c03e 100644 --- a/tools/python-packages/README.md +++ b/tools/python-packages/README.md @@ -1,9 +1,8 @@ # The Python packages Moonraker and klippy need, cross-built for mipsel The second half of phase 6. `tools/python/` builds the interpreter; -this builds what runs on it. **Now wired into `bin/patch.sh` section 5c, -step 4** -- see "What is left" at the bottom for what that did and did not -settle. `build.sh` is the measurement harness, preserved here because it is +this builds what runs on it. **Now a recipe apiece: `pkg/python-`, one +`.ipk` each** -- see "What is left" at the bottom for the history. `build.sh` is the measurement harness, preserved here because it is where the findings live, exactly as `tools/supervisor/` is for s6; it is not what a release runs any more, and the two have drifted on purpose (patch.sh builds offline from `vendor/` and needs no docker, because the build lane has @@ -173,16 +172,31 @@ same one-line `FF_PYTHON` switch, since 3.8's libnacl still needs it. Capability was settled first -- Moonraker serves, the database works, libsodium is ours -- and packaging and integration followed: -1. ~~**Wire into `bin/patch.sh`.**~~ **Done.** The package build runs inside - 5c against the untrimmed stage, BEFORE the trim, which was the - recommendation here and is still the only place all three things it needs - exist at once: the headers `INCLUDEPY` points at, the static C libraries in - `work/.py-dep` (pillow's zlib, cffi's libffi) and the x86-64 build-python. - The consequence is that `work/.py313` has **two stamps** -- `.version` for - the interpreter and `.pkg-version` for the packages -- and a package bump - rebuilds CPython too, which is the price of not caching a second untrimmed - tree. libsodium is its own section, 5d, cached separately in - `work/.sodium`. Every pin is in `versions.env` with a sha256 and +1. ~~**Wire into `bin/patch.sh`.**~~ ~~**Done.**~~ **Superseded: each package + is its own recipe.** The step below describes the arrangement that replaced + it, and the arrangement that then replaced *that*, because the second + correction is the interesting one. + + Wiring into section 5c meant building the packages against the untrimmed + stage, BEFORE the trim -- the only moment all three things they need + existed at once: the headers `INCLUDEPY` points at, the static C libraries + in `work/.py-dep` (pillow's zlib, cffi's libffi) and the x86-64 + build-python. All three were scratch directories deleted at the end of the + section, so `work/.py313` needed **two stamps** -- `.version` for the + interpreter and `.pkg-version` for the packages -- and a package bump + rebuilt CPython too, which was the price of not caching a second untrimmed + tree. + + **That price is what the recipes removed.** The three passing moments are + durable things now: the untrimmed stage is the feed's `anvil-python-dev` + package, the static C libraries are each recipe's own sysroot filled by + `pkg_deps`, and the build-python is `pkg_buildpython`'s shared cache under + `work/.py-host`. So there is no second stamp and no coupled rebuild -- + `make packages PKG=python-pillow` builds Pillow. Each package also carries + its upstream version into the feed index, which the single site-packages + tree could not express. + + libsodium is its own recipe. Every pin is in `versions.env` with a sha256 and `bin/fetch-assets.sh` fetches them, so **patch.sh no longer talks to a network**: no `get-pip.py`, no PyPI, `pip --no-index` against hashed sdists. The build-python gets its pip from `--with-ensurepip=install` out diff --git a/tools/python/README.md b/tools/python/README.md index 13bd5d5..3065947 100644 --- a/tools/python/README.md +++ b/tools/python/README.md @@ -9,7 +9,7 @@ Everything below was measured on the printer replica (real `rootfs.squashfs` under qemu-mipsel) or read out of the objects with `readelf`, not off a project page. `build.sh` beside this file rebuilds the whole thing in a throwaway container and is where the numbers came from; -`bin/patch.sh` section 5c is the same build wired into the real build lane, +`pkg/python` is the same build wired into the real build lane, and `test/integration/printer/case-python.sh` is the gate that proves the result works on the printer's own kernel. @@ -275,9 +275,11 @@ the extensions is the actual work in adopting this; not the interpreter. That is how the numbers above were measured, and it is kept here for the record and for trying a version bump before it becomes a pin. It is not how -the shipped interpreter is built: `bin/patch.sh` section 5c compiles it in -place inside the repo's own build image, from the sha256-pinned tarballs, and -caches it in `work/.py313`. +the shipped interpreter is built: `pkg/python` compiles it inside the repo's +own build image, from the sha256-pinned tarballs, against the seven static +libraries the feed builds as their own `-dev` packages, and caches it in +`work/pkg/python`. `bin/patch.sh` section 5c runs that recipe and stages what +it produced; it no longer contains a compiler invocation of its own. The gate, against the tree the build actually produced: diff --git a/tools/python/build.sh b/tools/python/build.sh index 87b903d..1789df5 100755 --- a/tools/python/build.sh +++ b/tools/python/build.sh @@ -2,10 +2,10 @@ # --------------------------------------------------------------------------- # Cross-build CPython 3.13 for the FlashForge Creator 5 Pro (Ingenic mipsel). # -# THIS IS THE MEASUREMENT HARNESS, NOT THE BUILD. bin/patch.sh section 5c is -# what produces the interpreter a package ships: it compiles in place, inside -# the repo's own build image, from the tarballs pinned in versions.env, and -# caches the result in work/.py313. The two are the same build -- same flags, +# THIS IS THE MEASUREMENT HARNESS, NOT THE BUILD. pkg/python is what produces +# the interpreter a package ships: it compiles inside the repo's own build +# image, from the tarballs pinned in versions.env, and caches the result in +# work/pkg/python. The two are the same build -- same flags, # same wrappers, same gates -- but this one re-execs itself inside a # throwaway debian:bookworm, which is how every number in README.md beside it # was measured, and which is the fastest way to try a change to the recipe diff --git a/versions.env b/versions.env index 56f3990..133558a 100644 --- a/versions.env +++ b/versions.env @@ -274,10 +274,15 @@ EXPAT_SHA256="fd03b7172b3bd7427a3e7a812063f74754f24542429b634e0db6511b53fb2278" # HELIX_FILE does, and its SPELLING is upstream's business (MarkupSafe, # dbus_next and streaming_form_data are not the names their projects use). # -# PYPKG_LIST is the build order and the ship list at once; bin/patch.sh -# iterates it and bin/fetch-assets.sh fetches it, so a package added here and -# nowhere else is fetched, built, gated and shipped. The variable names are -# the list entry uppercased with - as _, which is how both scripts find them. +# PYPKG_LIST IS THE SHIP LIST AND THE FETCH LIST -- no longer the build order, +# which pkg_order derives from each recipe's PKG_BUILD_DEPENDS. Each entry is a +# recipe: `distro` is pkg/python-distro, which produces anvil-python-distro. +# bin/fetch-assets.sh fetches from this list and bin/patch.sh stages from it, +# and qa/static checks the list against the recipe directories both ways, so an +# entry added here and nowhere else fails a test rather than being fetched, +# hashed and silently never built. The variable names are the list entry +# uppercased with - as _, which is how every reader finds them; the VERSION is +# not pinned separately, it is read out of the FILE name (see pypkg_version). PYPKG_LIST="distro inotify-simple libnacl dbus-next preprocess-cancellation pyserial pyserial-asyncio pycparser smart-open jinja2 setuptools tornado markupsafe cffi greenlet lmdb streaming-form-data pillow" @@ -297,8 +302,8 @@ PYPKG_DBUS_NEXT_SHA256="f4eae26909332ada528c0a3549dda8d4f088f9b365153952a408e280 # preprocess-cancellation 0.2.1 is a poetry project whose sdist declares NO # [build-system] table at all and ships no setup.py, so there is no backend to # call and pip's legacy fallback would have to guess. It is also ONE FILE, so -# bin/patch.sh copies preprocess_cancellation.py into site-packages rather -# than building it -- see the comment there. (The same release also declares +# pkg/python-preprocess-cancellation copies preprocess_cancellation.py into +# site-packages rather than building it -- see the comment there. (The same release also declares # version 0.0.0 in its own metadata, a broken upstream release, so a version # specifier would be refused by pip even if there were something to build.) PYPKG_PREPROCESS_CANCELLATION_FILE="preprocess_cancellation-0.2.1.tar.gz" @@ -326,10 +331,10 @@ PYPKG_JINJA2_SHA256="4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800 # belt and braces -- 4.9MB of it, 36% of site-packages, and the one obvious # thing to drop if the data partition ever gets tight.) # -# It is ALSO the host-side build backend for most of this list: patch.sh -# installs this same sdist into the throwaway x86-64 build-python. One pin, -# both jobs -- so the setuptools that generates the shipped .so files is the -# setuptools whose hash is checked here. +# It is ALSO the host-side build backend for most of this list: +# pkg_buildpython installs this same sdist into the shared x86-64 +# build-python. One pin, both jobs -- so the setuptools that generates the +# shipped .so files is the setuptools whose hash is checked here. PYPKG_SETUPTOOLS_FILE="setuptools-75.6.0.tar.gz" PYPKG_SETUPTOOLS_SHA256="8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6" @@ -352,7 +357,7 @@ PYPKG_CFFI_SHA256="1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52 # PyTypeObjects as designated initializers, which gcc 7.2 -- the only compiler # that produces this printer's ABI -- accepts ONLY in declaration order with # no gaps, and refuses anything else with "sorry, unimplemented: non-trivial -# designated initializers not supported". patch.sh runs +# designated initializers not supported". pkg/python-greenlet runs # tools/python-packages/fill-designators.py over the sources first. Bumping # this pin means re-running that, and it fails loudly rather than quietly: the # script refuses to guess when a designator is not in the target's own headers. @@ -361,8 +366,9 @@ PYPKG_GREENLET_SHA256="4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5 # lmdb -- Moonraker's database AT THE PIN ABOVE, and the package that # disappears the moment MOONRAKER_VERSION moves past 80c7620 (lmdb -> sqlite), # which is the whole reason for having built an interpreter with _sqlite3. -# patch.sh sets LMDB_FORCE_CPYTHON=1; do not "fix" that with LMDB_FORCE_CFFI=0 -# -- setup.py tests the variable for PRESENCE, so the string "0" selects cffi. +# pkg/python-lmdb sets LMDB_FORCE_CPYTHON=1; do not "fix" that with +# LMDB_FORCE_CFFI=0 -- setup.py tests the variable for PRESENCE, so the string +# "0" selects cffi. PYPKG_LMDB_FILE="lmdb-1.4.1.tar.gz" PYPKG_LMDB_SHA256="1f4c76af24e907593487c904ef5eba1993beb38ed385af82adb25a858f2d658d" # streaming-form-data -- Moonraker's upload parser, Cython. @@ -395,7 +401,7 @@ PYPKG_PILLOW_SHA256="72bacbaf24ac003fea9bff9837d1eedb6088758d41e100c1552930151f6 # ---- host-side build backends. THESE NEVER SHIP. ------------------------- # Two of the sdists above name a PEP 517 backend that is not setuptools, and -# bin/patch.sh builds everything with --no-build-isolation -- which is not a +# pkg_pywheel builds everything with --no-build-isolation -- which is not a # preference either: with isolation ON, pip rewrites PYTHONPATH for the build # subprocess, the cross _sysconfigdata module stops being importable, and the # build either dies outright ("No module named @@ -407,8 +413,9 @@ PYPKG_PILLOW_SHA256="72bacbaf24ac003fea9bff9837d1eedb6088758d41e100c1552930151f6 # # Both bootstrap themselves (their sdists declare backend-path), so no third # package is needed to install them, and neither is on PYPKG_LIST -- they go -# into the throwaway x86-64 build-python and are thrown away with it. -# setuptools, the third backend, is on the ship list above. +# into the shared x86-64 build-python that pkg_buildpython caches under +# work/.py-host, and never leave it. setuptools, the third backend, is on the +# ship list above and ships as a package of its own as well. PYPKG_HOST_LIST="flit-core poetry-core" PYPKG_FLIT_CORE_FILE="flit_core-3.10.1.tar.gz" # jinja2's backend PYPKG_FLIT_CORE_SHA256="66e5b87874a0d6e39691f0e22f09306736b633548670ad3c09ec9db03c5662f7"