firmware/bin/fetch-assets.sh

#!/usr/bin/env bash
# Download the pinned third-party payload pieces into vendor/.
#   ./bin/fetch-assets.sh [--all] [--stock]
#     --all    ignore the BUILD_* flags -- every payload piece, every model
#     --stock  the stock FlashForge package too (both models under --all, else
#              the one TARGET_MACHINE names)
#
# The two are orthogonal on purpose. The stock firmware is ~93MB per model and
# the packaging lane wants none of it, so `--all` alone still means "everything
# a feed needs" -- `make vendor` asks for both.
# Pins live in versions.env; a cached file with the right sha256 is not refetched.
set -euo pipefail
# shellcheck disable=SC1091
. "$(dirname "$0")/common.sh"
# Sourced for pkg_needs alone: "is any recipe going to have to compile?" -- the
# question that decides whether the ~203MB toolchain is worth downloading.
# shellcheck disable=SC1091
. "$ROOT/pkgs/lib.sh"
say() { printf '>> %s\n' "$*"; }

ALL=0
STOCK=0
for arg in "$@"; do
    case "$arg" in
        --all)   ALL=1 ;;
        --stock) STOCK=1 ;;
        *) echo "unknown option: $arg" >&2; exit 1 ;;
    esac
done

mkdir -p vendor

# get <url> <destination> <sha256>
get() {
    url="$1"; dest="$2"; want="$3"
    if [ -f "$dest" ]; then
        have=$(sha256sum "$dest" | cut -d' ' -f1)
        if [ "$have" = "$want" ]; then
            say "cached  $(basename "$dest")"
            return 0
        fi
        say "stale   $(basename "$dest") -- re-downloading"
    fi
    say "fetch   $url"
    # --retry: several hosts, and a plain -f turns one bad minute into a failed
    # build. --connect-timeout: without it a silent host burns curl's own
    # default (over a minute) before an attempt even counts as failed.
    curl -fL --progress-bar --connect-timeout 20 \
        --retry 5 --retry-connrefused --retry-all-errors --retry-delay 5 \
        -o "$dest.part" "$url"
    have=$(sha256sum "$dest.part" | cut -d' ' -f1)
    if [ "$want" = "SKIP" ]; then
        mv "$dest.part" "$dest"
        say "sha256  $have  <-- paste this into versions.env"
        return 0
    fi
    if [ "$have" != "$want" ]; then
        rm -f "$dest.part"
        echo "checksum mismatch for $(basename "$dest")" >&2
        echo "  expected $want" >&2
        echo "  got      $have" >&2
        exit 1
    fi
    mv "$dest.part" "$dest"
}

# get_git <url> <dir> <commit> <label> -- a checkout pinned by commit sha,
# for upstreams that publish no release tarball. The sha covers the whole
# tree, which is the same guarantee the sha256s above give in git's spelling.
#
# A TAG WOULD NOT DO: tags move, and GitLab regenerates tag archives, so
# neither the ref nor a .tar.gz of it is stable enough to pin against.
get_git() {
    url="$1"; dir="$2"; want="$3"; label="$4"
    if [ ! -d "$dir/.git" ]; then
        say "clone   $label"
        rm -rf "$dir"
        git clone -q "$url" "$dir"
    fi
    ( cd "$dir"
      # The url every time, not just on the first clone: a checkout made
      # before this file named a different host would keep fetching from it,
      # and that failure would only appear on the next pin bump.
      git remote set-url origin "$url"
      # Only if the pinned commit is missing, so the common case is offline.
      git cat-file -e "$want^{commit}" 2>/dev/null || git fetch -q origin
      git checkout -q "$want"
      have=$(git rev-parse HEAD)
      if [ "$have" != "$want" ]; then
          echo "   !! $label is at $have, not the pinned $want" >&2
          exit 1
      fi
      # The sha covers what git tracks and nothing about what was edited in
      # place, so a dirty checkout is refused rather than silently trusted.
      if [ -n "$(git status --porcelain)" ]; then
          echo "   !! $dir has local modifications -- the pinned commit" >&2
          echo "      no longer describes what is in it. Delete it and re-run." >&2
          exit 1
      fi )
    say "cached  $label ($want)"
}

# The stock FlashForge package. Not a payload piece -- nothing out of it ships
# (docs/how-it-works.md: a release carries no FlashForge component at all) --
# but bin/unpack.sh reads the printer.base.cfg and the stock root hash out of
# it, so the BUILD lane cannot start without one and the PACKAGING lane never
# wants one. Hence its own flag rather than a BUILD_* gate: `make packages` and
# the CI job behind it stay runnable on a bare checkout, which is most of the
# point of that lane.
#
# --all takes BOTH models because `make release` builds both; --stock takes
# only the one being built, so a single `make build` does not pull 186MB to
# use 93 of it.
#
# THE ONE DOWNLOAD THAT WILL NOT OVERWRITE AN OVERRIDE, unlike HELIX_TGZ and
# friends. Those name a public release anyone can re-fetch in a minute; this
# names 93MB of proprietary firmware a person went and downloaded, and
# clobbering it on a hash mismatch would destroy the only copy on the machine
# to replace it with one they did not ask for. An override is left alone and
# skipped; the sha256 still gates everything this script actually fetches.
stock_get() {   # <file> <sha256> <configured path>
    if [ "$3" != "$ROOT/vendor/$1" ]; then
        say "override $(basename "$3")  (not fetched, not checksummed)"
        return 0
    fi
    get "$STOCK_URL_BASE/$1" "$ROOT/vendor/$1" "$2"
}
if [ "$STOCK" = 1 ]; then
    if [ "$ALL" = 1 ] || [ "$TARGET_MACHINE" = Creator5Pro ]; then
        stock_get "$STOCK_FILE_CREATOR5PRO" "$STOCK_SHA256_CREATOR5PRO" \
                  "$STOCK_TGZ_CREATOR5PRO"
    fi
    if [ "$ALL" = 1 ] || [ "$TARGET_MACHINE" = Creator5 ]; then
        stock_get "$STOCK_FILE_CREATOR5" "$STOCK_SHA256_CREATOR5" \
                  "$STOCK_TGZ_CREATOR5"
    fi
fi

if [ "$ALL" = 1 ] || [ "${BUILD_MAINSAIL:-0}" = "1" ]; then
    get "https://github.com/mainsail-crew/mainsail/releases/download/$MAINSAIL_VERSION/mainsail.zip" \
        "$MAINSAIL_ZIP" "$MAINSAIL_SHA256"
fi

if [ "$ALL" = 1 ] || [ "${BUILD_FLUIDD:-0}" = "1" ]; then
    get "https://github.com/fluidd-core/fluidd/releases/download/$FLUIDD_VERSION/fluidd.zip" \
        "$FLUIDD_ZIP" "$FLUIDD_SHA256"
fi

if [ "$ALL" = 1 ] || [ "${BUILD_HELIX:-0}" = "1" ]; then
    # Klipper4FlashForge is the org's current name; the old URL only worked
    # through GitHub's rename redirect.
    get "https://github.com/Klipper4FlashForge/helixscreen/releases/download/$HELIX_VERSION/$HELIX_FILE" \
        "$HELIX_TGZ" "$HELIX_SHA256"
fi

# One source, named once, because pkgs/klipper is a recipe. Re-pin
# KLIPPER_VERSION and KLIPPER_SHA256 to build something else.
get "https://github.com/Klipper4FlashForge/klipper/archive/$KLIPPER_VERSION.tar.gz" \
    "$KLIPPER_TGZ" "$KLIPPER_SHA256"

# The Ingenic glibc toolchain -- ~203MB, shared by every recipe that compiles.
# The condition comes from pkg_needs, the code that writes the recipe stamps,
# rather than a second spelling that drifts.
if [ "$ALL" = 1 ] || pkg_needs; then
    get "https://github.com/ballaswag/k1-discovery/releases/download/$MIPS_TOOLCHAIN_VERSION/$MIPS_TOOLCHAIN_FILE" \
        "$MIPS_TOOLCHAIN_TGZ" "$MIPS_TOOLCHAIN_SHA256"
fi

if [ "$ALL" = 1 ] || [ "${BUILD_MOONRAKER:-0}" = "1" ]; then
    # Moonraker ships no release asset; /archive/<ref> takes a tag or a commit
    # sha, and the pin is a commit. The sha256 is what makes either safe.
    get "https://github.com/Arksine/moonraker/archive/$MOONRAKER_VERSION.tar.gz" \
        "$MOONRAKER_TGZ" "$MOONRAKER_SHA256"
fi

# moonraker-timelapse. No BUILD_ flag: the recipe is ungated (pkgs/timelapse/
# pkg.conf says why), so a conditional fetch would leave it nothing to unpack.
get "https://github.com/mainsail-crew/moonraker-timelapse/archive/$TIMELAPSE_VERSION.tar.gz" \
    "$TIMELAPSE_TGZ" "$TIMELAPSE_SHA256"

# The encoder for it. download.videolan.org, not code.videolan.org: the latter
# is behind bot protection that answers a download with an HTML page.
get "https://download.videolan.org/pub/videolan/x264/snapshots/x264-$X264_VERSION.tar.bz2" \
    "$X264_TGZ" "$X264_SHA256"
get "https://ffmpeg.org/releases/ffmpeg-$FFMPEG_VERSION.tar.xz" \
    "$FFMPEG_TGZ" "$FFMPEG_SHA256"

# The supervision stack. No BUILD_ flag: every package ships it.
get "https://skarnet.org/software/skalibs/skalibs-$SKALIBS_VERSION.tar.gz" \
    "$SKALIBS_TGZ" "$SKALIBS_SHA256"
get "https://skarnet.org/software/execline/execline-$EXECLINE_VERSION.tar.gz" \
    "$EXECLINE_TGZ" "$EXECLINE_SHA256"
get "https://skarnet.org/software/s6/s6-$S6_VERSION.tar.gz" \
    "$S6_TGZ" "$S6_SHA256"
get "https://skarnet.org/software/s6-rc/s6-rc-$S6RC_VERSION.tar.gz" \
    "$S6RC_TGZ" "$S6RC_SHA256"

# The compiler is fetched above on the same pkg_needs condition, asked again
# here so a build cannot pull 40MB of source and then stop for want of it.
get "https://www.python.org/ftp/python/$PY_VERSION/Python-$PY_VERSION.tgz" \
    "$PY_TGZ" "$PY_SHA256"
get "https://github.com/openssl/openssl/releases/download/openssl-$OPENSSL_VERSION/openssl-$OPENSSL_VERSION.tar.gz" \
    "$OPENSSL_TGZ" "$OPENSSL_SHA256"
# sqlite.org files the amalgamation under the YEAR of release, which is nowhere
# in the version number -- hence SQLITE_YEAR.
get "https://www.sqlite.org/$SQLITE_YEAR/sqlite-autoconf-$SQLITE_VERSION.tar.gz" \
    "$SQLITE_TGZ" "$SQLITE_SHA256"
# madler's own release asset, not zlib.net: that host sits behind an anti-bot
# interstitial that answers 200 with a 12KB HTML spinner page, which curl -f
# accepts and only the sha256 rejects -- as a mismatch that reads like
# tampering. The GitHub asset is byte-identical to the pin below.
get "https://github.com/madler/zlib/releases/download/v$ZLIB_VERSION/zlib-$ZLIB_VERSION.tar.gz" \
    "$ZLIB_TGZ" "$ZLIB_SHA256"
get "https://github.com/libffi/libffi/releases/download/v$LIBFFI_VERSION/libffi-$LIBFFI_VERSION.tar.gz" \
    "$LIBFFI_TGZ" "$LIBFFI_SHA256"
get "https://github.com/tukaani-project/xz/releases/download/v$XZ_VERSION/xz-$XZ_VERSION.tar.gz" \
    "$XZ_TGZ" "$XZ_SHA256"
get "https://sourceware.org/pub/bzip2/bzip2-$BZIP2_VERSION.tar.gz" \
    "$BZIP2_TGZ" "$BZIP2_SHA256"
# EXPAT_TAG, not EXPAT_VERSION: libexpat tags R_2_6_4 but names the file
# expat-2.6.4, so both spellings appear in the one URL.
get "https://github.com/libexpat/libexpat/releases/download/$EXPAT_TAG/expat-$EXPAT_VERSION.tar.gz" \
    "$EXPAT_TGZ" "$EXPAT_SHA256"

# pypi <project> <file> <sha256>
# Only /packages/source/<initial>/<project>/<file> can be composed from a pin;
# the other URL embeds a digest of the file itself. The project name is a
# property of the URL, not the version: `markupsafe` -> MarkupSafe-2.1.5.
pypi() {
    proj="$1"; file="$2"; sha="$3"
    get "https://files.pythonhosted.org/packages/source/$(printf '%.1s' "$proj")/$proj/$file" \
        "$ROOT/vendor/$file" "$sha"
}
# PYPKG_HOST_LIST too: the PEP 517 backends never reach a printer, but they
# produce the objects that do.
for p in $PYPKG_LIST $PYPKG_HOST_LIST; do
    pypi "$p" "$(pypkg_var "$p" FILE)" "$(pypkg_var "$p" SHA256)"
done
get "https://github.com/jedisct1/libsodium/releases/download/$SODIUM_VERSION-RELEASE/libsodium-$SODIUM_VERSION.tar.gz" \
    "$SODIUM_TGZ" "$SODIUM_SHA256"

# --- apk-tools
# The package manager, and the packager: the same checkout builds the printer's
# apk (pkgs/3rdparty/apk-tools, patched) and the build machine's
# (tools/apk-host, unpatched), so the tool that writes a package and the tool
# that reads it are one version by construction.
#
# A GIT CLONE, and the reason get_git exists: the upstream forge regenerates
# tag archives, so their sha256 is not a pin. The commit sha is.
#
# THE GITHUB MIRROR, NOT upstream's own gitlab.alpinelinux.org: that host
# answers datacenter addresses with HTTP 418 or nothing at all, so the clone
# succeeds from a developer's machine and fails in CI -- measured, on both CI
# jobs at once. Which host serves it does not affect what is fetched: the pin
# is a commit sha, and git names a commit by the hash of its content, so a
# mirror that answers with this sha is serving this tree or is not answering.
get_git https://github.com/alpinelinux/apk-tools.git \
    "$APK_TOOLS_DIR" "$APK_TOOLS_COMMIT" "apk-tools $APK_TOOLS_VERSION"

# --- the clock
# No BUILD_ flag: a printer with no RTC has the wrong time until this runs, and
# every printer here has no RTC.
get "https://github.com/troglobit/sntpd/releases/download/v$SNTPD_VERSION/sntpd-$SNTPD_VERSION.tar.gz" \
    "$SNTPD_TGZ" "$SNTPD_SHA256"

# The toolchain, on the pkg_needs condition that decides whether payload.sh has
# to compile at all. Wrong here is not a slow build but a stopped one.
if [ "$ALL" = 1 ] \
   || pkg_needs; then
    get "https://github.com/ballaswag/k1-discovery/releases/download/$MIPS_TOOLCHAIN_VERSION/$MIPS_TOOLCHAIN_FILE" \
        "$MIPS_TOOLCHAIN_TGZ" "$MIPS_TOOLCHAIN_SHA256"
fi