diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7295dc2..e07ba87 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: inputs: tag: - description: "Release tag to create/update, for example v1.5.0" + description: "Release tag to create/update, for example v1.6.0" required: false type: string draft: @@ -116,6 +116,13 @@ jobs: test -x "$sdk_dst/bin/orbis-clang" echo "PS4_PAYLOAD_SDK=$sdk_dst" >> "$GITHUB_ENV" + - name: Install PacBrew PS4 portlibs (libcurl) + if: matrix.platform == 'ps4' && matrix.zhttpd == '1' + run: | + set -euo pipefail + bash tools/fetch_pacbrew_ps4.sh "$GITHUB_WORKSPACE/pacbrew-ps4-openorbis" + echo "PS4_PACBREW_ROOT=$GITHUB_WORKSPACE/pacbrew-ps4-openorbis" >> "$GITHUB_ENV" + - name: Install PS5 PacBrew SDK bundle if: matrix.platform == 'ps5' run: | @@ -139,6 +146,13 @@ jobs: test -f "$sdk_dst/target/user/homebrew/etc/ca-bundle.crt" echo "PS5_PAYLOAD_SDK=$sdk_dst" >> "$GITHUB_ENV" + - name: Build PS5 BitTorrent engine + if: matrix.platform == 'ps5' && matrix.zhttpd == '1' + run: | + set -euo pipefail + sudo apt-get install -y libboost-dev + bash tools/build_ps5_libtorrent.sh "$PS5_PAYLOAD_SDK" + - name: Build release variant run: | set -euo pipefail @@ -159,6 +173,7 @@ jobs: ENABLE_ZHTTPD="${{ matrix.zhttpd }}" \ ENABLE_MCP=0 \ ENABLE_PKG_INSTALL=0 \ + ENABLE_LIBTORRENT="${{ matrix.zhttpd }}" \ clean all else make \ diff --git a/.gitignore b/.gitignore index 2d8ccd9..96627f9 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ dkms.conf .DS_Store /external/ps4-payload-sdk /external/ps5-payload-sdk +/external/pacbrew /external/ftpsrv-master /build /external/ps3libraries-master @@ -97,3 +98,4 @@ race log.txt /git-patch *.bak +/.commandcode diff --git a/Makefile b/Makefile index b6d43af..fca841c 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,9 @@ ifeq ($(TARGET),ps4) export LLVM_CONFIG := $(ORBIS_LLVM_CONFIG) include $(PS4_PAYLOAD_SDK)/toolchain/orbis.mk PLATFORM_DEFS := -DPLATFORM_PS4 -DPS4 - PLATFORM_LIBS := -lkernel -lpthread -lSceSysmodule -lSceSystemService -lSceUserService + # SceSysmodule is loaded at runtime with dlopen: the payload SDK does not + # ship a linkable library for it. + PLATFORM_LIBS := -lkernel -lpthread -lSceSystemService -lSceUserService PLATFORM_LDFLAGS := endif @@ -109,6 +111,7 @@ endif # Default to GCC if no target matched CC ?= gcc +CXX ?= c++ PLATFORM_LIBS ?= -lpthread # Event loop implementation — kqueue for BSD/macOS/PS4/PS5, epoll for Linux @@ -196,7 +199,11 @@ CFLAGS += -I./include/platform/ps5 -I./include/archive -I./include/runtime # LINKER FLAGS #============================================================================ -ifeq ($(HOST_OS),Linux) +# Console payloads are loaded by loaders (GoldHEN binloader, elfldr) that +# expect a position-independent executable, no matter which host built them. +ifneq ($(filter $(TARGET),ps4 ps5),) + LDFLAGS += -pie +else ifeq ($(HOST_OS),Linux) LDFLAGS += -pie endif LDFLAGS += $(PLATFORM_LDFLAGS) @@ -216,6 +223,9 @@ SOURCES += src/platform/fileio/directory.c SOURCES += src/platform/pal_alloc.c SOURCES += src/platform/pal_scratch.c SOURCES += src/platform/pal_notification.c +SOURCES += src/platform/pal_disc.c +SOURCES += src/platform/pal_volume.c +SOURCES += src/platform/pal_dns_filter.c SOURCES += src/platform/pal_filesystem.c SOURCES += src/platform/pal_filesystem_psx.c SOURCES += src/ftp/ftp_path.c @@ -234,13 +244,22 @@ SOURCES += src/ftp/ftp_buffer_pool.c SOURCES += src/ftp/ftp_log.c SOURCES += src/ftp/ftp_crypto.c SOURCES += src/app/main.c +SOURCES += src/app/instance_control.c SOURCES += src/platform/pal_resilient_server.c SOURCES += src/ftp/ftp_instance.c -# PS5-specific modules +# PS5 kernel syscall hook is opt-in until its firmware-specific install path +# has been validated on real hardware. The normal payload never loads it. +ENABLE_PS5_NET_FILTER ?= 0 ifeq ($(TARGET),ps5) + CFLAGS += -DENABLE_PS5_NET_FILTER=$(ENABLE_PS5_NET_FILTER) +endif +ifeq ($(TARGET),ps5) +SOURCES += src/platform/ps5/ps5_fw_offsets.c +ifeq ($(ENABLE_PS5_NET_FILTER),1) SOURCES += src/platform/ps5/ps5_net_filter.c endif +endif #============================================================================ # ZHTTPD (Web File Explorer) — compile-time toggle @@ -282,13 +301,27 @@ STRIP ?= strip ifeq ($(ENABLE_ZHTTPD),1) CFLAGS += -DENABLE_ZHTTPD=1 CFLAGS += -DENABLE_WEB_UPLOAD=1 - ENABLE_PKG_INSTALL ?= 0 + # PKG installation rides along with the web interface; override with =0. + ENABLE_PKG_INSTALL ?= 1 CFLAGS += -DENABLE_PKG_INSTALL=$(ENABLE_PKG_INSTALL) + ifeq ($(TARGET),ps5) + # The system installer cannot be driven from a payload process, so a + # small helper program is embedded here and delivered at runtime. + INSTALL_HELPER_ELF := $(BUILD_DIR)/install_helper.elf + INSTALL_HELPER_BLOB := $(BUILD_DIR)/generated/install_helper_blob.c + SOURCES += $(INSTALL_HELPER_BLOB) + SOURCES += src/http/games/ps5_install_helper.c + CFLAGS += -DZFTPD_INSTALL_HELPER=1 + endif ENABLE_LIBCURL ?= 1 SOURCES += $(EVENT_LOOP_SRC) SOURCES += src/http/http_server.c + SOURCES += src/http/http_listener.c + SOURCES += src/http/http_upload.c SOURCES += src/http/http_parser.c SOURCES += src/http/http_response.c + SOURCES += src/http/http_response_stream.c + SOURCES += src/http/http_json.c SOURCES += src/http/http_api.c SOURCES += src/http/http_api_common.c SOURCES += src/http/http_api_files.c @@ -297,6 +330,7 @@ ifeq ($(ENABLE_ZHTTPD),1) SOURCES += src/http/http_api_transfer.c SOURCES += src/http/http_static.c SOURCES += src/http/http_api_archive.c + SOURCES += src/http/http_api_dump.c SOURCES += src/http/http_csrf.c WEB_RESOURCE_FILES := $(shell find web -path web/legacy -prune -o -type f -print | sort) HTTP_RESOURCES_C := $(BUILD_DIR)/generated/http/http_resources.c @@ -304,7 +338,11 @@ ifeq ($(ENABLE_ZHTTPD),1) SOURCES += src/archive/exfat_unpacker.c SOURCES += src/archive/pkg_unpacker.c SOURCES += src/archive/builtin_unzip.c +SOURCES += src/archive/zip_writer.c + SOURCES += src/archive/archive_path.c SOURCES += src/transfer/transfer_manager.c + SOURCES += src/transfer/transfer_state.c + SOURCES += src/transfer/dump_job.c endif # NFS URL transfers are enabled by default for PS5 zhttp builds. Other targets @@ -362,41 +400,48 @@ endif # so we fall back to a simple file-existence check on bundled headers. #============================================================================ -# ── libarchive: auto-detect on desktop, opt-in on consoles ─────────────── +# ── libarchive: zhttp only; auto-detect on desktop ───────────────────── override ENABLE_LIBCURL ?= 0 -ifneq ($(filter $(TARGET),ps4 ps5),) - # PS4/PS5: no prebuilt static lib — explicit opt-in only (override ENABLE_LIBARCHIVE=1) +ifeq ($(ENABLE_ZHTTPD),1) +ifeq ($(TARGET),ps5) + # Optional PacBrew library for non-ZIP formats; ZIP keeps the builtin reader. override ENABLE_LIBARCHIVE ?= 0 ifeq ($(ENABLE_LIBARCHIVE),1) - $(info [INFO] libarchive not supported on cross-compile targets — disabling) - override ENABLE_LIBARCHIVE := 0 + PS5_PKG_CONFIG ?= $(PS5_PAYLOAD_SDK)/bin/prospero-pkg-config + _HAS_PS5_ARCHIVE := $(shell test -x $(PS5_PKG_CONFIG) && $(PS5_PKG_CONFIG) --exists libarchive && echo 1 || echo 0) + ifneq ($(_HAS_PS5_ARCHIVE),1) + $(error PS5 archive extraction requires PacBrew ps5-payload-libarchive) + endif + CFLAGS += -DENABLE_LIBARCHIVE=1 $(shell $(PS5_PKG_CONFIG) --cflags libarchive) + LIBS += $(shell $(PS5_PKG_CONFIG) --static --libs libarchive) + $(info [INFO] PS5 archive extraction: PacBrew libarchive) endif +else ifeq ($(TARGET),ps4) + # PS4 builds use the builtin ZIP backend. + override ENABLE_LIBARCHIVE := 0 else # Desktop (macOS/Linux): auto-detect unless user explicitly set it to 0 ifneq ($(ENABLE_LIBARCHIVE),0) _HAS_ARCHIVE := 0 - # 1) System headers + # 1) System installation _SYS_ARCHIVE := $(shell echo '\#include ' | $(CC) -xc -fsyntax-only - 2>/dev/null && echo 1 || echo 0) ifeq ($(_SYS_ARCHIVE),1) _HAS_ARCHIVE := 1 - $(info [INFO] libarchive: using system headers) + LIBS += -larchive + $(info [INFO] libarchive: using system installation) endif - # 2) macOS Homebrew (Apple Silicon, then Intel) + # 2) Homebrew keg (libarchive is keg-only on macOS) ifeq ($(_HAS_ARCHIVE),0) ifeq ($(HOST_OS),Darwin) - _HB_ARCHIVE := $(shell echo '\#include ' | $(CC) -xc -fsyntax-only -I/opt/homebrew/include - 2>/dev/null && echo 1 || echo 0) - ifeq ($(_HB_ARCHIVE),1) - _HAS_ARCHIVE := 1 - CFLAGS += -I/opt/homebrew/include - LDFLAGS += -L/opt/homebrew/lib - $(info [INFO] libarchive: using Homebrew (Apple Silicon) — /opt/homebrew) - else - _HB_ARCHIVE := $(shell echo '\#include ' | $(CC) -xc -fsyntax-only -I/usr/local/include - 2>/dev/null && echo 1 || echo 0) + _BREW_ARCHIVE_PREFIX := $(shell brew --prefix libarchive 2>/dev/null) + ifneq ($(_BREW_ARCHIVE_PREFIX),) + _HB_ARCHIVE := $(shell echo '\#include ' | $(CC) -xc -fsyntax-only -I$(_BREW_ARCHIVE_PREFIX)/include - 2>/dev/null && echo 1 || echo 0) ifeq ($(_HB_ARCHIVE),1) _HAS_ARCHIVE := 1 - CFLAGS += -I/usr/local/include - LDFLAGS += -L/usr/local/lib - $(info [INFO] libarchive: using Homebrew (Intel) — /usr/local) + CFLAGS += -I$(_BREW_ARCHIVE_PREFIX)/include + LDFLAGS += -L$(_BREW_ARCHIVE_PREFIX)/lib + LIBS += -larchive + $(info [INFO] libarchive: using Homebrew keg $(_BREW_ARCHIVE_PREFIX)) endif endif endif @@ -425,6 +470,9 @@ else endif endif endif +else + override ENABLE_LIBARCHIVE := 0 +endif # ── libcurl detection ───────────────────────────────────────────────────── # zftpd deliberately uses external libcurl instead of maintaining an HTTP/TLS @@ -444,16 +492,67 @@ ifeq ($(ENABLE_LIBCURL),1) endif GENERATED_CA_HEADER := $(BUILD_DIR)/generated/zftpd_ca_bundle.h CFLAGS += -DZFTPD_EMBEDDED_CA_BUNDLE=1 -I$(BUILD_DIR)/generated + CURL_CA_BUNDLE_SRC := $(PS5_CA_BUNDLE) $(info [INFO] URL downloads: external PacBrew libcurl + embedded CA bundle) else ifeq ($(TARGET),ps4) - PS4_CURL_CONFIG ?= $(shell command -v orbis-curl-config 2>/dev/null || true) - ifeq ($(strip $(PS4_CURL_CONFIG)),) + # PacBrew ps4-openorbis portlibs (libcurl + mbedTLS + zlib) as unpacked by + # tools/fetch_pacbrew_ps4.sh. Only the headers and the static libraries + # are used: the curl wrapper the package ships emits the OpenOrbis + # toolchain's own link flags, which do not belong in a payload-SDK link. + PS4_PACBREW_ROOT ?= $(shell for d in $(abspath external/pacbrew/ps4/openorbis) /opt/pacbrew/ps4/openorbis; do [ -f "$$d/usr/lib/libcurl.a" ] && { printf '%s' "$$d"; break; }; done) + + ifneq ($(strip $(PS4_PACBREW_ROOT)),) + # The OpenOrbis build reaches errno through musl's __errno_location() + # and uses fnmatch()/execl(); the payload libc has none of them. + SOURCES += src/platform/ps4/ps4_pacbrew_compat.c + CFLAGS += -DENABLE_LIBCURL=1 -DCURL_STATICLIB -I$(PS4_PACBREW_ROOT)/usr/include + LIBS += -L$(PS4_PACBREW_ROOT)/usr/lib -lcurl -lmbedtls -lmbedx509 -lmbedcrypto -lz -lSceNet + # Same trust-store contract as PS5: HTTPS verification must not degrade + # silently, so the Mozilla bundle is embedded at build time. + PS4_CA_BUNDLE ?= $(PS4_PACBREW_ROOT)/usr/etc/ca-bundle.crt + ifeq ($(wildcard $(PS4_CA_BUNDLE)),) + $(error PS4 libcurl CA bundle not found: $(PS4_CA_BUNDLE) — re-run tools/fetch_pacbrew_ps4.sh or set PS4_CA_BUNDLE=) + endif + GENERATED_CA_HEADER := $(BUILD_DIR)/generated/zftpd_ca_bundle.h + CFLAGS += -DZFTPD_EMBEDDED_CA_BUNDLE=1 -I$(BUILD_DIR)/generated + CURL_CA_BUNDLE_SRC := $(PS4_CA_BUNDLE) + $(info [INFO] URL downloads: PacBrew PS4 portlibs ($(PS4_PACBREW_ROOT)) + embedded CA bundle) + else + # A toolchain-provided pkg-config entry (or curl-config) still works. + # Probe the target environment only — a host libcurl would link code built + # for the wrong OS. + PS4_PKG_CONFIG ?= $(shell for c in $(PS4_PAYLOAD_SDK)/bin/orbis-pkg-config $(PS4_PAYLOAD_SDK)/bin/openorbis-pkg-config $(OPENORBIS)/bin/openorbis-pkg-config; do [ -x "$$c" ] && { printf '%s' "$$c"; break; }; done) + PS4_CURL_CONFIG ?= $(shell for c in $(PS4_PAYLOAD_SDK)/bin/orbis-curl-config $(PS4_PAYLOAD_SDK)/bin/curl-config $(PS4_PAYLOAD_SDK)/bin/openorbis-curl-config; do [ -x "$$c" ] && { printf '%s' "$$c"; break; }; done) + + _HAS_PS4_CURL := $(shell if [ -n "$(strip $(PS4_PKG_CONFIG))" ] && $(PS4_PKG_CONFIG) --exists libcurl >/dev/null 2>&1; then echo 1; elif [ -n "$(strip $(PS4_CURL_CONFIG))" ]; then echo 1; fi) + + ifneq ($(_HAS_PS4_CURL),1) $(info [INFO] PS4 external libcurl not found — URL downloader disabled) + $(info [INFO] run tools/fetch_pacbrew_ps4.sh to install PacBrew ps4-openorbis-portlibs) + $(info [INFO] or point PS4_PKG_CONFIG / PS4_CURL_CONFIG at a target config tool) override ENABLE_LIBCURL := 0 else - CFLAGS += -DENABLE_LIBCURL=1 $(shell $(PS4_CURL_CONFIG) --cflags) - LIBS += $(shell $(PS4_CURL_CONFIG) --static-libs) - $(info [INFO] URL downloads: external PS4 libcurl) + ifneq ($(strip $(PS4_PKG_CONFIG)),) + PS4_CURL_CFLAGS := $(shell $(PS4_PKG_CONFIG) --cflags libcurl) + PS4_CURL_LIBS := $(shell $(PS4_PKG_CONFIG) --static --libs libcurl) + else + PS4_CURL_CFLAGS := $(shell $(PS4_CURL_CONFIG) --cflags) + PS4_CURL_LIBS := $(shell $(PS4_CURL_CONFIG) --static-libs) + endif + CFLAGS += -DENABLE_LIBCURL=1 $(PS4_CURL_CFLAGS) + LIBS += $(PS4_CURL_LIBS) + + # Same trust-store contract as PS5: the PacBrew curl package ships the + # Mozilla bundle, and HTTPS verification must not degrade silently. + PS4_CA_BUNDLE ?= $(PS4_PAYLOAD_SDK)/target/user/homebrew/etc/ca-bundle.crt + ifeq ($(wildcard $(PS4_CA_BUNDLE)),) + $(error PS4 libcurl CA bundle not found: $(PS4_CA_BUNDLE) — override with PS4_CA_BUNDLE=) + endif + GENERATED_CA_HEADER := $(BUILD_DIR)/generated/zftpd_ca_bundle.h + CFLAGS += -DZFTPD_EMBEDDED_CA_BUNDLE=1 -I$(BUILD_DIR)/generated + CURL_CA_BUNDLE_SRC := $(PS4_CA_BUNDLE) + $(info [INFO] URL downloads: PacBrew PS4 libcurl + embedded CA bundle) + endif endif else _HAS_CURL := $(shell echo '\#include ' | $(CC) -xc -fsyntax-only - 2>/dev/null && echo 1 || echo 0) @@ -471,6 +570,44 @@ ifeq ($(ENABLE_LIBCURL),1) SOURCES += src/transfer/backend_curl.c endif +# Native BitTorrent engine for magnet links. PS5 needs a libtorrent-rasterbar +# port installed into its payload SDK; host libraries must never be linked +# into a console payload. +ENABLE_LIBTORRENT ?= 0 +ifeq ($(ENABLE_LIBTORRENT),1) + ifeq ($(TARGET),ps5) + PS5_PKG_CONFIG ?= $(PS5_PAYLOAD_SDK)/bin/prospero-pkg-config + ifneq ($(shell test -x $(PS5_PKG_CONFIG) && $(PS5_PKG_CONFIG) --exists libtorrent-rasterbar && echo 1),1) + $(error PS5 magnet downloads require a PS5 libtorrent-rasterbar port in the payload SDK) + endif + TORRENT_CFLAGS := $(shell $(PS5_PKG_CONFIG) --cflags libtorrent-rasterbar) + TORRENT_LIBS := $(shell $(PS5_PKG_CONFIG) --static --libs libtorrent-rasterbar) + else + ifneq ($(shell pkg-config --exists libtorrent-rasterbar 2>/dev/null && echo 1),1) + $(error ENABLE_LIBTORRENT=1 requires libtorrent-rasterbar development files) + endif + TORRENT_CFLAGS := $(shell pkg-config --cflags libtorrent-rasterbar) + TORRENT_LIBS := $(shell pkg-config --libs libtorrent-rasterbar) + ifeq ($(HOST_OS),Darwin) + TORRENT_BOOST_PREFIX := $(shell brew --prefix boost 2>/dev/null) + TORRENT_CFLAGS += -I$(TORRENT_BOOST_PREFIX)/include + TORRENT_OPENSSL_PREFIX := $(shell brew --prefix openssl@3 2>/dev/null) + TORRENT_CFLAGS += -I$(TORRENT_OPENSSL_PREFIX)/include + endif + endif + CFLAGS += -DENABLE_LIBTORRENT=1 + TORRENT_CXXFLAGS := -std=c++17 -Wall -Wextra -Wpedantic -Werror -O2 -fPIE \ + $(PLATFORM_DEFS) -I./include $(TORRENT_CFLAGS) + ifeq ($(TARGET),ps5) + TORRENT_CXXFLAGS += -fvisibility=hidden -fno-inline-functions + endif + LIBS += $(TORRENT_LIBS) + TORRENT_OBJECT := $(OBJ_DIR)/transfer/backend_torrent.o + CXX_LINK := $(CXX) +else + CXX_LINK := $(CC) +endif + # ── libnfs detection ────────────────────────────────────────────────────── ifeq ($(ENABLE_LIBNFS),1) ifeq ($(TARGET),ps5) @@ -496,6 +633,7 @@ endif OBJECTS := $(patsubst src/%.c,$(OBJ_DIR)/%.o,$(filter src/%.c,$(SOURCES))) OBJECTS += $(patsubst mcp/src/%.c,$(OBJ_DIR)/mcp/%.o,$(filter mcp/src/%.c,$(SOURCES))) OBJECTS += $(patsubst $(BUILD_DIR)/generated/%.c,$(OBJ_DIR)/generated/%.o,$(filter $(BUILD_DIR)/generated/%.c,$(SOURCES))) +OBJECTS += $(TORRENT_OBJECT) # FFI Object files FFI_SOURCES := ffi/c_core/pal_ffi.c @@ -612,7 +750,7 @@ $(PROJECT): all $(OUTPUT_ELF): $(OBJECTS) | $(BIN_DIR) @echo " [LD] $@" @mkdir -p $(BIN_DIR) - @$(CC) $(LDFLAGS) -o $@ $^ $(LIBS) + @$(CXX_LINK) $(LDFLAGS) -o $@ $^ $(LIBS) @echo "Build complete: $(PROJECT) ($(TARGET), $(BUILD_TYPE))" # FFI Build Targets @@ -638,7 +776,7 @@ endif $(FFI_OUTPUT): $(LIB_OBJECTS) $(FFI_OBJECTS) | $(BIN_DIR) @echo " [LD] $@ (Shared Library)" @mkdir -p $(BIN_DIR) - @$(CC) $(LDFLAGS) $(FFI_LDFLAGS) -fPIC -o $@ $(LIB_OBJECTS) $(FFI_OBJECTS) $(LIBS) + @$(CXX_LINK) $(LDFLAGS) $(FFI_LDFLAGS) -fPIC -o $@ $(LIB_OBJECTS) $(FFI_OBJECTS) $(LIBS) @echo "FFI C-Core built: $@" # Build all supported platforms (best-effort: includes only toolchains found on the host). @@ -735,18 +873,21 @@ debug-all: echo "==> Building $$t (debug)"; \ $(MAKE) TARGET=$$t BUILD_TYPE=debug clean all; \ done -# Generate the PS5 trust store from PacBrew's Mozilla CA bundle. -ifeq ($(TARGET),ps5) -ifeq ($(ENABLE_LIBCURL),1) +# Generate the embedded trust store from PacBrew's Mozilla CA bundle (PS4/PS5). +ifneq ($(strip $(CURL_CA_BUNDLE_SRC)),) $(OBJ_DIR)/transfer/backend_curl.o: $(GENERATED_CA_HEADER) -$(GENERATED_CA_HEADER): $(PS5_CA_BUNDLE) tools/embed_binary.py +$(GENERATED_CA_HEADER): $(CURL_CA_BUNDLE_SRC) tools/embed_binary.py @echo " [GEN] $@" @python3 tools/embed_binary.py $< $@ zftpd_ca_bundle endif -endif # Compile C source files +$(OBJ_DIR)/transfer/backend_torrent.o: src/transfer/backend_torrent.cpp | $(OBJ_DIR) $(DEP_DIR) + @echo " [CXX] $<" + @mkdir -p $(dir $@) $(dir $(DEP_DIR)/transfer/backend_torrent.d) + @$(CXX) $(TORRENT_CXXFLAGS) -MMD -MP -MF $(DEP_DIR)/transfer/backend_torrent.d -MT $@ -c $< -o $@ + $(OBJ_DIR)/%.o: src/%.c | $(OBJ_DIR) $(DEP_DIR) @echo " [CC] $<" @mkdir -p $(dir $@) $(dir $(DEP_DIR)/$*.d) @@ -821,13 +962,31 @@ TEST_BINS += $(BUILD_DIR)/tests/test_copy_atomic TEST_BINS += $(BUILD_DIR)/tests/test_ftp_commands TEST_BINS += $(BUILD_DIR)/tests/test_fileio_tree TEST_BINS += $(BUILD_DIR)/tests/test_fileio_basic +TEST_BINS += $(BUILD_DIR)/tests/test_ps5_fw_offsets ifeq ($(ENABLE_ZHTTPD),1) TEST_BINS += $(BUILD_DIR)/tests/test_transfer +TEST_BINS += $(BUILD_DIR)/tests/test_transfer_parallel +TEST_BINS += $(BUILD_DIR)/tests/test_transfer_state +TEST_BINS += $(BUILD_DIR)/tests/test_transfer_queue +TEST_BINS += $(BUILD_DIR)/tests/test_transfer_orphans +ifeq ($(ENABLE_LIBTORRENT),1) +TEST_BINS += $(BUILD_DIR)/tests/test_transfer_torrent +endif +TEST_BINS += $(BUILD_DIR)/tests/test_http_parser +TEST_BINS += $(BUILD_DIR)/tests/test_http_upload +TEST_BINS += $(BUILD_DIR)/tests/test_http_server +TEST_BINS += $(BUILD_DIR)/tests/test_http_response_stream TEST_BINS += $(BUILD_DIR)/tests/test_http_api_common TEST_BINS += $(BUILD_DIR)/tests/test_http_files TEST_BINS += $(BUILD_DIR)/tests/test_http_process TEST_BINS += $(BUILD_DIR)/tests/test_http_system TEST_BINS += $(BUILD_DIR)/tests/test_http_games +TEST_BINS += $(BUILD_DIR)/tests/test_archive_path +TEST_BINS += $(BUILD_DIR)/tests/test_builtin_unzip +TEST_BINS += $(BUILD_DIR)/tests/test_zip_writer +TEST_BINS += $(BUILD_DIR)/tests/test_pal_volume +TEST_BINS += $(BUILD_DIR)/tests/test_http_archive +TEST_BINS += $(BUILD_DIR)/tests/test_dump_job endif ifeq ($(filter $(TARGET),linux macos),) @@ -836,16 +995,20 @@ test: $(OUTPUT_BIN) else test: $(OUTPUT_ELF) $(TEST_BINS) @echo "Running tests..." - @for t in $(TEST_BINS); do ./$$t; done + @set -e; for t in $(TEST_BINS); do ./$$t; done endif $(BUILD_DIR)/tests/test_http_query: tests/test_http_query.c $(LIB_OBJECTS) | $(BUILD_DIR)/tests @echo " [CC] $<" - @$(CC) $(CFLAGS) -DFTP_AUTH_DELAY=0 -DFTP_PORT_ALLOW_FOREIGN_IP=1 -o $@ $< $(LIB_OBJECTS) $(LDFLAGS) $(LIBS) + @$(CXX_LINK) $(CFLAGS) -DFTP_AUTH_DELAY=0 -DFTP_PORT_ALLOW_FOREIGN_IP=1 -o $@ -x c $< -x none $(LIB_OBJECTS) $(LDFLAGS) $(LIBS) + +$(BUILD_DIR)/tests/test_transfer_torrent: tests/test_transfer_torrent.cpp $(TORRENT_OBJECT) | $(BUILD_DIR)/tests + @echo " [CXX] $<" + @$(CXX) $(TORRENT_CXXFLAGS) -o $@ $< $(TORRENT_OBJECT) $(LDFLAGS) $(TORRENT_LIBS) -lpthread $(BUILD_DIR)/tests/%: tests/%.c $(LIB_OBJECTS) | $(BUILD_DIR)/tests @echo " [CC] $<" - @$(CC) $(CFLAGS) -DFTP_AUTH_DELAY=0 -DFTP_PORT_ALLOW_FOREIGN_IP=1 -o $@ $< $(LIB_OBJECTS) $(LDFLAGS) $(LIBS) + @$(CXX_LINK) $(CFLAGS) -DFTP_AUTH_DELAY=0 -DFTP_PORT_ALLOW_FOREIGN_IP=1 -o $@ -x c $< -x none $(LIB_OBJECTS) $(LDFLAGS) $(LIBS) bin: $(OUTPUT_BIN) @@ -972,3 +1135,44 @@ web-deploy: @echo " [WEB] web-deploy is deprecated — web UI is now embedded in the binary" @echo " [WEB] Resources are generated automatically under build/.../generated" @echo " [WEB] Done — $(shell find web -path web/legacy -prune -o \( -name '*.css' -o -name '*.js' \) -print | wc -l | tr -d ' ') files deployed" + +#============================================================================ +# Flag changes must invalidate object files. +# +# Objects do not record the flags they were built with, so toggling a feature +# (ENABLE_LIBTORRENT, ENABLE_PKG_INSTALL, ...) used to relink stale objects and +# fail with undefined symbols. The stamp is rewritten only when the flags +# actually change, and every object depends on it, so a real change rebuilds. +#============================================================================ + +FLAG_STAMP := $(OBJ_DIR)/.build-flags + +.PHONY: FORCE +FORCE: + +$(FLAG_STAMP): FORCE + @mkdir -p $(OBJ_DIR) + @printf '%s\n' '$(CFLAGS)' '$(LDFLAGS)' '$(LIBS)' > $@.tmp + @if ! cmp -s $@.tmp $@; then \ + echo " [FLAGS] build flags changed — objects will be rebuilt"; \ + mv -f $@.tmp $@; \ + fi + @rm -f $@.tmp + +$(OBJECTS) $(LIB_OBJECTS): $(FLAG_STAMP) + +#============================================================================ +# PS5 installer helper +#============================================================================ + +ifneq ($(INSTALL_HELPER_ELF),) +$(INSTALL_HELPER_ELF): src/platform/ps5/install_helper/install_helper.c | $(BIN_DIR) + @echo " [CC] $<" + @$(CXX_LINK) $(CFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) + +$(INSTALL_HELPER_BLOB): $(INSTALL_HELPER_ELF) tools/gen_blob.py + @mkdir -p $(dir $@) + @python3 tools/gen_blob.py $< install_helper_elf $@ + +$(OBJ_DIR)/generated/install_helper_blob.o: $(INSTALL_HELPER_BLOB) +endif diff --git a/README.md b/README.md index f16b466..170ecec 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ To make these persistent, add them to `/etc/sysctl.conf`. - Launch after the system is fully booted and the loader is ready. The on-screen notification will display the IP and port. - For maximum throughput: direct cable from console to PC, static IPs, no router in between. - Avoid initiating transfers while background downloads or system updates are active — the network stack is shared. -- If you see **"payload already loaded"**: a previous instance is still active. `zftpd` will attempt to terminate it and restart on the default port. If that fails, it tries up to 9 subsequent ports (`FTP_DEFAULT_PORT+1` … `+9`). +- If you see **"payload already loaded"**: a previous instance is still active. The new payload asks it to shut down on its control port (`127.0.0.1:28888`) and the previous instance runs its own teardown even when its threads are blocked, so it always releases the FTP, HTTP and MCP ports. Instances that predate the control port — or one wedged after accepting it — are identified by thread name and stopped, never signalling the loader's process group. If a payload still holds the console, the new instance stops instead of binding ports another instance owns. @@ -301,7 +301,7 @@ Output artifacts are versioned and platform-tagged, placed in `build//`) and the rest wait, showing their position. +Queued jobs can be held back or cancelled before they ever start; up to +`TRANSFER_MAX_ACTIVE` (16) jobs are tracked. + +HTTP/FTP/NFS downloads write to `.zftpd.part` and rename it after +`fsync()`. Magnet downloads write a `.zftpd.part` directory containing +the torrent files; the directory is renamed after every piece is verified. +Download status, destination and progress are recorded +atomically in `/data/zftpd/transfers.state` (falling back to `/tmp/zftpd/`). +BitTorrent resume data is stored alongside that state file. On startup zftpd +restores the visible list and resumes unfinished jobs from the bytes already +on disk. Paused jobs stay paused, and failed +jobs remain visible for retry. The Transfers view can delete a saved partial +and its record, or use Start over to delete it and start the same link from +zero. Removing a completed record leaves the completed file untouched. ### Artifacts @@ -401,6 +442,12 @@ After console Rest Mode, ZHTTP auto-reconnects via `/api/status` (see [docs/rest Custom console toasts are available via `GET /api/notify?text=Hello` (useful for Home Assistant and similar local automation — see [docs/API.md](docs/API.md)). +The System view also drives the hardware: fan threshold, network-listener restart and **Blu-ray eject** (`POST /api/system/eject`, hidden on Digital Edition consoles). The eject path comes from [BD-EJ](https://github.com/seregonwar/BD-EJ). + +Downloads and previews send the raw file bytes. Console **SELF containers** can be requested decrypted instead, exactly like the FTP path does: enable *Settings → Downloads → Decrypt protected files* (or pass `?decrypt=1` to `/api/file/get`). The kernel pager is swapped in for that transfer only, so every other file — and every other client — keeps getting the on-disk bytes untouched. + +Selecting many items (folders included) offers **Download as ZIP**: the archive is built *while* it streams (`/api/archive/zip`), so nothing is written to the console and even multi-gigabyte selections need no free space. + --- ## Acknowledgements diff --git a/docs/dependencies.md b/docs/dependencies.md index 8faa04d..8f3cc44 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -2,7 +2,38 @@ zftpd keeps protocol implementations outside the project when a mature, maintained library already exists. Console builds use the ports shipped by -the PS5 PacBrew ecosystem rather than local compatibility shims. +the PacBrew ecosystem rather than local compatibility shims. + +## PS4 zhttp transfer stack + +| Component | Purpose | Source | +| --- | --- | --- | +| libcurl 7.80.0 | HTTP, HTTPS, FTP and FTPS transfers | PacBrew `ps4-openorbis-libcurl` (static, links `-lSceNet`) | +| mbedTLS 2.16.6 | TLS used by libcurl | PacBrew `ps4-openorbis-mbedtls` | +| zlib 1.3.1 | libcurl transfer dependency | PacBrew `ps4-openorbis-zlib` | +| Mozilla CA bundle | HTTPS trust store | downloaded by the fetch script and embedded at build time | + +`tools/fetch_pacbrew_ps4.sh` installs the pinned PacBrew packages into +`external/pacbrew/ps4/openorbis` (versions and SHA-256 are pinned in the +script, and the packages come from the public PacBrew repository). No pacman +and no OpenOrbis toolchain are needed: the payload keeps being built with +`PS4_PAYLOAD_SDK`, and only the headers and the static libraries are used — +the `curl-config` wrapper inside the package emits the OpenOrbis toolchain's +own link flags (its linker script, `-nostdlib`, `-lSceLibcInternal`), which +must not leak into a payload link. The Makefile picks the prefix up +automatically (`PS4_PACBREW_ROOT`, defaulting to the path above and to +`/opt/pacbrew/ps4/openorbis`), and the release workflow installs the same set +for the published PS4 artifacts. + +The OpenOrbis build of libcurl reaches `errno` through musl's +`__errno_location()` and uses `fnmatch()`/`execl()`, none of which the payload +libc exports; `src/platform/ps4/ps4_pacbrew_compat.c` bridges the three. + +With no prefix installed the URL downloader is simply disabled at build time. +A toolchain-provided `orbis-pkg-config` / `orbis-curl-config` is still probed +as a fallback, and `PS4_PKG_CONFIG`, `PS4_CURL_CONFIG` and `PS4_CA_BUNDLE` +override the detection. The build never falls back to the host's libcurl, +which would link code built for the wrong OS. ## PS5 zhttp transfer stack @@ -12,6 +43,8 @@ the PS5 PacBrew ecosystem rather than local compatibility shims. | OpenSSL | TLS used by libcurl | PacBrew dependency of `ps5-payload-curl` | | zlib / zstd / libpsl | libcurl transfer dependencies | PacBrew | | libnfs 5.0.3 | Userspace NFS client | `ps5-payload-dev/pacbrew-repo` (`ps5-payload-libnfs`) | +| libtorrent-rasterbar 2.1.2 | Native BitTorrent magnet downloads | Cross-built from the pinned upstream source by `tools/build_ps5_libtorrent.sh` | +| libarchive (optional) | Non-ZIP archive extraction with `ENABLE_LIBARCHIVE=1` | `ps5-payload-dev/pacbrew-repo` (`ps5-payload-libarchive`) | | Mozilla CA bundle | HTTPS trust store | generated by the PacBrew curl package and embedded at build time | The release workflow pins PacBrew `v0.40.2` and verifies the published @@ -26,10 +59,26 @@ Backends are selected by URL scheme: - `http://`, `https://`, `ftp://`, `ftps://` -> libcurl - `nfs://` -> libnfs +- `magnet:?` -> libtorrent-rasterbar in PS5 builds with `ENABLE_LIBTORRENT=1` -Downloads are written to `.zftpd.part`, support resume where the remote -protocol permits it, and are atomically renamed to the final filename only -after the transfer and `fsync()` succeed. +HTTP/FTP/NFS downloads are written to `.zftpd.part`, support resume +where the remote protocol permits it, and are renamed after the transfer and +`fsync()` succeed. Magnet downloads use a partial directory with the same +suffix and store resume data in `/data/zftpd` (or `/tmp/zftpd` fallback). +The directory becomes visible under its final name only after libtorrent +verifies all pieces. + +For new HTTP/HTTPS downloads of at least 128 MiB, libcurl first checks byte-range +support and a strong ETag, then measures one 32 MiB range. If that connection +is below 16 MB/s, four range requests run concurrently in 8 MiB chunks. Each +wave is verified and appended in order, so the partial file stays contiguous +and can still resume after interruption. An existing partial file can also use +four connections: before fetching its suffix, zftpd compares small regions at +the start, middle and end of the local prefix with the current remote file. +A mismatch stops the download without changing the partial file. This sampling +does not prove every byte of a legacy partial matches; new suffix ranges are +checked against the same strong ETag. Servers without range support or a strong +ETag continue with one connection. NFS URLs use libnfs syntax, for example: diff --git a/external/read.txt b/external/read.txt index 084c2a3..fc310c5 100644 --- a/external/read.txt +++ b/external/read.txt @@ -1 +1,2 @@ -put here ps4-payload-sdk, ps5-payload-sdk \ No newline at end of file +put here ps4-payload-sdk, ps5-payload-sdk +(pacbrew PS4 portlibs are installed by tools/fetch_pacbrew_ps4.sh into pacbrew/) diff --git a/src/platform/ps4/ps4_pacbrew_compat.c b/src/platform/ps4/ps4_pacbrew_compat.c new file mode 100644 index 0000000..0fb8061 --- /dev/null +++ b/src/platform/ps4/ps4_pacbrew_compat.c @@ -0,0 +1,94 @@ +/* + * MIT License — Copyright (c) 2026 SeregonWar + * See LICENSE for full text. + */ + +/** + * @file ps4_pacbrew_compat.c + * @brief libc glue for the PacBrew ps4-openorbis libcurl + * + * The PacBrew port of libcurl is built against the OpenOrbis musl libc while + * the payload itself links against the ps4-payload-sdk. Three symbols the + * OpenOrbis build expects do not exist in the payload runtime: + * + * __errno_location() musl reaches errno through this accessor; the payload + * libc exports __error() instead (see ). + * fnmatch() curl uses it for file:// wildcards. + * execl() curl uses it to delegate NTLM authentication. + * + * zftpd's downloader reaches neither of the two functions, so they are + * implemented with plain behavior rather than left to break the link. + * + * The file is compiled only for PS4 builds with ENABLE_LIBCURL=1. + */ + +#include +#include +#include +#include + +int *__errno_location(void); +int *__errno_location(void) { + return __error(); +} + +int fnmatch(const char *pattern, const char *string, int flags); +int fnmatch(const char *pattern, const char *string, int flags) { + (void)flags; + while (*pattern != '\0') { + if (*pattern == '*') { + while (*pattern == '*') pattern++; + if (*pattern == '\0') return 0; + while (*string != '\0') { + if (fnmatch(pattern, string, 0) == 0) return 0; + string++; + } + return FNM_NOMATCH; + } + if (*string == '\0') return FNM_NOMATCH; + if (*pattern == '?') { + pattern++; + string++; + continue; + } + if (*pattern == '[') { + const char *set = pattern + 1; + int negate = (*set == '!' || *set == '^'); + int matched = 0; + int first = 1; + char previous = '\0'; + if (negate) set++; + while (*set != '\0' && (*set != ']' || first)) { + first = 0; + if (*set == '-' && previous != '\0' && set[1] != '\0' && set[1] != ']') { + if ((unsigned char)previous <= (unsigned char)*string && + (unsigned char)*string <= (unsigned char)set[1]) + matched = 1; + set += 2; + previous = '\0'; + continue; + } + if (*set == *string) matched = 1; + previous = *set; + set++; + } + if (*set != ']' || matched == negate) return FNM_NOMATCH; + pattern = set + 1; + string++; + continue; + } + if (*pattern == '\\' && pattern[1] != '\0') pattern++; + if (*pattern != *string) return FNM_NOMATCH; + pattern++; + string++; + } + return (*string == '\0') ? 0 : FNM_NOMATCH; +} + +int execl(const char *path, const char *arg, ...); +int execl(const char *path, const char *arg, ...) { + (void)path; + (void)arg; + errno = ENOSYS; + return -1; +} diff --git a/tools/fetch_pacbrew_ps4.sh b/tools/fetch_pacbrew_ps4.sh new file mode 100755 index 0000000..abf5dd1 --- /dev/null +++ b/tools/fetch_pacbrew_ps4.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Fetch the PacBrew PS4 port libraries that back the URL downloader. +# +# PacBrew ships ps4-openorbis-libcurl (static, mbedTLS, links -lSceNet) and its +# dependencies as pacman packages. This script unpacks the pinned set into a +# plain prefix — no pacman and no OpenOrbis toolchain are needed, because the +# payload keeps being built with PS4_PAYLOAD_SDK: only the headers and the +# static libraries are used. (The curl-config wrapper inside the package emits +# OpenOrbis-only link flags such as its own linker script and -nostdlib, which +# must not leak into a payload link.) +# +# The Makefile detects the prefix automatically (PS4_PACBREW_ROOT), so after +# running this script `make TARGET=ps4 ENABLE_ZHTTPD=1` builds with the URL +# downloader enabled and the CA bundle embedded. +# +# Usage: tools/fetch_pacbrew_ps4.sh [PREFIX] +# PREFIX extraction root (default: external/pacbrew/ps4/openorbis) +# +# Environment: +# PACBREW_PS4_REPO package server (default: the public PacBrew repo) +# PS4_CA_BUNDLE_FILE reuse an existing Mozilla CA bundle instead of +# downloading one from curl.se +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +prefix="${1:-$script_dir/../external/pacbrew/ps4/openorbis}" +prefix="${prefix%/}" +repo="${PACBREW_PS4_REPO:-https://pacman.mydedibox.fr/pacbrew/packages}" +ca_file="${PS4_CA_BUNDLE_FILE:-}" +cache="$(dirname "$prefix")/.downloads" + +# name sha256 — pinned like the PS5 PacBrew bundle in the release workflow. +packages=( + "ps4-openorbis-zlib-1.3.1-2-any.pkg.tar.xz 2b082d25f34d923e2fd35e0b5f29b19e142e263da4380665bb0e80f0a6378700" + "ps4-openorbis-mbedtls-2.16.6-3-any.pkg.tar.xz 7fe7ad512c2ccf7b53865433753b9ddcaf6385eed36b8160d93134fa64f0dc58" + "ps4-openorbis-libcurl-7.80.0-3-any.pkg.tar.xz 249c987a8e41e1b7d320cd3129aa1752416dc19e830714d5a38be6c977f5db98" +) + +sha_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +mkdir -p "$cache" "$prefix" +prefix="$(cd "$prefix" && pwd -P)" +work="$(mktemp -d "${TMPDIR:-/tmp}/zftpd-pacbrew-ps4.XXXXXXXX")" +trap 'rm -rf "$work"' EXIT + +for entry in "${packages[@]}"; do + file="${entry%% *}" + want="${entry##* }" + archive="$cache/$file" + if [[ ! -f "$archive" ]] || [[ "$(sha_of "$archive")" != "$want" ]]; then + echo "downloading $file" + curl -fL --retry 3 -o "$archive.tmp" "$repo/$file" + mv "$archive.tmp" "$archive" + fi + if [[ "$(sha_of "$archive")" != "$want" ]]; then + echo "error: checksum mismatch for $archive" >&2 + exit 1 + fi + tar -xJf "$archive" -C "$work" +done + +# The packages carry the upstream /opt/pacbrew/ps4/openorbis layout; flatten +# it into the prefix so the include/ and lib/ trees can be used directly. +if [[ -d "$work/opt/pacbrew/ps4/openorbis" ]]; then + cp -R "$work/opt/pacbrew/ps4/openorbis/." "$prefix/" +fi +test -f "$prefix/usr/lib/libcurl.a" +test -f "$prefix/usr/lib/libmbedtls.a" +test -f "$prefix/usr/lib/libz.a" + +# HTTPS verification must not degrade silently: keep a Mozilla bundle next to +# the libraries, where the Makefile looks for it (embedded into the payload). +bundle="$prefix/usr/etc/ca-bundle.crt" +if [[ -n "$ca_file" ]]; then + mkdir -p "$(dirname "$bundle")" + cp "$ca_file" "$bundle" +fi +if [[ ! -s "$bundle" ]]; then + mkdir -p "$(dirname "$bundle")" + echo "downloading Mozilla CA bundle" + curl -fL --retry 3 -o "$bundle.tmp" https://curl.se/ca/cacert.pem + mv "$bundle.tmp" "$bundle" +fi +grep -q -- "-----BEGIN CERTIFICATE-----" "$bundle" + +echo +echo "PacBrew PS4 portlibs installed in $prefix" +echo " PS4_PACBREW_ROOT=$prefix" +echo " PS4_CA_BUNDLE=$bundle" +echo "make TARGET=ps4 ENABLE_ZHTTPD=1 now builds with the URL downloader enabled"