From b7b1ccaf5834ca46db91d4d49e095bd2d65585b6 Mon Sep 17 00:00:00 2001 From: phantomptr Date: Sat, 17 Oct 2026 11:07:03 -0700 Subject: [PATCH 1/3] Page through the app database so no installed game is hidden (#473) APP_DB_QUERY answered at most 256 rows, and fewer when its 64 KiB reply filled, with nothing saying the list was cut. A title installed by a homebrew installer (no /user/appmeta folder) that sorted past that point disappeared from the installed list. Payload: app.db.query takes an optional {"offset","limit"} window and answers offset/next/more/truncated; a full buffer now ends the page with more=true instead of closing the list. An empty request is page one, so older engines keep working. Engine: appdb_query follows the pages (at most 32) and marks the list truncated when it could not finish (or an older helper returned a full 256). The installed list reads app.db once per request, adds registered titles whose app folder exists, reports incomplete, and a truncated list never proves a title absent for install verification. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RxCGb8Asp1A1AQYiv1kbgS --- CHANGELOG.md | 9 + client/src/api/ps5.ts | 14 +- client/src/screens/InstalledApps/index.tsx | 5 + client/src/screens/Library/index.tsx | 2 +- engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c | 6 +- .../crates/ps5upload-core/src/diagnostics.rs | 86 ++++++- .../crates/ps5upload-core/src/pkg_install.rs | 58 ++++- .../ps5upload-core/tests/appdb_paging.rs | 210 ++++++++++++++++ engine/crates/ps5upload-engine/src/lib.rs | 212 ++++++++++++---- payload/include/content_db.h | 27 +- payload/src/content_db.c | 146 ++++++++--- payload/src/mgmt_table.def | 2 +- payload/src/runtime.c | 28 ++- payload/tests/content_db_selftest.c | 237 ++++++++++++++++++ protocol/ava1/MGMT_METHODS.md | 2 +- 15 files changed, 947 insertions(+), 97 deletions(-) create mode 100644 engine/crates/ps5upload-core/tests/appdb_paging.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c8de113a..59dd48a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ What's new in ps5upload, written for humans. --- +## Unreleased + +- **Big app databases no longer hide installed games.** The console's app database was read + only up to its first 256 entries, and a long list could be cut shorter still, with nothing + saying so. On a console with many entries, a game installed by a homebrew installer (no + home-screen metadata folder) that sorted past that point was missing from Games. The app now + reads the whole database page by page, also lists any game it registered whose folder is on + the console, and logs a warning when a list could not be read in full (#473). + ## 7.0.34 - **Unity games no longer crash when they run as a folder.** Some game dumps name a file diff --git a/client/src/api/ps5.ts b/client/src/api/ps5.ts index c3439bd9..2fa3c39e 100644 --- a/client/src/api/ps5.ts +++ b/client/src/api/ps5.ts @@ -2853,6 +2853,10 @@ export interface AppDbEntry { export interface AppDbList { apps: AppDbEntry[]; err?: string; + /** Which reader answered: "sqlite", "sqlite-immutable" or "scan". */ + source?: string; + /** True when app.db holds rows this list does not (#473). */ + truncated?: boolean; } export async function appdbQuery(addr: string): Promise { @@ -3519,6 +3523,9 @@ export interface InstalledAppsResult { /** True if the payload couldn't report the "registered" set (older * payload); the UI then shows everything as "installed" with a note. */ registeredUnavailable: boolean; + /** True if the console's list may be missing titles (a listing hit its + * page size, or app.db could not be read in full, #473). */ + incomplete: boolean; } /** List every installed title on the PS5, grouped by origin. Routes through @@ -3538,6 +3545,7 @@ export async function appsInstalled( system?: boolean; }>; registered_unavailable?: boolean; + incomplete?: boolean; }>("ps5_apps_installed", { addr }); const titles: InstalledTitle[] = (res?.titles ?? []) .filter((t): t is { title_id: string } & typeof t => !!t.title_id) @@ -3550,7 +3558,11 @@ export async function appsInstalled( source: t.source ?? "", system: !!t.system, })); - return { titles, registeredUnavailable: !!res?.registered_unavailable }; + return { + titles, + registeredUnavailable: !!res?.registered_unavailable, + incomplete: !!res?.incomplete, + }; } export type InstalledPkgKind = "base" | "patch" | "dlc"; diff --git a/client/src/screens/InstalledApps/index.tsx b/client/src/screens/InstalledApps/index.tsx index b36e737f..af14b559 100644 --- a/client/src/screens/InstalledApps/index.tsx +++ b/client/src/screens/InstalledApps/index.tsx @@ -812,6 +812,11 @@ export default function InstalledAppsScreen() { if (probe.isStale()) return; setTitles(res.titles); setRegisteredUnavailable(res.registeredUnavailable); + if (res.incomplete) { + console.warn( + "[installed] the console's installed list may be incomplete (a listing or app.db read did not finish); refresh to retry", + ); + } try { const sdk = await sdkScan(mgmtAddr(probe.host)); if (!probe.isStale()) setSdkState(sdk); diff --git a/client/src/screens/Library/index.tsx b/client/src/screens/Library/index.tsx index 9d03f11d..e9edb769 100644 --- a/client/src/screens/Library/index.tsx +++ b/client/src/screens/Library/index.tsx @@ -281,7 +281,7 @@ export default function LibraryScreen() { "[library] installed-apps probe failed; disk-image rows will show filenames only:", e, ); - return { titles: [], registeredUnavailable: true }; + return { titles: [], registeredUnavailable: true, incomplete: true }; }), ]); // Two ways a registered title points back at a Library row: diff --git a/engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c b/engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c index bf08e99c..b5de9665 100644 --- a/engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c +++ b/engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c @@ -174,9 +174,9 @@ static int handle_appinfo_set(void *st, int fd, uint64_t t, const char *b, uint6 if (strstr(b, "RUNNING")) return ok_body("{\"ok\":false,\"err\":\"title is running\"}"); return ok_body("{\"ok\":true,\"err\":null}"); } -static int handle_appdb_query(void *st, int fd, uint64_t t) { - (void)st; (void)fd; (void)t; - note(AVA1_METHOD_APP_DB_QUERY, NULL); +static int handle_appdb_query(void *st, int fd, uint64_t t, const char *b, uint64_t l) { + (void)st; (void)fd; (void)t; (void)l; + note(AVA1_METHOD_APP_DB_QUERY, b); return ok_body("{\"apps\":[{\"title_id\":\"CUSA00001\",\"app_id\":57368,\"name\":\"Stub\"}],\"source\":\"sql\"}"); } static int handle_focus_probe(void *st, int fd, uint64_t t) { diff --git a/engine/crates/ps5upload-core/src/diagnostics.rs b/engine/crates/ps5upload-core/src/diagnostics.rs index 18af6dce..368a8d1b 100644 --- a/engine/crates/ps5upload-core/src/diagnostics.rs +++ b/engine/crates/ps5upload-core/src/diagnostics.rs @@ -247,12 +247,92 @@ pub struct AppDbList { /// a caller that sees a blank name knows whether to believe it. #[serde(default)] pub source: Option, + /// One page's window (#473): the first row it holds, the row the next page starts at, and + /// whether one follows. Absent from an older helper, which answers one unpaged reply, and + /// from the list [`appdb_query`] returns, which is every page joined. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offset: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub more: Option, + /// True when app.db holds rows this list does not: the helper could not deliver them, a + /// page failed (`err` says why), the page bound was reached, or an older helper answered + /// a full 256 rows (its old cap, which it could not report). A truncated list is never + /// proof that a title is absent. + #[serde(default)] + pub truncated: bool, } -/// Query app.db for the title_id ↔ app_id ↔ name mapping. +/// The most app.db pages [`appdb_query`] reads. A page holds up to 512 rows, so this is +/// far past any real console; it bounds the loop against a helper that never says it is done. +pub const APPDB_MAX_PAGES: usize = 32; + +/// The row count an older, unpaged helper stopped at without saying so. +const APPDB_LEGACY_CAP: usize = 256; + +/// Query app.db for the title_id ↔ app_id ↔ name mapping: every page of it (#473). +/// +/// The helper answers a window (`{"offset":N}` in; `next`, `more` and `truncated` out); this +/// follows it until `more` is false, at most [`APPDB_MAX_PAGES`] calls. An older helper ignores +/// the window and answers once, which is then the whole answer. Whatever stops the walk early +/// sets `truncated` on the result, so callers can tell a partial list from a complete one: +/// a failure after the first page keeps the rows read so far, with the reason in `err`. pub fn appdb_query(addr: &str) -> Result { - let resp = mgmt::call(addr, m::APP_DB_QUERY, &[])?; - Ok(serde_json::from_slice(&resp)?) + let mut list: Option = None; + let mut offset: u64 = 0; + for _ in 0..APPDB_MAX_PAGES { + let body = serde_json::json!({ "offset": offset }).to_string(); + let page = mgmt::call(addr, m::APP_DB_QUERY, body.as_bytes()) + .and_then(|resp| Ok(serde_json::from_slice::(&resp)?)); + let mut page = match (page, list.as_mut()) { + (Ok(p), _) => p, + // Page one failing is the call failing, as it always was. + (Err(e), None) => return Err(e), + (Err(e), Some(l)) => { + l.truncated = true; + l.err = Some(format!("app.db page at row {offset}: {e:#}")); + break; + } + }; + let paged = page.more.is_some() || page.next.is_some(); + let more = page.more.unwrap_or(false); + let next = page.next.unwrap_or(offset + page.apps.len() as u64); + let got = page.apps.len(); + let page_truncated = page.truncated + || (!paged && got >= APPDB_LEGACY_CAP) + // A page that claims more but does not move forward would loop. + || (more && next <= offset); + match list.as_mut() { + None => { + page.truncated = page_truncated; + list = Some(page); + } + Some(l) => { + l.apps.append(&mut page.apps); + l.truncated |= page_truncated; + if page.err.is_some() { + l.err = page.err.take(); + l.truncated = true; + } + } + } + let l = list.as_mut().expect("set above"); + if !paged || !more || next <= offset || l.err.is_some() { + l.offset = None; + l.next = None; + l.more = None; + return Ok(list.expect("set above")); + } + offset = next; + } + // Here only when a later page failed or the page bound ran out with pages still to read. + let mut l = list.expect("APPDB_MAX_PAGES > 0"); + l.truncated = true; + l.offset = None; + l.next = None; + l.more = None; + Ok(l) } /// One (key, value) row of appinfo.db for a title. diff --git a/engine/crates/ps5upload-core/src/pkg_install.rs b/engine/crates/ps5upload-core/src/pkg_install.rs index 41856af4..4b4a07df 100644 --- a/engine/crates/ps5upload-core/src/pkg_install.rs +++ b/engine/crates/ps5upload-core/src/pkg_install.rs @@ -320,14 +320,26 @@ fn probe_app_pkg_at(addr: &str, app_dir: &str) -> PkgProbe { /// Whether app.db lists `title_id`. `Some(true/false)` only when the query /// actually returned a row set (`err == None`); `None` when the database /// could not be read at all, so an unreadable database never counts as -/// "the title is absent". +/// "the title is absent". Likewise a list the helper could only partly +/// deliver (`truncated`, #473): finding the title there is proof, missing +/// it is not. fn appdb_has_title(addr: &str, title_id: &str) -> Option { match crate::diagnostics::appdb_query(addr) { - Ok(list) if list.err.is_none() => Some(list.apps.iter().any(|a| a.title_id == title_id)), + Ok(list) if list.err.is_none() => appdb_list_has_title(&list, title_id), _ => None, } } +fn appdb_list_has_title(list: &crate::diagnostics::AppDbList, title_id: &str) -> Option { + if list.apps.iter().any(|a| a.title_id == title_id) { + Some(true) + } else if list.truncated { + None + } else { + Some(false) + } +} + /// Check whether `title_id` (e.g. "PPSA01650") installed a launchable title /// on the PS5. /// @@ -498,3 +510,45 @@ mod tests { assert_eq!(p, InstallPhase::Done); } } + +#[cfg(test)] +mod appdb_list_has_title_tests { + use super::appdb_list_has_title; + use crate::diagnostics::{AppDbEntry, AppDbList}; + + fn list(ids: &[&str], truncated: bool) -> AppDbList { + AppDbList { + apps: ids + .iter() + .map(|t| AppDbEntry { + title_id: t.to_string(), + app_id: 1, + name: String::new(), + }) + .collect(), + err: None, + source: None, + offset: None, + next: None, + more: None, + truncated, + } + } + + /// #473: a title missing from a list the helper could not finish is unknown, not absent. + #[test] + fn a_truncated_list_never_proves_a_title_absent() { + assert_eq!( + appdb_list_has_title(&list(&["CUSA00001"], true), "CUSA00001"), + Some(true) + ); + assert_eq!( + appdb_list_has_title(&list(&["CUSA00001"], true), "PPSA00002"), + None + ); + assert_eq!( + appdb_list_has_title(&list(&["CUSA00001"], false), "PPSA00002"), + Some(false) + ); + } +} diff --git a/engine/crates/ps5upload-core/tests/appdb_paging.rs b/engine/crates/ps5upload-core/tests/appdb_paging.rs new file mode 100644 index 00000000..c164b8dd --- /dev/null +++ b/engine/crates/ps5upload-core/tests/appdb_paging.rs @@ -0,0 +1,210 @@ +//! #473: `appdb_query` pages through app.db instead of trusting one capped reply. +//! +//! The helper used to answer at most 256 rows (and fewer when its buffer filled) with nothing +//! saying the list was cut. It now answers a window (`offset`/`limit` in, `next`/`more`/`truncated` +//! out); the engine follows it to the end, bounded, and says when the list is not whole. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use ps5upload_core::diagnostics::{appdb_query, APPDB_MAX_PAGES}; +use ps5upload_core::mgmt::{self, Method, MgmtTransport}; +use serde_json::{json, Value}; + +type Script = Box Result>> + Send + Sync>; + +struct Fake { + seen: Mutex>, + reply: Script, +} + +impl MgmtTransport for Fake { + fn call( + &self, + _addr: &str, + method: Method, + _label: &str, + body: &[u8], + _timeout: Duration, + ) -> Result>> { + assert_eq!(method.id, 56, "app.db_query"); + let req: Value = if body.is_empty() { + json!({}) + } else { + serde_json::from_slice(body).expect("the request is JSON") + }; + self.seen.lock().unwrap().push(req.clone()); + (self.reply)(&req) + } +} + +fn fake( + f: impl Fn(&Value) -> Result>> + Send + Sync + 'static, +) -> (Arc, mgmt::ScopedTransport) { + let t = Arc::new(Fake { + seen: Mutex::default(), + reply: Box::new(f), + }); + let g = mgmt::scoped_transport(t.clone()); + (t, g) +} + +fn title(i: usize) -> String { + format!("CUSA{i:05}") +} + +/// A helper that serves `rows` titles (plus a late-sorting ZZZZ99999) `page` at a time. +fn paged_helper(rows: usize, page: usize) -> impl Fn(&Value) -> Result>> { + move |req| { + let mut all: Vec = (0..rows).map(title).collect(); + all.push("ZZZZ99999".into()); + let off = req["offset"].as_u64().unwrap_or(0) as usize; + let end = (off + page).min(all.len()); + let apps: Vec = all[off.min(all.len())..end] + .iter() + .map(|t| json!({"title_id": t, "app_id": 1, "name": format!("Game {t}")})) + .collect(); + Ok(Some( + serde_json::to_vec(&json!({ + "apps": apps, + "source": "sqlite", + "offset": off, + "next": end, + "more": end < all.len(), + "truncated": false, + })) + .unwrap(), + )) + } +} + +const A: &str = "10.0.0.5"; + +#[test] +fn pages_until_the_helper_says_there_is_no_more() { + for rows in [255usize, 256, 257, 1000, 1500] { + let (t, _g) = fake(paged_helper(rows, 400)); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), rows + 1, "{rows} rows"); + assert!(!d.truncated, "{rows} rows"); + assert!(d.err.is_none()); + assert_eq!(d.apps.last().unwrap().title_id, "ZZZZ99999"); + assert_eq!(d.source.as_deref(), Some("sqlite")); + let seen = t.seen.lock().unwrap(); + assert_eq!(seen.len(), (rows + 1).div_ceil(400), "{rows} rows"); + for (i, req) in seen.iter().enumerate() { + assert_eq!(req["offset"], json!(i * 400), "page {i}"); + } + } +} + +#[test] +fn an_older_helper_answers_once_and_a_full_256_is_not_called_whole() { + // No next/more: the helper predates paging, so one reply is all there is. + let (t, _g) = fake(|_| { + let apps: Vec = (0..256) + .map(|i| json!({"title_id": title(i), "app_id": 1, "name": "n"})) + .collect(); + Ok(Some( + serde_json::to_vec(&json!({"apps": apps, "source": "sqlite"})).unwrap(), + )) + }); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), 256); + assert!(d.truncated, "256 rows from an older helper may be its cap"); + assert_eq!(t.seen.lock().unwrap().len(), 1); + drop(_g); + + let (_t, _g) = fake(|_| { + Ok(Some( + br#"{"apps":[{"title_id":"CUSA00001","app_id":1,"name":"n"}],"source":"scan"}"# + .to_vec(), + )) + }); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), 1); + assert!(!d.truncated); +} + +#[test] +fn a_truncated_page_marks_the_whole_list_truncated() { + let (_t, _g) = fake(|req| { + let off = req["offset"].as_u64().unwrap_or(0); + Ok(Some( + serde_json::to_vec(&if off == 0 { + json!({"apps":[{"title_id":"CUSA00001","app_id":1,"name":"a"}], + "offset":0,"next":1,"more":true,"truncated":false}) + } else { + json!({"apps":[],"offset":1,"next":1,"more":false,"truncated":true}) + }) + .unwrap(), + )) + }); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), 1); + assert!(d.truncated); +} + +#[test] +fn a_page_that_makes_no_progress_stops_the_loop() { + let (t, _g) = fake(|_| { + Ok(Some( + br#"{"apps":[],"offset":0,"next":0,"more":true,"truncated":false}"#.to_vec(), + )) + }); + let d = appdb_query(A).unwrap(); + assert!(d.truncated); + assert_eq!(t.seen.lock().unwrap().len(), 1); +} + +#[test] +fn the_page_count_is_bounded() { + // A helper that always has more: the engine gives up after APPDB_MAX_PAGES calls. + let (t, _g) = fake(|req| { + let off = req["offset"].as_u64().unwrap_or(0); + Ok(Some( + serde_json::to_vec(&json!({ + "apps":[{"title_id": title(off as usize), "app_id": 1, "name": "n"}], + "offset": off, "next": off + 1, "more": true, "truncated": false + })) + .unwrap(), + )) + }); + let d = appdb_query(A).unwrap(); + assert!(d.truncated); + assert_eq!(d.apps.len(), APPDB_MAX_PAGES); + assert_eq!(t.seen.lock().unwrap().len(), APPDB_MAX_PAGES); +} + +#[test] +fn errors_keep_their_meaning() { + // The helper's err envelope on page one is the answer, as before. + let (_t, _g) = fake(|_| Ok(Some(br#"{"err":"cannot read app.db","apps":[]}"#.to_vec()))); + let d = appdb_query(A).unwrap(); + assert_eq!(d.err.as_deref(), Some("cannot read app.db")); + assert!(d.apps.is_empty()); + drop(_g); + + // A transport failure on page one is still an error. + let (_t, _g) = fake(|_| Err(anyhow!("connection reset"))); + assert!(appdb_query(A).is_err()); + drop(_g); + + // A failure after page one keeps the rows read so far, flagged incomplete with the reason, + // so a caller never mistakes the partial list for the whole one. + let (_t, _g) = fake(|req| { + if req["offset"].as_u64().unwrap_or(0) == 0 { + Ok(Some( + br#"{"apps":[{"title_id":"CUSA00001","app_id":1,"name":"a"}],"offset":0,"next":1,"more":true,"truncated":false}"# + .to_vec(), + )) + } else { + Err(anyhow!("connection reset")) + } + }); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), 1); + assert!(d.truncated); + assert!(d.err.as_deref().unwrap_or("").contains("connection reset")); +} diff --git a/engine/crates/ps5upload-engine/src/lib.rs b/engine/crates/ps5upload-engine/src/lib.rs index 317f74bf..96f22bd8 100644 --- a/engine/crates/ps5upload-engine/src/lib.rs +++ b/engine/crates/ps5upload-engine/src/lib.rs @@ -4933,6 +4933,40 @@ struct InstalledAppsResponse { /// UI then can't reliably tag the "registered" group, so it shows /// everything under "installed" with a soft note rather than erroring. registered_unavailable: bool, + /// True when the list may be missing titles: a directory listing hit its page size, + /// or app.db could not be read in full (a failed query, or a helper that could not + /// deliver every row, #473). The titles shown are real; there may be more. + incomplete: bool, +} + +/// app.db, read at most once per request: a paged read is several calls (#473), and the +/// installed list asks it for names title by title. +struct AppDbOnce<'a> { + addr: &'a str, + got: Option>, +} + +impl<'a> AppDbOnce<'a> { + fn new(addr: &'a str) -> Self { + Self { addr, got: None } + } + + fn get(&mut self) -> &Result { + let addr = self.addr; + self.got + .get_or_insert_with(|| appdb_query(addr).map_err(|e| format!("{e:#}"))) + } + + /// app.db's display name for `title_id`, when it has one that is not the id itself. + fn name_of(&mut self, title_id: &str) -> Option { + let list = self.get().as_ref().ok()?; + list.apps + .iter() + .filter(|a| a.title_id == title_id) + .map(|a| a.name.trim()) + .find(|n| !n.is_empty() && *n != title_id) + .map(str::to_string) + } } /// PS5 title-ids are 4 uppercase letters + 5 digits (PPSA01234, NPXS39041, @@ -4950,7 +4984,7 @@ fn looks_like_title_id(name: &str) -> bool { /// `app.db` (raw file scan when sqlite is unavailable). Returns None on /// any failure (no metadata file — common for system apps — bad JSON, /// bad SFO, path denied); the caller falls back to the bare title_id. -fn appmeta_title_name(addr: &str, title_id: &str) -> Option { +fn appmeta_title_name(addr: &str, title_id: &str, appdb: &mut AppDbOnce<'_>) -> Option { // /user/appmeta/ first; then the app's own sce_sys, which is all a // title installed by a homebrew installer has on FW 13.60. for dir in [ @@ -4981,17 +5015,7 @@ fn appmeta_title_name(addr: &str, title_id: &str) -> Option { // This is the only way to resolve PS4 BC titles (CUSA IDs) whose // metadata lives solely in app.db — their appmeta dirs have no // param.sfo or param.json. - if let Ok(appdb) = appdb_query(addr) { - for app in &appdb.apps { - if app.title_id == title_id { - let name = app.name.trim(); - if !name.is_empty() && name != title_id { - return Some(name.to_string()); - } - } - } - } - None + appdb.name_of(title_id) } /// Titles app.db knows are installed that the /user/appmeta scan missed, @@ -5024,6 +5048,26 @@ fn appdb_installed_additions( out } +/// Titles we registered whose app folder exists but that neither the /user/appmeta scan +/// nor app.db produced (#473): a homebrew-installed title has no appmeta folder, and app.db +/// may not have answered for it (a failed or partial read). The registration and the folder +/// on disk are evidence enough. Sorted, so the list does not reshuffle between refreshes. +fn registered_only_additions<'a>( + registered: impl IntoIterator, + app_dirs: &std::collections::HashSet, + seen: &std::collections::HashSet, +) -> Vec { + let mut out: Vec = registered + .into_iter() + .map(str::trim) + .filter(|tid| looks_like_title_id(tid) && app_dirs.contains(*tid) && !seen.contains(*tid)) + .map(str::to_string) + .collect(); + out.sort(); + out.dedup(); + out +} + /// GET /api/ps5/apps/installed?addr=IP:MGMT_PORT async fn ps5_apps_installed( State(state): State, @@ -5050,6 +5094,7 @@ async fn ps5_apps_installed( } }; + let mut appdb = AppDbOnce::new(&addr); // Full installed set: every /user/appmeta// dir. let listing = list_dir( &addr, @@ -5074,7 +5119,7 @@ async fn ps5_apps_installed( // the id (older registrations stored id-as-name), try // param.json for a friendlier label. let name = if reg.title_name.trim().is_empty() || reg.title_name == tid { - appmeta_title_name(&addr, &tid).unwrap_or_else(|| tid.clone()) + appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone()) } else { reg.title_name.clone() }; @@ -5087,7 +5132,8 @@ async fn ps5_apps_installed( system, }); } else { - let name = appmeta_title_name(&addr, &tid).unwrap_or_else(|| tid.clone()); + let name = + appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone()); titles.push(InstalledApp { title_id: tid.clone(), title_name: name, @@ -5138,40 +5184,81 @@ async fn ps5_apps_installed( complete = complete && crate::console_snapshot::app_folders_whole(internal_listing, app_dirs.len()); if !app_dirs.is_empty() { - match appdb_query(&addr) { - Ok(appdb) => { - let seen: std::collections::HashSet = - titles.iter().map(|t| t.title_id.clone()).collect(); - let rows: Vec<(String, String)> = appdb - .apps - .into_iter() - .map(|a| (a.title_id, a.name)) - .collect(); - for (tid, db_name) in appdb_installed_additions(&rows, &app_dirs, &seen) { - let name = match db_name { - Some(n) => n, - None => { - appmeta_title_name(&addr, &tid).unwrap_or_else(|| tid.clone()) - } - }; - let (origin, image_backed, source) = match reg_map.get(&tid) { - Some(reg) => ("registered", reg.image_backed, reg.src.clone()), - None => ("pkg", false, String::new()), - }; - titles.push(InstalledApp { - title_id: tid, - title_name: name, - origin: origin.to_string(), - image_backed, - source, - system: false, - }); + let rows: Option> = match appdb.get() { + Ok(list) => { + // `err` with no rows: the helper could not read app.db at all. + if list.truncated || list.err.is_some() { + // Every row the helper delivered is used; the list just may not be whole. + crate::log_warn!( + "apps/installed: app.db read incomplete ({} rows{})", + list.apps.len(), + list.err + .as_deref() + .map(|e| format!(": {e}")) + .unwrap_or_default() + ); + complete = false; } + Some( + list.apps + .iter() + .map(|a| (a.title_id.clone(), a.name.clone())) + .collect(), + ) } Err(e) => { - crate::log_warn!("apps/installed: app.db query failed: {e:#}"); + crate::log_warn!("apps/installed: app.db query failed: {e}"); complete = false; + None } + }; + if let Some(rows) = rows { + let seen: std::collections::HashSet = + titles.iter().map(|t| t.title_id.clone()).collect(); + for (tid, db_name) in appdb_installed_additions(&rows, &app_dirs, &seen) { + let name = match db_name { + Some(n) => n, + None => appmeta_title_name(&addr, &tid, &mut appdb) + .unwrap_or_else(|| tid.clone()), + }; + let (origin, image_backed, source) = match reg_map.get(&tid) { + Some(reg) => ("registered", reg.image_backed, reg.src.clone()), + None => ("pkg", false, String::new()), + }; + titles.push(InstalledApp { + title_id: tid, + title_name: name, + origin: origin.to_string(), + image_backed, + source, + system: false, + }); + } + } + // Registered titles whose folder is there but that neither listing above + // produced (#473): never let a registration vanish because app.db did not + // answer for it. + let seen: std::collections::HashSet = + titles.iter().map(|t| t.title_id.clone()).collect(); + for tid in + registered_only_additions(reg_map.keys().map(String::as_str), &app_dirs, &seen) + { + let Some(reg) = reg_map.get(&tid) else { + continue; + }; + let name = if reg.title_name.trim().is_empty() || reg.title_name == tid { + appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone()) + } else { + reg.title_name.clone() + }; + titles.push(InstalledApp { + system: tid.starts_with("NPXS"), + title_id: tid, + title_name: name, + origin: "registered".to_string(), + image_backed: reg.image_backed, + source: reg.src.clone(), + }); } } // Kept for the game page. System apps are not games. @@ -5211,6 +5298,7 @@ async fn ps5_apps_installed( Ok(InstalledAppsResponse { titles, registered_unavailable, + incomplete: !complete, }) }) .await @@ -12537,7 +12625,7 @@ mod job_stage_tests { #[cfg(test)] mod appdb_installed_additions_tests { - use super::appdb_installed_additions; + use super::{appdb_installed_additions, registered_only_additions}; use std::collections::HashSet; fn set(v: &[&str]) -> HashSet { @@ -12568,6 +12656,42 @@ mod appdb_installed_additions_tests { ); } + /// #473: a title past the first 256 app.db rows is still added once the + /// rows arrive — the window, not the filter, was what hid it. + #[test] + fn a_late_sorting_title_past_row_256_is_kept() { + let mut rows: Vec<(String, String)> = (0..1000) + .map(|i| (format!("CUSA{i:05}"), format!("Game {i}"))) + .collect(); + rows.insert(0, ("AAAA00001".to_string(), "Early".to_string())); + rows.push(("ZZZZ99999".to_string(), "Late Title".to_string())); + let got = appdb_installed_additions(&rows, &set(&["ZZZZ99999"]), &HashSet::new()); + assert_eq!( + got, + vec![("ZZZZ99999".to_string(), Some("Late Title".to_string()))] + ); + } + + /// #473: a registered title with its app folder but no appmeta and no + /// app.db row (a failed or partial app.db read) is still listed. + #[test] + fn registered_titles_with_a_folder_are_reconciled() { + let got = registered_only_additions( + [ + "PPSA00002", + "ZZZZ99999", + "CUSA00001", + "PPSA00003", + "not-an-id", + ], + &set(&["ZZZZ99999", "CUSA00001", "PPSA00002", "not-an-id"]), + // already listed by the appmeta scan or app.db + &set(&["CUSA00001"]), + ); + // PPSA00003 has no folder: a stale registration is not an installed title. + assert_eq!(got, vec!["PPSA00002".to_string(), "ZZZZ99999".to_string()]); + } + #[test] fn skips_titles_the_appmeta_scan_already_found() { let rows = vec![("PLDM00001".to_string(), "Payload Manager".to_string())]; diff --git a/payload/include/content_db.h b/payload/include/content_db.h index 67e840e6..b0c3515a 100644 --- a/payload/include/content_db.h +++ b/payload/include/content_db.h @@ -78,18 +78,37 @@ int content_db_select_text(const char *db_path, const char *sql, int max_rows, const char *const *vals)); /* - * Installed titles from app.db: title id, app id and display name. + * Installed titles from app.db, one page at a time (#473): title id, app + * id and display name for rows [offset, offset + limit) in title order. * * Emits {"apps":[{"title_id":"..","app_id":N,"name":".."}, ...], - * "source":"sqlite"|"scan"} + * "source":"sqlite"|"sqlite-immutable"|"scan", + * "offset":O,"next":N,"more":bool,"truncated":bool} + * + * `next` is the row the following page starts at; `more` is true while + * rows remain after it. A page that fills `buf` before `limit` rows stops + * early with `more` set, so a small buffer pages rather than silently + * dropping the rest. `truncated` is true only when rows exist that no + * page can deliver: one row larger than `buf`, or the byte scanner (the + * fallback for a database that will not open) hitting its own bounds. + * + * `limit` 0 (or above CONTENT_DB_APPS_PAGE_MAX) means + * CONTENT_DB_APPS_PAGE_MAX. An older engine sends no window at all and + * reads page one; the extra fields are ignored by its decoder. * * `source` is reported rather than hidden: a caller that sees "scan" * knows the SQL open failed and the row set came from the byte-level * reader, which recovers ids reliably but names only sometimes. * - * Never fails outright — falls back to appdb_scan.h. Returns 0 on - * success, non-zero only if `buf` is too small to hold even an error. + * Never fails outright — falls back to appdb_scan.h, and an unreadable + * database is {"err":"..","apps":[]}. Returns 0 on success, non-zero only + * if `buf` is too small to hold even an error. */ +#define CONTENT_DB_APPS_PAGE_MAX 512 +int content_db_apps_page_json(unsigned offset, unsigned limit, char *buf, + size_t cap, size_t *written); + +/* Page one: content_db_apps_page_json(0, 0, ...). */ int content_db_apps_json(char *buf, size_t cap, size_t *written); /* diff --git a/payload/src/content_db.c b/payload/src/content_db.c index 8e3ad03b..ba7b6ed6 100644 --- a/payload/src/content_db.c +++ b/payload/src/content_db.c @@ -268,15 +268,45 @@ static void cdb_build_app_sql(const cdb_app_schema_t *sc, char *sql, sc->table, sc->col_title, sc->col_title); } -/* Row-level app list: SQL first, byte scan if the database will not open. - * content_db_apps_json() and activity.c's "recently played" both build on - * this so there is one schema-discovery implementation, not two. */ -int content_db_apps(content_db_app_t *out, int max, const char **source) { +/* The byte scanner's bounds. It reads at most CDB_SCAN_RAW_MAX of the file + * and recovers at most CDB_SCAN_ROWS_MAX titles; a database past either is + * reported as truncated, never as a complete list. */ +#define CDB_SCAN_RAW_MAX (1024u * 1024u) +#define CDB_SCAN_ROWS_MAX 4096 + +static void cdb_fill_row(content_db_app_t *r, const char *tid, + const char *name, int app_id) { + memset(r, 0, sizeof(*r)); + snprintf(r->title_id, sizeof(r->title_id), "%s", tid); + snprintf(r->name, sizeof(r->name), "%s", name ? name : ""); + r->app_id = app_id; +} + +/* + * Row-level app list, windowed: rows [offset, offset + max) of the title + * table, in title order, go to `out`. SQL first, byte scan if the database + * will not open. content_db_apps(), the paged JSON and activity.c's + * "recently played" all build on this, so there is one schema-discovery + * implementation, not two. + * + * `total` (optional) receives how many rows the table holds -- with the + * scanner, a lower bound that is at least offset + max + 1 when more rows + * follow the window. `capped` (optional) is set when rows exist that the + * reader could not reach (the scanner's bounds). + * + * Returns the number of rows written to `out`, or -1 if nothing could be + * read at all. + */ +static int cdb_apps_window(content_db_app_t *out, unsigned offset, int max, + unsigned *total, int *capped, const char **source) { + if (total) *total = 0; + if (capped) *capped = 0; if (!out || max <= 0) return -1; if (source) *source = "scan"; sqlite3 *db = NULL; const char *how = "sqlite"; + unsigned rows_seen = 0; int count = 0; if (cdb_open_ro(CONTENT_DB_APP, &db, &how) == SQLITE_OK && db) { @@ -286,23 +316,23 @@ int content_db_apps(content_db_app_t *out, int max, const char **source) { /* First candidate that actually yields rows wins: a console with * several users has several of these tables, and the ones * belonging to other users are empty. */ - for (int ci = 0; ci < ncand && count == 0; ci++) { + for (int ci = 0; ci < ncand && rows_seen == 0; ci++) { char sql[512]; cdb_build_app_sql(&cands[ci], sql, sizeof(sql)); sqlite3_stmt *st = NULL; if (sqlite3_prepare_v2(db, sql, -1, &st, NULL) != SQLITE_OK) continue; - while (count < max && sqlite3_step(st) == SQLITE_ROW) { + /* Every row is counted (so the caller knows whether more + * follow); only the window is copied. */ + while (sqlite3_step(st) == SQLITE_ROW) { const unsigned char *tid = sqlite3_column_text(st, 0); if (!tid) continue; + unsigned idx = rows_seen++; + if (idx < offset || count >= max) continue; const unsigned char *nm = sqlite3_column_text(st, 2); - memset(&out[count], 0, sizeof(out[count])); - snprintf(out[count].title_id, sizeof(out[count].title_id), - "%s", (const char *)tid); - snprintf(out[count].name, sizeof(out[count].name), "%s", - nm ? (const char *)nm : ""); - out[count].app_id = sqlite3_column_int(st, 1); + cdb_fill_row(&out[count], (const char *)tid, + (const char *)nm, sqlite3_column_int(st, 1)); count++; } sqlite3_finalize(st); @@ -314,49 +344,65 @@ int content_db_apps(content_db_app_t *out, int max, const char **source) { * console has no games -- and install verification reads an empty list * as a failed install. Fall through to the scan rather than assert a * confident zero. */ - if (count > 0) { + if (rows_seen > 0) { if (source) *source = how; + if (total) *total = rows_seen; return count; } int fd = open(CONTENT_DB_APP, O_RDONLY); if (fd < 0) return -1; - const size_t raw_cap = 1024 * 1024; - unsigned char *raw = (unsigned char *)malloc(raw_cap); + unsigned char *raw = (unsigned char *)malloc(CDB_SCAN_RAW_MAX); if (!raw) { close(fd); return -1; } - size_t total = 0; - while (total < raw_cap) { - ssize_t r = read(fd, raw + total, raw_cap - total); + size_t got = 0; + while (got < CDB_SCAN_RAW_MAX) { + ssize_t r = read(fd, raw + got, CDB_SCAN_RAW_MAX - got); if (r <= 0) break; - total += (size_t)r; + got += (size_t)r; + } + /* A file larger than the read window has rows the scan never sees. */ + int raw_capped = 0; + if (got == CDB_SCAN_RAW_MAX) { + unsigned char probe; + raw_capped = read(fd, &probe, 1) > 0; } close(fd); + /* One past the window, so "more rows follow" is known without + * holding every row. */ + unsigned long want = (unsigned long)offset + (unsigned long)max + 1ul; + int scan_max = want > CDB_SCAN_ROWS_MAX ? CDB_SCAN_ROWS_MAX : (int)want; appdb_entry_t *entries = - (appdb_entry_t *)malloc(sizeof(appdb_entry_t) * (size_t)max); + (appdb_entry_t *)malloc(sizeof(appdb_entry_t) * (size_t)scan_max); if (!entries) { free(raw); return -1; } - int scanned = appdb_scan_entries_ex(raw, total, entries, max, 1); + int scanned = appdb_scan_entries_ex(raw, got, entries, scan_max, 1); free(raw); if (scanned < 0) { free(entries); return -1; } - for (int i = 0; i < scanned; i++) { - memset(&out[i], 0, sizeof(out[i])); - snprintf(out[i].title_id, sizeof(out[i].title_id), "%s", - entries[i].title_id); - snprintf(out[i].name, sizeof(out[i].name), "%s", entries[i].name); - out[i].app_id = 0; + for (int i = (int)(offset < (unsigned)scanned ? offset : (unsigned)scanned); + i < scanned && count < max; i++) { + cdb_fill_row(&out[count], entries[i].title_id, entries[i].name, 0); + count++; } free(entries); + if (capped) + *capped = raw_capped || (scanned == CDB_SCAN_ROWS_MAX && + scan_max == CDB_SCAN_ROWS_MAX); + if (total) *total = (unsigned)scanned; if (source) *source = "scan"; - return scanned; + return count; +} + +int content_db_apps(content_db_app_t *out, int max, const char **source) { + return cdb_apps_window(out, 0, max, NULL, NULL, source); } int content_db_select_text(const char *db_path, const char *sql, int max_rows, @@ -395,16 +441,26 @@ int content_db_select_text(const char *db_path, const char *sql, int max_rows, return rows; } -int content_db_apps_json(char *buf, size_t cap, size_t *written) { +/* The longest tail a page can end with: + * ],"source":"sqlite-immutable","offset":4294967295,"next":4294967295, + * "more":false,"truncated":false} -- about 110 bytes. */ +#define CDB_PAGE_TAIL_RESERVE 160 + +int content_db_apps_page_json(unsigned offset, unsigned limit, char *buf, + size_t cap, size_t *written) { if (!buf || cap < 64) return -1; + if (limit == 0 || limit > CONTENT_DB_APPS_PAGE_MAX) + limit = CONTENT_DB_APPS_PAGE_MAX; - const int max_rows = 256; content_db_app_t *rows = - (content_db_app_t *)malloc(sizeof(content_db_app_t) * (size_t)max_rows); + (content_db_app_t *)malloc(sizeof(content_db_app_t) * (size_t)limit); if (!rows) return cdb_emit_apps_err(buf, cap, written, "out of memory"); const char *source = "scan"; - int count = content_db_apps(rows, max_rows, &source); + unsigned total = 0; + int capped = 0; + int count = cdb_apps_window(rows, offset, (int)limit, &total, &capped, + &source); if (count < 0) { free(rows); return cdb_emit_apps_err(buf, cap, written, "cannot read app.db"); @@ -424,20 +480,40 @@ int content_db_apps_json(char *buf, size_t cap, size_t *written) { "%s{\"title_id\":\"%s\",\"app_id\":%d," "\"name\":\"%s\"}", emitted ? "," : "", tid, rows[i].app_id, name); - if (more < 0 || (size_t)(n + more) + 40 >= cap) break; + /* Full: stop here and let `next` say where the following page + * starts. This used to close the JSON as if it were the whole + * list (#473). */ + if (more < 0 || (size_t)(n + more) + CDB_PAGE_TAIL_RESERVE >= cap) + break; n += more; emitted++; } free(rows); - int end2 = snprintf(buf + n, cap - (size_t)n, "],\"source\":\"%s\"}", - source); + unsigned next = offset + (unsigned)emitted; + /* Rows remain after this page: the table holds more than `next`. */ + int more_rows = total > next; + /* A row that cannot fit even on an empty page can never be delivered: + * say so and stop, rather than send the caller round in a loop. */ + int stuck = more_rows && emitted == 0 && count > 0; + int truncated = capped || stuck; + if (stuck) more_rows = 0; + + int end2 = snprintf(buf + n, cap - (size_t)n, + "],\"source\":\"%s\",\"offset\":%u,\"next\":%u," + "\"more\":%s,\"truncated\":%s}", + source, offset, next, more_rows ? "true" : "false", + truncated ? "true" : "false"); if (end2 < 0 || (size_t)(n + end2) >= cap) return -1; n += end2; if (written) *written = (size_t)n; return 0; } +int content_db_apps_json(char *buf, size_t cap, size_t *written) { + return content_db_apps_page_json(0, 0, buf, cap, written); +} + /* ── appinfo.db ──────────────────────────────────────────────────────── */ int content_db_appinfo_json(const char *title_id, const char *keys, diff --git a/payload/src/mgmt_table.def b/payload/src/mgmt_table.def index c21166fc..2b7e9766 100644 --- a/payload/src/mgmt_table.def +++ b/payload/src/mgmt_table.def @@ -57,7 +57,7 @@ MGMT_H0(AVA1_METHOD_APP_LAUNCH_BROWSER, MGMT_FRAME_APP_LAUNCH_BROWSER, MGMT_FRAM MGMT_H1(AVA1_METHOD_APP_LIFECYCLE, MGMT_FRAME_APP_LIFECYCLE, MGMT_FRAME_APP_LIFECYCLE_ACK, MGMT_SONY, handle_app_lifecycle, mgmt_call_text) MGMT_H1(AVA1_METHOD_APP_INFO_QUERY, MGMT_FRAME_APPINFO_QUERY, MGMT_FRAME_APPINFO_QUERY_ACK, 0, handle_appinfo_query, mgmt_call_text) MGMT_H1(AVA1_METHOD_APP_INFO_SET, MGMT_FRAME_APPINFO_SET, MGMT_FRAME_APPINFO_SET_ACK, MGMT_SONY, handle_appinfo_set, mgmt_call_text) -MGMT_H0(AVA1_METHOD_APP_DB_QUERY, MGMT_FRAME_APPDB_QUERY, MGMT_FRAME_APPDB_QUERY_ACK, 0, handle_appdb_query, mgmt_call_text) +MGMT_H1(AVA1_METHOD_APP_DB_QUERY, MGMT_FRAME_APPDB_QUERY, MGMT_FRAME_APPDB_QUERY_ACK, 0, handle_appdb_query, mgmt_call_text) MGMT_H0(AVA1_METHOD_PROC_FOCUS, MGMT_FRAME_FOCUS_PROBE, MGMT_FRAME_FOCUS_PROBE_ACK, 0, handle_focus_probe, mgmt_call_text) MGMT_H1(AVA1_METHOD_PROC_LIST, MGMT_FRAME_PROC_LIST, MGMT_FRAME_PROC_LIST_ACK, 0, handle_proc_list, mgmt_call_text) MGMT_H1(AVA1_METHOD_PROC_KILL, MGMT_FRAME_PROCESS_KILL, MGMT_FRAME_PROCESS_KILL_ACK, 0, handle_process_kill, mgmt_call_text) diff --git a/payload/src/runtime.c b/payload/src/runtime.c index 285c0704..5f5709bb 100644 --- a/payload/src/runtime.c +++ b/payload/src/runtime.c @@ -6991,10 +6991,32 @@ static int handle_shell_exec(runtime_state_t *state, const char *body, * The scan survives inside content_db_apps_json as the fallback for a * database that will not open at all (locked by the shell, or damaged), * which is a genuine runtime condition rather than a permanent one. + * + * Paged (#473): the request may carry {"offset":N,"limit":M}; the reply + * says where the next page starts ("next") and whether one follows + * ("more"), see content_db_apps_page_json. An empty request -- what an + * older engine sends -- is page one. */ -static int handle_appdb_query(runtime_state_t *state) { +static int handle_appdb_query(runtime_state_t *state, const char *body, + uint64_t body_len) { if (!state) return -1; + uint64_t offset = 0, limit = 0; + if (body && body_len > 0) { + /* mgmt_json_u64 reads a C string; the body is not terminated. */ + char req[256]; + size_t n = body_len < sizeof(req) - 1 ? (size_t)body_len + : sizeof(req) - 1; + memcpy(req, body, n); + req[n] = '\0'; + if (mgmt_json_u64(req, "offset", &offset) < 0 || + mgmt_json_u64(req, "limit", &limit) < 0 || offset > UINT32_MAX || + limit > UINT32_MAX) { + const char *err = "{\"err\":\"bad_window\",\"apps\":[]}"; + return mgmt_reply(MGMT_FRAME_APPDB_QUERY_ACK, err, strlen(err)); + } + } + const size_t cap = 64 * 1024; char *resp = (char *)malloc(cap); if (!resp) { @@ -7004,7 +7026,9 @@ static int handle_appdb_query(runtime_state_t *state) { size_t written = 0; int rc; - if (content_db_apps_json(resp, cap, &written) != 0 || written == 0) { + if (content_db_apps_page_json((unsigned)offset, (unsigned)limit, resp, cap, + &written) != 0 || + written == 0) { const char *err = "{\"err\":\"appdb_unreadable\",\"apps\":[]}"; rc = mgmt_reply(MGMT_FRAME_APPDB_QUERY_ACK, err, strlen(err)); } else { diff --git a/payload/tests/content_db_selftest.c b/payload/tests/content_db_selftest.c index 1a1cbbb7..0ec3649d 100644 --- a/payload/tests/content_db_selftest.c +++ b/payload/tests/content_db_selftest.c @@ -129,6 +129,115 @@ static int contains(const char *hay, const char *needle) { return strstr(hay, needle) != NULL; } + +/* ── app.db pagination fixtures (#473) ──────────────────────────────── + * + * A big app table: `n` rows CUSA00000.. in title order, plus (when + * `late` is set) one late-sorting row ZZZZ99999 -- the shape of a title + * that used to fall off the end of the fixed 256-row window. */ +static void make_big_app_db(const char *path, int n, int late, + int long_names) { + unlink(path); + sqlite3 *db = NULL; + assert(sqlite3_open(path, &db) == SQLITE_OK); + exec_or_die(db, + "CREATE TABLE tbl_info_1000000 (" + " titleId TEXT, appId INTEGER, titleName TEXT);"); + exec_or_die(db, "BEGIN"); + sqlite3_stmt *st = NULL; + assert(sqlite3_prepare_v2(db, + "INSERT INTO tbl_info_1000000 VALUES (?,?,?)", + -1, &st, NULL) == SQLITE_OK); + for (int i = 0; i < n; i++) { + char tid[16], name[300]; + snprintf(tid, sizeof tid, "CUSA%05d", i); + if (long_names) { + /* 200+ characters with quotes, backslashes and a control + * byte: every one of them grows when escaped. */ + int o = snprintf(name, sizeof name, "Game \"%d\" \\ \t", i); + while (o < 240) name[o++] = 'x'; + name[o] = '\0'; + } else { + snprintf(name, sizeof name, "Game %d", i); + } + sqlite3_bind_text(st, 1, tid, -1, SQLITE_TRANSIENT); + sqlite3_bind_int(st, 2, 1000 + i); + sqlite3_bind_text(st, 3, name, -1, SQLITE_TRANSIENT); + assert(sqlite3_step(st) == SQLITE_DONE); + sqlite3_reset(st); + } + sqlite3_finalize(st); + if (late) + exec_or_die(db, "INSERT INTO tbl_info_1000000 VALUES" + " ('ZZZZ99999', 9, 'Late Title')"); + exec_or_die(db, "COMMIT"); + sqlite3_close(db); +} + +/* The unsigned number after "key": in `json`, or -1. */ +static long json_num(const char *json, const char *key) { + char needle[40]; + snprintf(needle, sizeof needle, "\"%s\":", key); + const char *p = strstr(json, needle); + if (!p) return -1; + return strtol(p + strlen(needle), NULL, 10); +} + +static int json_true(const char *json, const char *key) { + char needle[40]; + snprintf(needle, sizeof needle, "\"%s\":true", key); + return strstr(json, needle) != NULL; +} + +/* Count the title ids in one page and check each is the next expected + * one, so a skipped or repeated row fails, not just a short count. */ +static int page_ids(const char *json, int *seen_upto, int *saw_late) { + int n = 0; + const char *p = json; + while ((p = strstr(p, "\"title_id\":\"")) != NULL) { + p += strlen("\"title_id\":\""); + if (strncmp(p, "ZZZZ99999", 9) == 0) { + *saw_late = 1; + } else { + char want[16]; + snprintf(want, sizeof want, "CUSA%05d", *seen_upto); + assert(strncmp(p, want, 9) == 0); + (*seen_upto)++; + } + n++; + } + return n; +} + +/* Page through the whole table the way the engine does; returns how many + * rows arrived. Fails if a page claims more but makes no progress. */ +static int page_all(char *buf, size_t cap, unsigned limit, int *saw_late, + int *pages, int *truncated) { + int seen_upto = 0, total = 0; + unsigned offset = 0; + *saw_late = 0; + *pages = 0; + *truncated = 0; + for (;;) { + size_t written = 0; + assert(content_db_apps_page_json(offset, limit, buf, cap, &written) == + 0); + assert(written > 0 && written < cap); + assert(buf[written - 1] == '}'); + (*pages)++; + int got = page_ids(buf, &seen_upto, saw_late); + total += got; + assert(json_num(buf, "offset") == (long)offset); + assert(json_num(buf, "next") == (long)(offset + (unsigned)got)); + if (json_true(buf, "truncated")) *truncated = 1; + if (!json_true(buf, "more")) break; + assert(got > 0); + offset += (unsigned)got; + assert(*pages < 10000); + } + return total; +} + int main(void) { char buf[64 * 1024]; size_t written = 0; @@ -278,6 +387,134 @@ int main(void) { printf("✓ non-SELECT refused; table intact\n"); } + + /* ── app.db pagination (#473) ───────────────────────────────────── */ + { + static char pbuf[64 * 1024]; + int late, pages, trunc; + const int sizes[] = {255, 256, 257, 1000, 1500}; + for (size_t i = 0; i < sizeof sizes / sizeof *sizes; i++) { + make_big_app_db(CONTENT_DB_APP, sizes[i], 1, 0); + int got = page_all(pbuf, sizeof pbuf, 0, &late, &pages, &trunc); + assert(got == sizes[i] + 1); + assert(late); + assert(!trunc); + printf("✓ %d rows + a late-sorting title: all %d arrive in %d page(s)\n", + sizes[i], got, pages); + } + + /* The no-argument call (what an older engine sends) is page one: + * never a silent cap that looks complete. */ + make_big_app_db(CONTENT_DB_APP, 1000, 1, 0); + assert(content_db_apps_json(pbuf, sizeof pbuf, &written) == 0); + assert(json_num(pbuf, "offset") == 0); + assert(json_true(pbuf, "more")); + assert(!json_true(pbuf, "truncated")); + assert(json_num(pbuf, "next") > 256); + printf("✓ the unpaged call says more rows follow (next=%ld)\n", + json_num(pbuf, "next")); + + /* An earlier-sorting row added later shifts the window but does + * not push the late title out. */ + { + sqlite3 *db = NULL; + assert(sqlite3_open(CONTENT_DB_APP, &db) == SQLITE_OK); + exec_or_die(db, "INSERT INTO tbl_info_1000000 VALUES" + " ('AAAA00001', 1, 'Early Title')"); + sqlite3_close(db); + int seen = 0, saw_late = 0, saw_early = 0; + unsigned off = 0; + for (;;) { + assert(content_db_apps_page_json(off, 0, pbuf, sizeof pbuf, + &written) == 0); + const char *p = pbuf; + int got = 0; + while ((p = strstr(p, "\"title_id\":\"")) != NULL) { + p += 12; + if (!strncmp(p, "ZZZZ99999", 9)) saw_late = 1; + if (!strncmp(p, "AAAA00001", 9)) saw_early = 1; + got++; + } + seen += got; + if (!json_true(pbuf, "more")) break; + off += (unsigned)got; + } + assert(seen == 1002 && saw_late && saw_early); + printf("✓ an earlier-sorting row does not push the late title out\n"); + } + + /* An explicit limit is honoured. */ + make_big_app_db(CONTENT_DB_APP, 300, 0, 0); + assert(content_db_apps_page_json(100, 50, pbuf, sizeof pbuf, + &written) == 0); + assert(contains(pbuf, "\"title_id\":\"CUSA00100\"")); + assert(contains(pbuf, "\"title_id\":\"CUSA00149\"")); + assert(!contains(pbuf, "\"title_id\":\"CUSA00150\"")); + assert(json_num(pbuf, "next") == 150); + assert(json_true(pbuf, "more")); + assert(content_db_apps_page_json(300, 50, pbuf, sizeof pbuf, + &written) == 0); + assert(contains(pbuf, "\"apps\":[]")); + assert(!json_true(pbuf, "more")); + assert(json_num(pbuf, "next") == 300); + printf("✓ offset/limit select a window; past the end is empty and final\n"); + + /* A small buffer stops early and says where -- a continuation, + * not a complete-looking list. */ + make_big_app_db(CONTENT_DB_APP, 300, 1, 1); + { + static char small[2048]; + assert(content_db_apps_page_json(0, 0, small, sizeof small, + &written) == 0); + long next = json_num(small, "next"); + assert(next > 0 && next < 300); + assert(json_true(small, "more")); + assert(!json_true(small, "truncated")); + assert(contains(small, "Game \\\"0\\\" \\\\ \\t")); + int got = page_all(small, sizeof small, 0, &late, &pages, &trunc); + assert(got == 301 && late && !trunc); + printf("✓ a 2 KiB buffer pages long escaped names: 301 rows in %d pages\n", + pages); + } + + /* A buffer that cannot hold even one row reports truncation and + * stops, rather than looping or claiming an empty success. */ + { + static char tiny[160]; + assert(content_db_apps_page_json(0, 0, tiny, sizeof tiny, + &written) == 0); + assert(contains(tiny, "\"apps\":[]")); + assert(json_true(tiny, "truncated")); + assert(!json_true(tiny, "more")); + printf("✓ a buffer too small for one row is reported as truncated\n"); + } + + /* The row API is no longer clipped at 256 either. */ + make_big_app_db(CONTENT_DB_APP, 600, 0, 0); + { + content_db_app_t *big = + (content_db_app_t *)malloc(sizeof(content_db_app_t) * 700); + assert(big); + assert(content_db_apps(big, 700, NULL) == 600); + assert(strcmp(big[599].title_id, "CUSA00599") == 0); + free(big); + } + printf("✓ row API returns all 600 rows\n"); + + /* A query error is still the err envelope, and says it is final. */ + { + FILE *f = fopen(CONTENT_DB_APP, "wb"); + assert(f); + fputs("not a database", f); + fclose(f); + assert(content_db_apps_page_json(0, 0, pbuf, sizeof pbuf, + &written) == 0); + assert(contains(pbuf, "\"err\":") && contains(pbuf, "\"apps\":[]")); + assert(!json_true(pbuf, "more")); + printf("✓ unreadable app.db on a paged call still answers err+apps\n"); + } + } + /* ── home-screen icon hide / unhide (#407) ─────────────────────── */ { make_icon_db(CONTENT_DB_APP); diff --git a/protocol/ava1/MGMT_METHODS.md b/protocol/ava1/MGMT_METHODS.md index 12887447..0c1afcf4 100644 --- a/protocol/ava1/MGMT_METHODS.md +++ b/protocol/ava1/MGMT_METHODS.md @@ -59,7 +59,7 @@ ones (`ava1-ctest/tests/mgmt*.rs`, `ps5upload-ava1/tests/mgmt*.rs`). Methods 119 | ProcModules 114 | 61 `proc.modules` | `MgmtText` | `src/runtime.c` `handle_proc_modules` | `ps5upload-core/src/diagnostics.rs` | | | ShellExec 116 | 87 `shell.exec` | `MgmtText` | `src/runtime.c` `handle_shell_exec` | `ps5upload-core/src/diagnostics.rs` | | | Crc32File 118 | 20 `job.run (op CRC32)` | `JobRun` -> `Status` (ext `state`, `result`, `code`) | `src/fs_jobs.c` `op_crc32` | `ps5upload-core/src/diagnostics.rs` | `ava1-ctest/tests/job_run.rs` `crc32_of_a_large_file_is_a_job_that_can_be_cancelled`; engine `mgmt_job.rs` `recursive_chmod_hash_crc32_fsck_and_backup_run_as_jobs_with_their_reply_shapes` | -| AppDbQuery 120 | 56 `app.db_query` | `MgmtText` | `src/runtime.c` `handle_appdb_query` | `ps5upload-core/src/diagnostics.rs` | | +| AppDbQuery 120 | 56 `app.db_query` | `MgmtText`; request `{"offset":N,"limit":M}` (both optional, empty = page one), reply adds `offset`, `next`, `more`, `truncated` (#473) | `src/runtime.c` `handle_appdb_query` | `ps5upload-core/src/diagnostics.rs` (pages until `more` is false) | | | NetSpeedTest 122 | 11 `net.speedtest` | `MgmtText` | `src/runtime.c` `handle_net_speed_test` | `ps5upload-core/src/diagnostics.rs` | | | PkgDirectMount 124 | 42 `fs.mount_pkg` | `MgmtText` | `src/runtime.c` `handle_pkg_direct_mount` | `ps5upload-core/src/diagnostics.rs` | host-tested | | UfsFsck 126 | 20 `job.run (op FSCK)` | `JobRun` -> `Status` (ext `state`, `result`, `code`) | `src/runtime.c` `handle_ufs_fsck` | `ps5upload-core/src/diagnostics.rs` | `ava1-ctest/tests/job_run.rs` `fsck_keeps_its_ok_false_body_as_the_answer_and_a_frame_error_is_a_failure`; engine `mgmt_job.rs` (same shapes test) | From 61178d75263cedeb20cd0bcfaf7c4719decc2df9 Mon Sep 17 00:00:00 2001 From: phantomptr Date: Sat, 17 Oct 2026 11:07:03 -0700 Subject: [PATCH 2/3] FAQ: use PS5Upload from your phone or another computer Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RxCGb8Asp1A1AQYiv1kbgS --- FAQ.md | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/FAQ.md b/FAQ.md index 613cb001..10f182ab 100644 --- a/FAQ.md +++ b/FAQ.md @@ -163,7 +163,9 @@ Set `PS5UPLOAD_BANDWIDTH_MBPS` (megabytes per second) before starting the app or zero means no cap. The pre-6.0 name of this variable still works for one release. **Q: Can I run the engine on a different machine (self-hosted)?** -Yes. Run it on a server or NAS and point the app at it in **Settings → Engine URL**. Images: +Yes. Run it on a server or NAS and point the app at it in **Settings → Engine URL**. For a +step-by-step guide that ends with the app open on a phone, see "Use PS5Upload from your phone or +another computer" below. Images: `ghcr.io/phantomptr/ps5upload-engine` (engine only) and `ghcr.io/phantomptr/ps5upload-engine-webui` (full web UI), tagged `:latest` or `:`. @@ -197,7 +199,8 @@ in the compose file to the folder's owner (`ls -ln` shows the numbers), or run **Q: Can I use ps5upload from a web browser (no desktop app)?** Yes. The `webui` image (above) serves the full app over HTTP; open `http://:19113` from an -allowed IP. Same security rules: no password, trusted LAN only. +allowed IP. Same security rules: no password, trusted LAN only. The next question walks through +it from start to finish. - **Upload works**, but the file picker browses the **engine's** disk, not your browser machine's. Mount your games into the container (`-v /host/games:/pkgs:ro`) and set @@ -209,6 +212,87 @@ allowed IP. Same security rules: no password, trusted LAN only. report. Everything that works on the PS5 itself is the same as on desktop. - The docker image can be built using `PS5UPLOAD_BASE_URL` to change the root path of the app. +**Q: Use PS5Upload from your phone or another computer** +Run PS5Upload on one machine, for example a Mac tucked away next to the router, and use the whole +app from a browser anywhere on your home network, including Safari on an iPhone or iPad. This +uses the web UI Docker image, not the desktop app. + +**Why not the desktop app?** Its engine only takes orders from its own window: a request from any +other device is refused, and it has no web version of the app to give a browser. The Docker image +`ghcr.io/phantomptr/ps5upload-engine-webui` is the same engine with the full app built in. + +**1. Install Docker.** On a Mac, install [Docker Desktop](https://www.docker.com/products/docker-desktop/) +and start it. On Linux, a NAS or a mini PC, any recent Docker with Compose works. + +**2. Write the compose file.** Make a folder (for example `~/ps5upload`), and in it a file +named `compose.yaml`. On **Linux**, copy [`engine/compose.yaml`](engine/compose.yaml) as it is and +edit the three values marked `EDIT`. On a **Mac or Windows PC**, Docker Desktop has no host +networking, so publish the engine's port instead and tell the engine the computer's own address: + +```yaml +services: + ps5upload: + image: ghcr.io/phantomptr/ps5upload-engine-webui:latest + container_name: ps5upload + restart: unless-stopped + ports: + - "19113:19113" + environment: + PS5_ADDR: "192.168.1.50" # your PS5's IP + PS5UPLOAD_PKG_HOST_IP: "192.168.1.10" # this computer's IP (step 3) + PS5UPLOAD_ALLOW_IP: "192.168.1.0/24" # who may use it (step 5) + PS5UPLOAD_BROWSE_ROOTS: "/pkgs" + volumes: + - ps5upload-data:/data + - /Users/you/PS5:/pkgs:ro # your packages and game folders + +volumes: + ps5upload-data: +``` + +- Keep port **19113** the same on both sides (`"19113:19113"`). The PS5 downloads packages from + that port during Stream & install, and the engine tells it to use 19113. +- `PS5UPLOAD_PKG_HOST_IP` is the address the PS5 is told to download from. Without it, a stream + install hands the PS5 an address inside Docker that it cannot reach. +- `/pkgs` is what the web UI's file picker shows: change `/Users/you/PS5` to the folder that holds + your `.pkg` files and games. The picker browses this computer, not your phone. To send a package + that is on the phone itself, use **Install Package → From this device**. +- `ps5upload-data` holds the engine's pairing key and saved settings. Don't delete it, or the PS5 + sees a new device and you pair again. +- Quit the PS5Upload desktop app on this computer first: it also uses port 19113. + +**3. Find the computer's IP address.** On a Mac: **System Settings → Wi-Fi** (or **Network → +Ethernet**) → **Details** → **TCP/IP**, or run `ipconfig getifaddr en0` in Terminal. Put it in +`PS5UPLOAD_PKG_HOST_IP`. Give the computer and the PS5 fixed addresses in your router if you can, +so they don't change. + +**4. Start it.** In that folder run `docker compose up -d`. `docker compose pull` followed by +`docker compose up -d` updates it later. + +**5. Open it on your phone.** On the same Wi-Fi, open `http://:19113/` in Safari +(or any browser), for example `http://192.168.1.10:19113/`. On an iPhone, tap **Share → Add to +Home Screen** to get an app icon. If the page says **"loopback only: this engine refused"** +followed by an address, the engine saw your phone as that address. Add it to `PS5UPLOAD_ALLOW_IP` +(comma-separated, for example `"192.168.1.0/24,192.168.65.1"`) and run `docker compose up -d` +again. Docker Desktop often shows every device as one internal address like that one, which is +also the address the PS5's downloads come from, so stream installs need it allowed too. + +**6. Pair with the PS5.** On the **Connection** screen press **Send helper**, and the web UI +pairs by itself. If your desktop app is already paired and the console shows no code, see "The +web UI says it isn't paired, and no code shows on the console." + +**7. Keep the computer awake.** A sleeping computer takes the engine with it, and the page stops +answering. On a Mac, turn on **System Settings → Battery → Options → Prevent automatic sleeping on +power adapter when the display is off** (desktop Macs: **Energy**), or run `caffeinate -s` in a +Terminal window and leave it open. A MacBook still sleeps when its lid is closed unless an +external display is attached, so leave the lid open. In Docker Desktop's settings, turn on +**Start Docker Desktop when you sign in to your computer** so it comes back after a restart; +`restart: unless-stopped` brings the engine back with it. + +**Keep it on your home network.** The engine has **no password**: anyone who can open the page can +install, delete and read files on your PS5. `PS5UPLOAD_ALLOW_IP` limits who that is; keep it to +your own network, and never forward port 19113 on your router or open it to the internet. + **Q: What is "Stream install"?** It installs a `.pkg` **straight from your computer**: the PS5 pulls the bytes over your network, so nothing is copied to the console first (no package plus installed game taking room). Keep the From 264b8096f854521cebe937bd045a710e6867b6af Mon Sep 17 00:00:00 2001 From: phantomptr Date: Sat, 17 Oct 2026 11:13:58 -0700 Subject: [PATCH 3/3] Review fixes for #473: reconcile folders only on a partial app.db read - Registered-only titles are added only when app.db could not be read in full; the helper's registered list is every /user/app folder, so with a complete read a folder app.db does not list is a leftover, not a game. NPXS ids are excluded, and a candidate needs a mount.lnk source or a param.json/param.sfo in its sce_sys. - app.db pages carry the table's row count (SQL reader only); the engine marks the list truncated when it changes between pages. - A SQL walk that ends on anything but SQLITE_DONE reports truncated. - FAQ: the phone guide is phrased as a question; references updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RxCGb8Asp1A1AQYiv1kbgS --- CHANGELOG.md | 5 +- FAQ.md | 10 +-- .../crates/ps5upload-core/src/diagnostics.rs | 11 +++ .../crates/ps5upload-core/src/pkg_install.rs | 1 + .../ps5upload-core/tests/appdb_paging.rs | 22 +++++ engine/crates/ps5upload-engine/src/lib.rs | 84 +++++++++++++++---- payload/include/content_db.h | 8 +- payload/src/content_db.c | 16 +++- payload/tests/content_db_selftest.c | 1 + 9 files changed, 129 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59dd48a1..d48af188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,9 @@ What's new in ps5upload, written for humans. only up to its first 256 entries, and a long list could be cut shorter still, with nothing saying so. On a console with many entries, a game installed by a homebrew installer (no home-screen metadata folder) that sorted past that point was missing from Games. The app now - reads the whole database page by page, also lists any game it registered whose folder is on - the console, and logs a warning when a list could not be read in full (#473). + reads the whole database page by page. If the database still can't be read in full, games + whose folder and registration are on the console are listed anyway, and the app logs a warning + that the list may be incomplete (#473). ## 7.0.34 diff --git a/FAQ.md b/FAQ.md index 10f182ab..7bf55462 100644 --- a/FAQ.md +++ b/FAQ.md @@ -164,8 +164,8 @@ zero means no cap. The pre-6.0 name of this variable still works for one release **Q: Can I run the engine on a different machine (self-hosted)?** Yes. Run it on a server or NAS and point the app at it in **Settings → Engine URL**. For a -step-by-step guide that ends with the app open on a phone, see "Use PS5Upload from your phone or -another computer" below. Images: +step-by-step guide that ends with the app open on a phone, see "How do I use PS5Upload from my phone +or another computer?" below. Images: `ghcr.io/phantomptr/ps5upload-engine` (engine only) and `ghcr.io/phantomptr/ps5upload-engine-webui` (full web UI), tagged `:latest` or `:`. @@ -199,8 +199,8 @@ in the compose file to the folder's owner (`ls -ln` shows the numbers), or run **Q: Can I use ps5upload from a web browser (no desktop app)?** Yes. The `webui` image (above) serves the full app over HTTP; open `http://:19113` from an -allowed IP. Same security rules: no password, trusted LAN only. The next question walks through -it from start to finish. +allowed IP. Same security rules: no password, trusted LAN only. "How do I use PS5Upload from my +phone or another computer?" below walks through it from start to finish. - **Upload works**, but the file picker browses the **engine's** disk, not your browser machine's. Mount your games into the container (`-v /host/games:/pkgs:ro`) and set @@ -212,7 +212,7 @@ it from start to finish. report. Everything that works on the PS5 itself is the same as on desktop. - The docker image can be built using `PS5UPLOAD_BASE_URL` to change the root path of the app. -**Q: Use PS5Upload from your phone or another computer** +**Q: How do I use PS5Upload from my phone or another computer?** Run PS5Upload on one machine, for example a Mac tucked away next to the router, and use the whole app from a browser anywhere on your home network, including Safari on an iPhone or iPad. This uses the web UI Docker image, not the desktop app. diff --git a/engine/crates/ps5upload-core/src/diagnostics.rs b/engine/crates/ps5upload-core/src/diagnostics.rs index 368a8d1b..9c8f8417 100644 --- a/engine/crates/ps5upload-core/src/diagnostics.rs +++ b/engine/crates/ps5upload-core/src/diagnostics.rs @@ -256,6 +256,10 @@ pub struct AppDbList { pub next: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub more: Option, + /// The table's row count when the page was read (SQL reader only). A change between pages + /// means rows moved under the walk, so one may have been skipped: the list is truncated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total: Option, /// True when app.db holds rows this list does not: the helper could not deliver them, a /// page failed (`err` says why), the page bound was reached, or an older helper answered /// a full 256 rows (its old cap, which it could not report). A truncated list is never @@ -299,7 +303,12 @@ pub fn appdb_query(addr: &str) -> Result { let more = page.more.unwrap_or(false); let next = page.next.unwrap_or(offset + page.apps.len() as u64); let got = page.apps.len(); + let total_moved = match (list.as_ref().and_then(|l| l.total), page.total) { + (Some(a), Some(b)) => a != b, + _ => false, + }; let page_truncated = page.truncated + || total_moved || (!paged && got >= APPDB_LEGACY_CAP) // A page that claims more but does not move forward would loop. || (more && next <= offset); @@ -322,6 +331,7 @@ pub fn appdb_query(addr: &str) -> Result { l.offset = None; l.next = None; l.more = None; + l.total = None; return Ok(list.expect("set above")); } offset = next; @@ -332,6 +342,7 @@ pub fn appdb_query(addr: &str) -> Result { l.offset = None; l.next = None; l.more = None; + l.total = None; Ok(l) } diff --git a/engine/crates/ps5upload-core/src/pkg_install.rs b/engine/crates/ps5upload-core/src/pkg_install.rs index 4b4a07df..3e1b26ac 100644 --- a/engine/crates/ps5upload-core/src/pkg_install.rs +++ b/engine/crates/ps5upload-core/src/pkg_install.rs @@ -531,6 +531,7 @@ mod appdb_list_has_title_tests { offset: None, next: None, more: None, + total: None, truncated, } } diff --git a/engine/crates/ps5upload-core/tests/appdb_paging.rs b/engine/crates/ps5upload-core/tests/appdb_paging.rs index c164b8dd..bba10c3f 100644 --- a/engine/crates/ps5upload-core/tests/appdb_paging.rs +++ b/engine/crates/ps5upload-core/tests/appdb_paging.rs @@ -73,6 +73,7 @@ fn paged_helper(rows: usize, page: usize) -> impl Fn(&Value) -> Result( registered: impl IntoIterator, app_dirs: &std::collections::HashSet, seen: &std::collections::HashSet, + appdb_whole: bool, ) -> Vec { + if appdb_whole { + return Vec::new(); + } let mut out: Vec = registered .into_iter() .map(str::trim) - .filter(|tid| looks_like_title_id(tid) && app_dirs.contains(*tid) && !seen.contains(*tid)) + .filter(|tid| { + looks_like_title_id(tid) + && !tid.starts_with("NPXS") + && app_dirs.contains(*tid) + && !seen.contains(*tid) + }) .map(str::to_string) .collect(); out.sort(); @@ -5068,6 +5082,14 @@ fn registered_only_additions<'a>( out } +/// True when `/user/app//sce_sys` holds a param.json or param.sfo: the folder is a title, +/// not the leftovers of a failed install. +fn app_folder_has_param(addr: &str, title_id: &str) -> bool { + ["param.json", "param.sfo"] + .iter() + .any(|f| fs_read(addr, &format!("/user/app/{title_id}/sce_sys/{f}"), 0, 16).is_ok()) +} + /// GET /api/ps5/apps/installed?addr=IP:MGMT_PORT async fn ps5_apps_installed( State(state): State, @@ -5184,6 +5206,9 @@ async fn ps5_apps_installed( complete = complete && crate::console_snapshot::app_folders_whole(internal_listing, app_dirs.len()); if !app_dirs.is_empty() { + // Whether app.db answered in full: only then is a folder it does not list + // known not to be an installed title. + let mut appdb_whole = true; let rows: Option> = match appdb.get() { Ok(list) => { // `err` with no rows: the helper could not read app.db at all. @@ -5198,6 +5223,7 @@ async fn ps5_apps_installed( .unwrap_or_default() ); complete = false; + appdb_whole = false; } Some( list.apps @@ -5209,6 +5235,7 @@ async fn ps5_apps_installed( Err(e) => { crate::log_warn!("apps/installed: app.db query failed: {e}"); complete = false; + appdb_whole = false; None } }; @@ -5235,24 +5262,30 @@ async fn ps5_apps_installed( }); } } - // Registered titles whose folder is there but that neither listing above - // produced (#473): never let a registration vanish because app.db did not - // answer for it. + // When app.db could not be read in full, a title it did not get to still + // has its folder and its registration (#473): list it if the folder holds a + // title (our mount.lnk source, or a param file), never a bare leftover folder. let seen: std::collections::HashSet = titles.iter().map(|t| t.title_id.clone()).collect(); - for tid in - registered_only_additions(reg_map.keys().map(String::as_str), &app_dirs, &seen) - { + for tid in registered_only_additions( + reg_map.keys().map(String::as_str), + &app_dirs, + &seen, + appdb_whole, + ) { let Some(reg) = reg_map.get(&tid) else { continue; }; + if reg.src.trim().is_empty() && !app_folder_has_param(&addr, &tid) { + continue; + } let name = if reg.title_name.trim().is_empty() || reg.title_name == tid { appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone()) } else { reg.title_name.clone() }; titles.push(InstalledApp { - system: tid.starts_with("NPXS"), + system: false, title_id: tid, title_name: name, origin: "registered".to_string(), @@ -12672,26 +12705,43 @@ mod appdb_installed_additions_tests { ); } - /// #473: a registered title with its app folder but no appmeta and no - /// app.db row (a failed or partial app.db read) is still listed. + /// #473: when app.db could not be read in full, a registered title with its + /// app folder but no appmeta and no app.db row is a candidate. #[test] - fn registered_titles_with_a_folder_are_reconciled() { + fn an_incomplete_appdb_read_reconciles_registered_titles_with_a_folder() { let got = registered_only_additions( [ "PPSA00002", "ZZZZ99999", "CUSA00001", "PPSA00003", + "NPXS40056", "not-an-id", ], - &set(&["ZZZZ99999", "CUSA00001", "PPSA00002", "not-an-id"]), + &set(&[ + "ZZZZ99999", + "CUSA00001", + "PPSA00002", + "NPXS40056", + "not-an-id", + ]), // already listed by the appmeta scan or app.db &set(&["CUSA00001"]), + false, ); - // PPSA00003 has no folder: a stale registration is not an installed title. + // PPSA00003 has no folder; NPXS is a system id, never added this way. assert_eq!(got, vec!["PPSA00002".to_string(), "ZZZZ99999".to_string()]); } + /// A complete app.db read that does not list a folder means the folder is not an + /// installed title (a failed install, a partial uninstall): it is not added. + #[test] + fn a_complete_appdb_read_adds_no_folder_it_does_not_list() { + let got = + registered_only_additions(["PPSA00002"], &set(&["PPSA00002"]), &HashSet::new(), true); + assert!(got.is_empty()); + } + #[test] fn skips_titles_the_appmeta_scan_already_found() { let rows = vec![("PLDM00001".to_string(), "Payload Manager".to_string())]; diff --git a/payload/include/content_db.h b/payload/include/content_db.h index b0c3515a..cfc92b52 100644 --- a/payload/include/content_db.h +++ b/payload/include/content_db.h @@ -83,13 +83,17 @@ int content_db_select_text(const char *db_path, const char *sql, int max_rows, * * Emits {"apps":[{"title_id":"..","app_id":N,"name":".."}, ...], * "source":"sqlite"|"sqlite-immutable"|"scan", - * "offset":O,"next":N,"more":bool,"truncated":bool} + * "offset":O,"next":N,"more":bool,"truncated":bool,"total":T} * + * `total` is the table's row count, sent only from SQL (the scanner's is a + * lower bound); a caller that sees it change between pages knows rows + * moved under it. * `next` is the row the following page starts at; `more` is true while * rows remain after it. A page that fills `buf` before `limit` rows stops * early with `more` set, so a small buffer pages rather than silently * dropping the rest. `truncated` is true only when rows exist that no - * page can deliver: one row larger than `buf`, or the byte scanner (the + * page can deliver: one row larger than `buf`, a SQL walk that stopped + * before the end (SQLITE_BUSY and the like), or the byte scanner (the * fallback for a database that will not open) hitting its own bounds. * * `limit` 0 (or above CONTENT_DB_APPS_PAGE_MAX) means diff --git a/payload/src/content_db.c b/payload/src/content_db.c index ba7b6ed6..6cd98121 100644 --- a/payload/src/content_db.c +++ b/payload/src/content_db.c @@ -325,7 +325,8 @@ static int cdb_apps_window(content_db_app_t *out, unsigned offset, int max, continue; /* Every row is counted (so the caller knows whether more * follow); only the window is copied. */ - while (sqlite3_step(st) == SQLITE_ROW) { + int step; + while ((step = sqlite3_step(st)) == SQLITE_ROW) { const unsigned char *tid = sqlite3_column_text(st, 0); if (!tid) continue; unsigned idx = rows_seen++; @@ -335,6 +336,9 @@ static int cdb_apps_window(content_db_app_t *out, unsigned offset, int max, (const char *)nm, sqlite3_column_int(st, 1)); count++; } + /* A walk that ended on anything but DONE (BUSY under the + * shell's write, a damaged page) did not see every row. */ + if (rows_seen > 0 && step != SQLITE_DONE && capped) *capped = 1; sqlite3_finalize(st); } sqlite3_close(db); @@ -499,11 +503,17 @@ int content_db_apps_page_json(unsigned offset, unsigned limit, char *buf, int truncated = capped || stuck; if (stuck) more_rows = 0; + /* `total` is exact only from SQL: the scanner's is a lower bound that + * grows with the window, so it is left out rather than mislead a + * caller comparing pages. */ + char total_field[32] = ""; + if (strcmp(source, "scan") != 0) + snprintf(total_field, sizeof(total_field), ",\"total\":%u", total); int end2 = snprintf(buf + n, cap - (size_t)n, "],\"source\":\"%s\",\"offset\":%u,\"next\":%u," - "\"more\":%s,\"truncated\":%s}", + "\"more\":%s,\"truncated\":%s%s}", source, offset, next, more_rows ? "true" : "false", - truncated ? "true" : "false"); + truncated ? "true" : "false", total_field); if (end2 < 0 || (size_t)(n + end2) >= cap) return -1; n += end2; if (written) *written = (size_t)n; diff --git a/payload/tests/content_db_selftest.c b/payload/tests/content_db_selftest.c index 0ec3649d..50299d6f 100644 --- a/payload/tests/content_db_selftest.c +++ b/payload/tests/content_db_selftest.c @@ -411,6 +411,7 @@ int main(void) { assert(json_true(pbuf, "more")); assert(!json_true(pbuf, "truncated")); assert(json_num(pbuf, "next") > 256); + assert(json_num(pbuf, "total") == 1001); printf("✓ the unpaged call says more rows follow (next=%ld)\n", json_num(pbuf, "next"));