From b7b1ccaf5834ca46db91d4d49e095bd2d65585b6 Mon Sep 17 00:00:00 2001 From: phantomptr Date: Sat, 17 Oct 2026 11:07:03 -0700 Subject: [PATCH] Page through the app database so no installed game is hidden (#473) APP_DB_QUERY answered at most 256 rows, and fewer when its 64 KiB reply filled, with nothing saying the list was cut. A title installed by a homebrew installer (no /user/appmeta folder) that sorted past that point disappeared from the installed list. Payload: app.db.query takes an optional {"offset","limit"} window and answers offset/next/more/truncated; a full buffer now ends the page with more=true instead of closing the list. An empty request is page one, so older engines keep working. Engine: appdb_query follows the pages (at most 32) and marks the list truncated when it could not finish (or an older helper returned a full 256). The installed list reads app.db once per request, adds registered titles whose app folder exists, reports incomplete, and a truncated list never proves a title absent for install verification. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RxCGb8Asp1A1AQYiv1kbgS --- CHANGELOG.md | 9 + client/src/api/ps5.ts | 14 +- client/src/screens/InstalledApps/index.tsx | 5 + client/src/screens/Library/index.tsx | 2 +- engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c | 6 +- .../crates/ps5upload-core/src/diagnostics.rs | 86 ++++++- .../crates/ps5upload-core/src/pkg_install.rs | 58 ++++- .../ps5upload-core/tests/appdb_paging.rs | 210 ++++++++++++++++ engine/crates/ps5upload-engine/src/lib.rs | 212 ++++++++++++---- payload/include/content_db.h | 27 +- payload/src/content_db.c | 146 ++++++++--- payload/src/mgmt_table.def | 2 +- payload/src/runtime.c | 28 ++- payload/tests/content_db_selftest.c | 237 ++++++++++++++++++ protocol/ava1/MGMT_METHODS.md | 2 +- 15 files changed, 947 insertions(+), 97 deletions(-) create mode 100644 engine/crates/ps5upload-core/tests/appdb_paging.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c8de113a..59dd48a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ What's new in ps5upload, written for humans. --- +## Unreleased + +- **Big app databases no longer hide installed games.** The console's app database was read + only up to its first 256 entries, and a long list could be cut shorter still, with nothing + saying so. On a console with many entries, a game installed by a homebrew installer (no + home-screen metadata folder) that sorted past that point was missing from Games. The app now + reads the whole database page by page, also lists any game it registered whose folder is on + the console, and logs a warning when a list could not be read in full (#473). + ## 7.0.34 - **Unity games no longer crash when they run as a folder.** Some game dumps name a file diff --git a/client/src/api/ps5.ts b/client/src/api/ps5.ts index c3439bd9..2fa3c39e 100644 --- a/client/src/api/ps5.ts +++ b/client/src/api/ps5.ts @@ -2853,6 +2853,10 @@ export interface AppDbEntry { export interface AppDbList { apps: AppDbEntry[]; err?: string; + /** Which reader answered: "sqlite", "sqlite-immutable" or "scan". */ + source?: string; + /** True when app.db holds rows this list does not (#473). */ + truncated?: boolean; } export async function appdbQuery(addr: string): Promise { @@ -3519,6 +3523,9 @@ export interface InstalledAppsResult { /** True if the payload couldn't report the "registered" set (older * payload); the UI then shows everything as "installed" with a note. */ registeredUnavailable: boolean; + /** True if the console's list may be missing titles (a listing hit its + * page size, or app.db could not be read in full, #473). */ + incomplete: boolean; } /** List every installed title on the PS5, grouped by origin. Routes through @@ -3538,6 +3545,7 @@ export async function appsInstalled( system?: boolean; }>; registered_unavailable?: boolean; + incomplete?: boolean; }>("ps5_apps_installed", { addr }); const titles: InstalledTitle[] = (res?.titles ?? []) .filter((t): t is { title_id: string } & typeof t => !!t.title_id) @@ -3550,7 +3558,11 @@ export async function appsInstalled( source: t.source ?? "", system: !!t.system, })); - return { titles, registeredUnavailable: !!res?.registered_unavailable }; + return { + titles, + registeredUnavailable: !!res?.registered_unavailable, + incomplete: !!res?.incomplete, + }; } export type InstalledPkgKind = "base" | "patch" | "dlc"; diff --git a/client/src/screens/InstalledApps/index.tsx b/client/src/screens/InstalledApps/index.tsx index b36e737f..af14b559 100644 --- a/client/src/screens/InstalledApps/index.tsx +++ b/client/src/screens/InstalledApps/index.tsx @@ -812,6 +812,11 @@ export default function InstalledAppsScreen() { if (probe.isStale()) return; setTitles(res.titles); setRegisteredUnavailable(res.registeredUnavailable); + if (res.incomplete) { + console.warn( + "[installed] the console's installed list may be incomplete (a listing or app.db read did not finish); refresh to retry", + ); + } try { const sdk = await sdkScan(mgmtAddr(probe.host)); if (!probe.isStale()) setSdkState(sdk); diff --git a/client/src/screens/Library/index.tsx b/client/src/screens/Library/index.tsx index 9d03f11d..e9edb769 100644 --- a/client/src/screens/Library/index.tsx +++ b/client/src/screens/Library/index.tsx @@ -281,7 +281,7 @@ export default function LibraryScreen() { "[library] installed-apps probe failed; disk-image rows will show filenames only:", e, ); - return { titles: [], registeredUnavailable: true }; + return { titles: [], registeredUnavailable: true, incomplete: true }; }), ]); // Two ways a registered title points back at a Library row: diff --git a/engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c b/engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c index bf08e99c..b5de9665 100644 --- a/engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c +++ b/engine/crates/ava1-ctest/csrc/mgmt_t6_shim.c @@ -174,9 +174,9 @@ static int handle_appinfo_set(void *st, int fd, uint64_t t, const char *b, uint6 if (strstr(b, "RUNNING")) return ok_body("{\"ok\":false,\"err\":\"title is running\"}"); return ok_body("{\"ok\":true,\"err\":null}"); } -static int handle_appdb_query(void *st, int fd, uint64_t t) { - (void)st; (void)fd; (void)t; - note(AVA1_METHOD_APP_DB_QUERY, NULL); +static int handle_appdb_query(void *st, int fd, uint64_t t, const char *b, uint64_t l) { + (void)st; (void)fd; (void)t; (void)l; + note(AVA1_METHOD_APP_DB_QUERY, b); return ok_body("{\"apps\":[{\"title_id\":\"CUSA00001\",\"app_id\":57368,\"name\":\"Stub\"}],\"source\":\"sql\"}"); } static int handle_focus_probe(void *st, int fd, uint64_t t) { diff --git a/engine/crates/ps5upload-core/src/diagnostics.rs b/engine/crates/ps5upload-core/src/diagnostics.rs index 18af6dce..368a8d1b 100644 --- a/engine/crates/ps5upload-core/src/diagnostics.rs +++ b/engine/crates/ps5upload-core/src/diagnostics.rs @@ -247,12 +247,92 @@ pub struct AppDbList { /// a caller that sees a blank name knows whether to believe it. #[serde(default)] pub source: Option, + /// One page's window (#473): the first row it holds, the row the next page starts at, and + /// whether one follows. Absent from an older helper, which answers one unpaged reply, and + /// from the list [`appdb_query`] returns, which is every page joined. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offset: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub more: Option, + /// True when app.db holds rows this list does not: the helper could not deliver them, a + /// page failed (`err` says why), the page bound was reached, or an older helper answered + /// a full 256 rows (its old cap, which it could not report). A truncated list is never + /// proof that a title is absent. + #[serde(default)] + pub truncated: bool, } -/// Query app.db for the title_id ↔ app_id ↔ name mapping. +/// The most app.db pages [`appdb_query`] reads. A page holds up to 512 rows, so this is +/// far past any real console; it bounds the loop against a helper that never says it is done. +pub const APPDB_MAX_PAGES: usize = 32; + +/// The row count an older, unpaged helper stopped at without saying so. +const APPDB_LEGACY_CAP: usize = 256; + +/// Query app.db for the title_id ↔ app_id ↔ name mapping: every page of it (#473). +/// +/// The helper answers a window (`{"offset":N}` in; `next`, `more` and `truncated` out); this +/// follows it until `more` is false, at most [`APPDB_MAX_PAGES`] calls. An older helper ignores +/// the window and answers once, which is then the whole answer. Whatever stops the walk early +/// sets `truncated` on the result, so callers can tell a partial list from a complete one: +/// a failure after the first page keeps the rows read so far, with the reason in `err`. pub fn appdb_query(addr: &str) -> Result { - let resp = mgmt::call(addr, m::APP_DB_QUERY, &[])?; - Ok(serde_json::from_slice(&resp)?) + let mut list: Option = None; + let mut offset: u64 = 0; + for _ in 0..APPDB_MAX_PAGES { + let body = serde_json::json!({ "offset": offset }).to_string(); + let page = mgmt::call(addr, m::APP_DB_QUERY, body.as_bytes()) + .and_then(|resp| Ok(serde_json::from_slice::(&resp)?)); + let mut page = match (page, list.as_mut()) { + (Ok(p), _) => p, + // Page one failing is the call failing, as it always was. + (Err(e), None) => return Err(e), + (Err(e), Some(l)) => { + l.truncated = true; + l.err = Some(format!("app.db page at row {offset}: {e:#}")); + break; + } + }; + let paged = page.more.is_some() || page.next.is_some(); + let more = page.more.unwrap_or(false); + let next = page.next.unwrap_or(offset + page.apps.len() as u64); + let got = page.apps.len(); + let page_truncated = page.truncated + || (!paged && got >= APPDB_LEGACY_CAP) + // A page that claims more but does not move forward would loop. + || (more && next <= offset); + match list.as_mut() { + None => { + page.truncated = page_truncated; + list = Some(page); + } + Some(l) => { + l.apps.append(&mut page.apps); + l.truncated |= page_truncated; + if page.err.is_some() { + l.err = page.err.take(); + l.truncated = true; + } + } + } + let l = list.as_mut().expect("set above"); + if !paged || !more || next <= offset || l.err.is_some() { + l.offset = None; + l.next = None; + l.more = None; + return Ok(list.expect("set above")); + } + offset = next; + } + // Here only when a later page failed or the page bound ran out with pages still to read. + let mut l = list.expect("APPDB_MAX_PAGES > 0"); + l.truncated = true; + l.offset = None; + l.next = None; + l.more = None; + Ok(l) } /// One (key, value) row of appinfo.db for a title. diff --git a/engine/crates/ps5upload-core/src/pkg_install.rs b/engine/crates/ps5upload-core/src/pkg_install.rs index 41856af4..4b4a07df 100644 --- a/engine/crates/ps5upload-core/src/pkg_install.rs +++ b/engine/crates/ps5upload-core/src/pkg_install.rs @@ -320,14 +320,26 @@ fn probe_app_pkg_at(addr: &str, app_dir: &str) -> PkgProbe { /// Whether app.db lists `title_id`. `Some(true/false)` only when the query /// actually returned a row set (`err == None`); `None` when the database /// could not be read at all, so an unreadable database never counts as -/// "the title is absent". +/// "the title is absent". Likewise a list the helper could only partly +/// deliver (`truncated`, #473): finding the title there is proof, missing +/// it is not. fn appdb_has_title(addr: &str, title_id: &str) -> Option { match crate::diagnostics::appdb_query(addr) { - Ok(list) if list.err.is_none() => Some(list.apps.iter().any(|a| a.title_id == title_id)), + Ok(list) if list.err.is_none() => appdb_list_has_title(&list, title_id), _ => None, } } +fn appdb_list_has_title(list: &crate::diagnostics::AppDbList, title_id: &str) -> Option { + if list.apps.iter().any(|a| a.title_id == title_id) { + Some(true) + } else if list.truncated { + None + } else { + Some(false) + } +} + /// Check whether `title_id` (e.g. "PPSA01650") installed a launchable title /// on the PS5. /// @@ -498,3 +510,45 @@ mod tests { assert_eq!(p, InstallPhase::Done); } } + +#[cfg(test)] +mod appdb_list_has_title_tests { + use super::appdb_list_has_title; + use crate::diagnostics::{AppDbEntry, AppDbList}; + + fn list(ids: &[&str], truncated: bool) -> AppDbList { + AppDbList { + apps: ids + .iter() + .map(|t| AppDbEntry { + title_id: t.to_string(), + app_id: 1, + name: String::new(), + }) + .collect(), + err: None, + source: None, + offset: None, + next: None, + more: None, + truncated, + } + } + + /// #473: a title missing from a list the helper could not finish is unknown, not absent. + #[test] + fn a_truncated_list_never_proves_a_title_absent() { + assert_eq!( + appdb_list_has_title(&list(&["CUSA00001"], true), "CUSA00001"), + Some(true) + ); + assert_eq!( + appdb_list_has_title(&list(&["CUSA00001"], true), "PPSA00002"), + None + ); + assert_eq!( + appdb_list_has_title(&list(&["CUSA00001"], false), "PPSA00002"), + Some(false) + ); + } +} diff --git a/engine/crates/ps5upload-core/tests/appdb_paging.rs b/engine/crates/ps5upload-core/tests/appdb_paging.rs new file mode 100644 index 00000000..c164b8dd --- /dev/null +++ b/engine/crates/ps5upload-core/tests/appdb_paging.rs @@ -0,0 +1,210 @@ +//! #473: `appdb_query` pages through app.db instead of trusting one capped reply. +//! +//! The helper used to answer at most 256 rows (and fewer when its buffer filled) with nothing +//! saying the list was cut. It now answers a window (`offset`/`limit` in, `next`/`more`/`truncated` +//! out); the engine follows it to the end, bounded, and says when the list is not whole. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use ps5upload_core::diagnostics::{appdb_query, APPDB_MAX_PAGES}; +use ps5upload_core::mgmt::{self, Method, MgmtTransport}; +use serde_json::{json, Value}; + +type Script = Box Result>> + Send + Sync>; + +struct Fake { + seen: Mutex>, + reply: Script, +} + +impl MgmtTransport for Fake { + fn call( + &self, + _addr: &str, + method: Method, + _label: &str, + body: &[u8], + _timeout: Duration, + ) -> Result>> { + assert_eq!(method.id, 56, "app.db_query"); + let req: Value = if body.is_empty() { + json!({}) + } else { + serde_json::from_slice(body).expect("the request is JSON") + }; + self.seen.lock().unwrap().push(req.clone()); + (self.reply)(&req) + } +} + +fn fake( + f: impl Fn(&Value) -> Result>> + Send + Sync + 'static, +) -> (Arc, mgmt::ScopedTransport) { + let t = Arc::new(Fake { + seen: Mutex::default(), + reply: Box::new(f), + }); + let g = mgmt::scoped_transport(t.clone()); + (t, g) +} + +fn title(i: usize) -> String { + format!("CUSA{i:05}") +} + +/// A helper that serves `rows` titles (plus a late-sorting ZZZZ99999) `page` at a time. +fn paged_helper(rows: usize, page: usize) -> impl Fn(&Value) -> Result>> { + move |req| { + let mut all: Vec = (0..rows).map(title).collect(); + all.push("ZZZZ99999".into()); + let off = req["offset"].as_u64().unwrap_or(0) as usize; + let end = (off + page).min(all.len()); + let apps: Vec = all[off.min(all.len())..end] + .iter() + .map(|t| json!({"title_id": t, "app_id": 1, "name": format!("Game {t}")})) + .collect(); + Ok(Some( + serde_json::to_vec(&json!({ + "apps": apps, + "source": "sqlite", + "offset": off, + "next": end, + "more": end < all.len(), + "truncated": false, + })) + .unwrap(), + )) + } +} + +const A: &str = "10.0.0.5"; + +#[test] +fn pages_until_the_helper_says_there_is_no_more() { + for rows in [255usize, 256, 257, 1000, 1500] { + let (t, _g) = fake(paged_helper(rows, 400)); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), rows + 1, "{rows} rows"); + assert!(!d.truncated, "{rows} rows"); + assert!(d.err.is_none()); + assert_eq!(d.apps.last().unwrap().title_id, "ZZZZ99999"); + assert_eq!(d.source.as_deref(), Some("sqlite")); + let seen = t.seen.lock().unwrap(); + assert_eq!(seen.len(), (rows + 1).div_ceil(400), "{rows} rows"); + for (i, req) in seen.iter().enumerate() { + assert_eq!(req["offset"], json!(i * 400), "page {i}"); + } + } +} + +#[test] +fn an_older_helper_answers_once_and_a_full_256_is_not_called_whole() { + // No next/more: the helper predates paging, so one reply is all there is. + let (t, _g) = fake(|_| { + let apps: Vec = (0..256) + .map(|i| json!({"title_id": title(i), "app_id": 1, "name": "n"})) + .collect(); + Ok(Some( + serde_json::to_vec(&json!({"apps": apps, "source": "sqlite"})).unwrap(), + )) + }); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), 256); + assert!(d.truncated, "256 rows from an older helper may be its cap"); + assert_eq!(t.seen.lock().unwrap().len(), 1); + drop(_g); + + let (_t, _g) = fake(|_| { + Ok(Some( + br#"{"apps":[{"title_id":"CUSA00001","app_id":1,"name":"n"}],"source":"scan"}"# + .to_vec(), + )) + }); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), 1); + assert!(!d.truncated); +} + +#[test] +fn a_truncated_page_marks_the_whole_list_truncated() { + let (_t, _g) = fake(|req| { + let off = req["offset"].as_u64().unwrap_or(0); + Ok(Some( + serde_json::to_vec(&if off == 0 { + json!({"apps":[{"title_id":"CUSA00001","app_id":1,"name":"a"}], + "offset":0,"next":1,"more":true,"truncated":false}) + } else { + json!({"apps":[],"offset":1,"next":1,"more":false,"truncated":true}) + }) + .unwrap(), + )) + }); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), 1); + assert!(d.truncated); +} + +#[test] +fn a_page_that_makes_no_progress_stops_the_loop() { + let (t, _g) = fake(|_| { + Ok(Some( + br#"{"apps":[],"offset":0,"next":0,"more":true,"truncated":false}"#.to_vec(), + )) + }); + let d = appdb_query(A).unwrap(); + assert!(d.truncated); + assert_eq!(t.seen.lock().unwrap().len(), 1); +} + +#[test] +fn the_page_count_is_bounded() { + // A helper that always has more: the engine gives up after APPDB_MAX_PAGES calls. + let (t, _g) = fake(|req| { + let off = req["offset"].as_u64().unwrap_or(0); + Ok(Some( + serde_json::to_vec(&json!({ + "apps":[{"title_id": title(off as usize), "app_id": 1, "name": "n"}], + "offset": off, "next": off + 1, "more": true, "truncated": false + })) + .unwrap(), + )) + }); + let d = appdb_query(A).unwrap(); + assert!(d.truncated); + assert_eq!(d.apps.len(), APPDB_MAX_PAGES); + assert_eq!(t.seen.lock().unwrap().len(), APPDB_MAX_PAGES); +} + +#[test] +fn errors_keep_their_meaning() { + // The helper's err envelope on page one is the answer, as before. + let (_t, _g) = fake(|_| Ok(Some(br#"{"err":"cannot read app.db","apps":[]}"#.to_vec()))); + let d = appdb_query(A).unwrap(); + assert_eq!(d.err.as_deref(), Some("cannot read app.db")); + assert!(d.apps.is_empty()); + drop(_g); + + // A transport failure on page one is still an error. + let (_t, _g) = fake(|_| Err(anyhow!("connection reset"))); + assert!(appdb_query(A).is_err()); + drop(_g); + + // A failure after page one keeps the rows read so far, flagged incomplete with the reason, + // so a caller never mistakes the partial list for the whole one. + let (_t, _g) = fake(|req| { + if req["offset"].as_u64().unwrap_or(0) == 0 { + Ok(Some( + br#"{"apps":[{"title_id":"CUSA00001","app_id":1,"name":"a"}],"offset":0,"next":1,"more":true,"truncated":false}"# + .to_vec(), + )) + } else { + Err(anyhow!("connection reset")) + } + }); + let d = appdb_query(A).unwrap(); + assert_eq!(d.apps.len(), 1); + assert!(d.truncated); + assert!(d.err.as_deref().unwrap_or("").contains("connection reset")); +} diff --git a/engine/crates/ps5upload-engine/src/lib.rs b/engine/crates/ps5upload-engine/src/lib.rs index 317f74bf..96f22bd8 100644 --- a/engine/crates/ps5upload-engine/src/lib.rs +++ b/engine/crates/ps5upload-engine/src/lib.rs @@ -4933,6 +4933,40 @@ struct InstalledAppsResponse { /// UI then can't reliably tag the "registered" group, so it shows /// everything under "installed" with a soft note rather than erroring. registered_unavailable: bool, + /// True when the list may be missing titles: a directory listing hit its page size, + /// or app.db could not be read in full (a failed query, or a helper that could not + /// deliver every row, #473). The titles shown are real; there may be more. + incomplete: bool, +} + +/// app.db, read at most once per request: a paged read is several calls (#473), and the +/// installed list asks it for names title by title. +struct AppDbOnce<'a> { + addr: &'a str, + got: Option>, +} + +impl<'a> AppDbOnce<'a> { + fn new(addr: &'a str) -> Self { + Self { addr, got: None } + } + + fn get(&mut self) -> &Result { + let addr = self.addr; + self.got + .get_or_insert_with(|| appdb_query(addr).map_err(|e| format!("{e:#}"))) + } + + /// app.db's display name for `title_id`, when it has one that is not the id itself. + fn name_of(&mut self, title_id: &str) -> Option { + let list = self.get().as_ref().ok()?; + list.apps + .iter() + .filter(|a| a.title_id == title_id) + .map(|a| a.name.trim()) + .find(|n| !n.is_empty() && *n != title_id) + .map(str::to_string) + } } /// PS5 title-ids are 4 uppercase letters + 5 digits (PPSA01234, NPXS39041, @@ -4950,7 +4984,7 @@ fn looks_like_title_id(name: &str) -> bool { /// `app.db` (raw file scan when sqlite is unavailable). Returns None on /// any failure (no metadata file — common for system apps — bad JSON, /// bad SFO, path denied); the caller falls back to the bare title_id. -fn appmeta_title_name(addr: &str, title_id: &str) -> Option { +fn appmeta_title_name(addr: &str, title_id: &str, appdb: &mut AppDbOnce<'_>) -> Option { // /user/appmeta/ first; then the app's own sce_sys, which is all a // title installed by a homebrew installer has on FW 13.60. for dir in [ @@ -4981,17 +5015,7 @@ fn appmeta_title_name(addr: &str, title_id: &str) -> Option { // This is the only way to resolve PS4 BC titles (CUSA IDs) whose // metadata lives solely in app.db — their appmeta dirs have no // param.sfo or param.json. - if let Ok(appdb) = appdb_query(addr) { - for app in &appdb.apps { - if app.title_id == title_id { - let name = app.name.trim(); - if !name.is_empty() && name != title_id { - return Some(name.to_string()); - } - } - } - } - None + appdb.name_of(title_id) } /// Titles app.db knows are installed that the /user/appmeta scan missed, @@ -5024,6 +5048,26 @@ fn appdb_installed_additions( out } +/// Titles we registered whose app folder exists but that neither the /user/appmeta scan +/// nor app.db produced (#473): a homebrew-installed title has no appmeta folder, and app.db +/// may not have answered for it (a failed or partial read). The registration and the folder +/// on disk are evidence enough. Sorted, so the list does not reshuffle between refreshes. +fn registered_only_additions<'a>( + registered: impl IntoIterator, + app_dirs: &std::collections::HashSet, + seen: &std::collections::HashSet, +) -> Vec { + let mut out: Vec = registered + .into_iter() + .map(str::trim) + .filter(|tid| looks_like_title_id(tid) && app_dirs.contains(*tid) && !seen.contains(*tid)) + .map(str::to_string) + .collect(); + out.sort(); + out.dedup(); + out +} + /// GET /api/ps5/apps/installed?addr=IP:MGMT_PORT async fn ps5_apps_installed( State(state): State, @@ -5050,6 +5094,7 @@ async fn ps5_apps_installed( } }; + let mut appdb = AppDbOnce::new(&addr); // Full installed set: every /user/appmeta// dir. let listing = list_dir( &addr, @@ -5074,7 +5119,7 @@ async fn ps5_apps_installed( // the id (older registrations stored id-as-name), try // param.json for a friendlier label. let name = if reg.title_name.trim().is_empty() || reg.title_name == tid { - appmeta_title_name(&addr, &tid).unwrap_or_else(|| tid.clone()) + appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone()) } else { reg.title_name.clone() }; @@ -5087,7 +5132,8 @@ async fn ps5_apps_installed( system, }); } else { - let name = appmeta_title_name(&addr, &tid).unwrap_or_else(|| tid.clone()); + let name = + appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone()); titles.push(InstalledApp { title_id: tid.clone(), title_name: name, @@ -5138,40 +5184,81 @@ async fn ps5_apps_installed( complete = complete && crate::console_snapshot::app_folders_whole(internal_listing, app_dirs.len()); if !app_dirs.is_empty() { - match appdb_query(&addr) { - Ok(appdb) => { - let seen: std::collections::HashSet = - titles.iter().map(|t| t.title_id.clone()).collect(); - let rows: Vec<(String, String)> = appdb - .apps - .into_iter() - .map(|a| (a.title_id, a.name)) - .collect(); - for (tid, db_name) in appdb_installed_additions(&rows, &app_dirs, &seen) { - let name = match db_name { - Some(n) => n, - None => { - appmeta_title_name(&addr, &tid).unwrap_or_else(|| tid.clone()) - } - }; - let (origin, image_backed, source) = match reg_map.get(&tid) { - Some(reg) => ("registered", reg.image_backed, reg.src.clone()), - None => ("pkg", false, String::new()), - }; - titles.push(InstalledApp { - title_id: tid, - title_name: name, - origin: origin.to_string(), - image_backed, - source, - system: false, - }); + let rows: Option> = match appdb.get() { + Ok(list) => { + // `err` with no rows: the helper could not read app.db at all. + if list.truncated || list.err.is_some() { + // Every row the helper delivered is used; the list just may not be whole. + crate::log_warn!( + "apps/installed: app.db read incomplete ({} rows{})", + list.apps.len(), + list.err + .as_deref() + .map(|e| format!(": {e}")) + .unwrap_or_default() + ); + complete = false; } + Some( + list.apps + .iter() + .map(|a| (a.title_id.clone(), a.name.clone())) + .collect(), + ) } Err(e) => { - crate::log_warn!("apps/installed: app.db query failed: {e:#}"); + crate::log_warn!("apps/installed: app.db query failed: {e}"); complete = false; + None } + }; + if let Some(rows) = rows { + let seen: std::collections::HashSet = + titles.iter().map(|t| t.title_id.clone()).collect(); + for (tid, db_name) in appdb_installed_additions(&rows, &app_dirs, &seen) { + let name = match db_name { + Some(n) => n, + None => appmeta_title_name(&addr, &tid, &mut appdb) + .unwrap_or_else(|| tid.clone()), + }; + let (origin, image_backed, source) = match reg_map.get(&tid) { + Some(reg) => ("registered", reg.image_backed, reg.src.clone()), + None => ("pkg", false, String::new()), + }; + titles.push(InstalledApp { + title_id: tid, + title_name: name, + origin: origin.to_string(), + image_backed, + source, + system: false, + }); + } + } + // Registered titles whose folder is there but that neither listing above + // produced (#473): never let a registration vanish because app.db did not + // answer for it. + let seen: std::collections::HashSet = + titles.iter().map(|t| t.title_id.clone()).collect(); + for tid in + registered_only_additions(reg_map.keys().map(String::as_str), &app_dirs, &seen) + { + let Some(reg) = reg_map.get(&tid) else { + continue; + }; + let name = if reg.title_name.trim().is_empty() || reg.title_name == tid { + appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone()) + } else { + reg.title_name.clone() + }; + titles.push(InstalledApp { + system: tid.starts_with("NPXS"), + title_id: tid, + title_name: name, + origin: "registered".to_string(), + image_backed: reg.image_backed, + source: reg.src.clone(), + }); } } // Kept for the game page. System apps are not games. @@ -5211,6 +5298,7 @@ async fn ps5_apps_installed( Ok(InstalledAppsResponse { titles, registered_unavailable, + incomplete: !complete, }) }) .await @@ -12537,7 +12625,7 @@ mod job_stage_tests { #[cfg(test)] mod appdb_installed_additions_tests { - use super::appdb_installed_additions; + use super::{appdb_installed_additions, registered_only_additions}; use std::collections::HashSet; fn set(v: &[&str]) -> HashSet { @@ -12568,6 +12656,42 @@ mod appdb_installed_additions_tests { ); } + /// #473: a title past the first 256 app.db rows is still added once the + /// rows arrive — the window, not the filter, was what hid it. + #[test] + fn a_late_sorting_title_past_row_256_is_kept() { + let mut rows: Vec<(String, String)> = (0..1000) + .map(|i| (format!("CUSA{i:05}"), format!("Game {i}"))) + .collect(); + rows.insert(0, ("AAAA00001".to_string(), "Early".to_string())); + rows.push(("ZZZZ99999".to_string(), "Late Title".to_string())); + let got = appdb_installed_additions(&rows, &set(&["ZZZZ99999"]), &HashSet::new()); + assert_eq!( + got, + vec![("ZZZZ99999".to_string(), Some("Late Title".to_string()))] + ); + } + + /// #473: a registered title with its app folder but no appmeta and no + /// app.db row (a failed or partial app.db read) is still listed. + #[test] + fn registered_titles_with_a_folder_are_reconciled() { + let got = registered_only_additions( + [ + "PPSA00002", + "ZZZZ99999", + "CUSA00001", + "PPSA00003", + "not-an-id", + ], + &set(&["ZZZZ99999", "CUSA00001", "PPSA00002", "not-an-id"]), + // already listed by the appmeta scan or app.db + &set(&["CUSA00001"]), + ); + // PPSA00003 has no folder: a stale registration is not an installed title. + assert_eq!(got, vec!["PPSA00002".to_string(), "ZZZZ99999".to_string()]); + } + #[test] fn skips_titles_the_appmeta_scan_already_found() { let rows = vec![("PLDM00001".to_string(), "Payload Manager".to_string())]; diff --git a/payload/include/content_db.h b/payload/include/content_db.h index 67e840e6..b0c3515a 100644 --- a/payload/include/content_db.h +++ b/payload/include/content_db.h @@ -78,18 +78,37 @@ int content_db_select_text(const char *db_path, const char *sql, int max_rows, const char *const *vals)); /* - * Installed titles from app.db: title id, app id and display name. + * Installed titles from app.db, one page at a time (#473): title id, app + * id and display name for rows [offset, offset + limit) in title order. * * Emits {"apps":[{"title_id":"..","app_id":N,"name":".."}, ...], - * "source":"sqlite"|"scan"} + * "source":"sqlite"|"sqlite-immutable"|"scan", + * "offset":O,"next":N,"more":bool,"truncated":bool} + * + * `next` is the row the following page starts at; `more` is true while + * rows remain after it. A page that fills `buf` before `limit` rows stops + * early with `more` set, so a small buffer pages rather than silently + * dropping the rest. `truncated` is true only when rows exist that no + * page can deliver: one row larger than `buf`, or the byte scanner (the + * fallback for a database that will not open) hitting its own bounds. + * + * `limit` 0 (or above CONTENT_DB_APPS_PAGE_MAX) means + * CONTENT_DB_APPS_PAGE_MAX. An older engine sends no window at all and + * reads page one; the extra fields are ignored by its decoder. * * `source` is reported rather than hidden: a caller that sees "scan" * knows the SQL open failed and the row set came from the byte-level * reader, which recovers ids reliably but names only sometimes. * - * Never fails outright — falls back to appdb_scan.h. Returns 0 on - * success, non-zero only if `buf` is too small to hold even an error. + * Never fails outright — falls back to appdb_scan.h, and an unreadable + * database is {"err":"..","apps":[]}. Returns 0 on success, non-zero only + * if `buf` is too small to hold even an error. */ +#define CONTENT_DB_APPS_PAGE_MAX 512 +int content_db_apps_page_json(unsigned offset, unsigned limit, char *buf, + size_t cap, size_t *written); + +/* Page one: content_db_apps_page_json(0, 0, ...). */ int content_db_apps_json(char *buf, size_t cap, size_t *written); /* diff --git a/payload/src/content_db.c b/payload/src/content_db.c index 8e3ad03b..ba7b6ed6 100644 --- a/payload/src/content_db.c +++ b/payload/src/content_db.c @@ -268,15 +268,45 @@ static void cdb_build_app_sql(const cdb_app_schema_t *sc, char *sql, sc->table, sc->col_title, sc->col_title); } -/* Row-level app list: SQL first, byte scan if the database will not open. - * content_db_apps_json() and activity.c's "recently played" both build on - * this so there is one schema-discovery implementation, not two. */ -int content_db_apps(content_db_app_t *out, int max, const char **source) { +/* The byte scanner's bounds. It reads at most CDB_SCAN_RAW_MAX of the file + * and recovers at most CDB_SCAN_ROWS_MAX titles; a database past either is + * reported as truncated, never as a complete list. */ +#define CDB_SCAN_RAW_MAX (1024u * 1024u) +#define CDB_SCAN_ROWS_MAX 4096 + +static void cdb_fill_row(content_db_app_t *r, const char *tid, + const char *name, int app_id) { + memset(r, 0, sizeof(*r)); + snprintf(r->title_id, sizeof(r->title_id), "%s", tid); + snprintf(r->name, sizeof(r->name), "%s", name ? name : ""); + r->app_id = app_id; +} + +/* + * Row-level app list, windowed: rows [offset, offset + max) of the title + * table, in title order, go to `out`. SQL first, byte scan if the database + * will not open. content_db_apps(), the paged JSON and activity.c's + * "recently played" all build on this, so there is one schema-discovery + * implementation, not two. + * + * `total` (optional) receives how many rows the table holds -- with the + * scanner, a lower bound that is at least offset + max + 1 when more rows + * follow the window. `capped` (optional) is set when rows exist that the + * reader could not reach (the scanner's bounds). + * + * Returns the number of rows written to `out`, or -1 if nothing could be + * read at all. + */ +static int cdb_apps_window(content_db_app_t *out, unsigned offset, int max, + unsigned *total, int *capped, const char **source) { + if (total) *total = 0; + if (capped) *capped = 0; if (!out || max <= 0) return -1; if (source) *source = "scan"; sqlite3 *db = NULL; const char *how = "sqlite"; + unsigned rows_seen = 0; int count = 0; if (cdb_open_ro(CONTENT_DB_APP, &db, &how) == SQLITE_OK && db) { @@ -286,23 +316,23 @@ int content_db_apps(content_db_app_t *out, int max, const char **source) { /* First candidate that actually yields rows wins: a console with * several users has several of these tables, and the ones * belonging to other users are empty. */ - for (int ci = 0; ci < ncand && count == 0; ci++) { + for (int ci = 0; ci < ncand && rows_seen == 0; ci++) { char sql[512]; cdb_build_app_sql(&cands[ci], sql, sizeof(sql)); sqlite3_stmt *st = NULL; if (sqlite3_prepare_v2(db, sql, -1, &st, NULL) != SQLITE_OK) continue; - while (count < max && sqlite3_step(st) == SQLITE_ROW) { + /* Every row is counted (so the caller knows whether more + * follow); only the window is copied. */ + while (sqlite3_step(st) == SQLITE_ROW) { const unsigned char *tid = sqlite3_column_text(st, 0); if (!tid) continue; + unsigned idx = rows_seen++; + if (idx < offset || count >= max) continue; const unsigned char *nm = sqlite3_column_text(st, 2); - memset(&out[count], 0, sizeof(out[count])); - snprintf(out[count].title_id, sizeof(out[count].title_id), - "%s", (const char *)tid); - snprintf(out[count].name, sizeof(out[count].name), "%s", - nm ? (const char *)nm : ""); - out[count].app_id = sqlite3_column_int(st, 1); + cdb_fill_row(&out[count], (const char *)tid, + (const char *)nm, sqlite3_column_int(st, 1)); count++; } sqlite3_finalize(st); @@ -314,49 +344,65 @@ int content_db_apps(content_db_app_t *out, int max, const char **source) { * console has no games -- and install verification reads an empty list * as a failed install. Fall through to the scan rather than assert a * confident zero. */ - if (count > 0) { + if (rows_seen > 0) { if (source) *source = how; + if (total) *total = rows_seen; return count; } int fd = open(CONTENT_DB_APP, O_RDONLY); if (fd < 0) return -1; - const size_t raw_cap = 1024 * 1024; - unsigned char *raw = (unsigned char *)malloc(raw_cap); + unsigned char *raw = (unsigned char *)malloc(CDB_SCAN_RAW_MAX); if (!raw) { close(fd); return -1; } - size_t total = 0; - while (total < raw_cap) { - ssize_t r = read(fd, raw + total, raw_cap - total); + size_t got = 0; + while (got < CDB_SCAN_RAW_MAX) { + ssize_t r = read(fd, raw + got, CDB_SCAN_RAW_MAX - got); if (r <= 0) break; - total += (size_t)r; + got += (size_t)r; + } + /* A file larger than the read window has rows the scan never sees. */ + int raw_capped = 0; + if (got == CDB_SCAN_RAW_MAX) { + unsigned char probe; + raw_capped = read(fd, &probe, 1) > 0; } close(fd); + /* One past the window, so "more rows follow" is known without + * holding every row. */ + unsigned long want = (unsigned long)offset + (unsigned long)max + 1ul; + int scan_max = want > CDB_SCAN_ROWS_MAX ? CDB_SCAN_ROWS_MAX : (int)want; appdb_entry_t *entries = - (appdb_entry_t *)malloc(sizeof(appdb_entry_t) * (size_t)max); + (appdb_entry_t *)malloc(sizeof(appdb_entry_t) * (size_t)scan_max); if (!entries) { free(raw); return -1; } - int scanned = appdb_scan_entries_ex(raw, total, entries, max, 1); + int scanned = appdb_scan_entries_ex(raw, got, entries, scan_max, 1); free(raw); if (scanned < 0) { free(entries); return -1; } - for (int i = 0; i < scanned; i++) { - memset(&out[i], 0, sizeof(out[i])); - snprintf(out[i].title_id, sizeof(out[i].title_id), "%s", - entries[i].title_id); - snprintf(out[i].name, sizeof(out[i].name), "%s", entries[i].name); - out[i].app_id = 0; + for (int i = (int)(offset < (unsigned)scanned ? offset : (unsigned)scanned); + i < scanned && count < max; i++) { + cdb_fill_row(&out[count], entries[i].title_id, entries[i].name, 0); + count++; } free(entries); + if (capped) + *capped = raw_capped || (scanned == CDB_SCAN_ROWS_MAX && + scan_max == CDB_SCAN_ROWS_MAX); + if (total) *total = (unsigned)scanned; if (source) *source = "scan"; - return scanned; + return count; +} + +int content_db_apps(content_db_app_t *out, int max, const char **source) { + return cdb_apps_window(out, 0, max, NULL, NULL, source); } int content_db_select_text(const char *db_path, const char *sql, int max_rows, @@ -395,16 +441,26 @@ int content_db_select_text(const char *db_path, const char *sql, int max_rows, return rows; } -int content_db_apps_json(char *buf, size_t cap, size_t *written) { +/* The longest tail a page can end with: + * ],"source":"sqlite-immutable","offset":4294967295,"next":4294967295, + * "more":false,"truncated":false} -- about 110 bytes. */ +#define CDB_PAGE_TAIL_RESERVE 160 + +int content_db_apps_page_json(unsigned offset, unsigned limit, char *buf, + size_t cap, size_t *written) { if (!buf || cap < 64) return -1; + if (limit == 0 || limit > CONTENT_DB_APPS_PAGE_MAX) + limit = CONTENT_DB_APPS_PAGE_MAX; - const int max_rows = 256; content_db_app_t *rows = - (content_db_app_t *)malloc(sizeof(content_db_app_t) * (size_t)max_rows); + (content_db_app_t *)malloc(sizeof(content_db_app_t) * (size_t)limit); if (!rows) return cdb_emit_apps_err(buf, cap, written, "out of memory"); const char *source = "scan"; - int count = content_db_apps(rows, max_rows, &source); + unsigned total = 0; + int capped = 0; + int count = cdb_apps_window(rows, offset, (int)limit, &total, &capped, + &source); if (count < 0) { free(rows); return cdb_emit_apps_err(buf, cap, written, "cannot read app.db"); @@ -424,20 +480,40 @@ int content_db_apps_json(char *buf, size_t cap, size_t *written) { "%s{\"title_id\":\"%s\",\"app_id\":%d," "\"name\":\"%s\"}", emitted ? "," : "", tid, rows[i].app_id, name); - if (more < 0 || (size_t)(n + more) + 40 >= cap) break; + /* Full: stop here and let `next` say where the following page + * starts. This used to close the JSON as if it were the whole + * list (#473). */ + if (more < 0 || (size_t)(n + more) + CDB_PAGE_TAIL_RESERVE >= cap) + break; n += more; emitted++; } free(rows); - int end2 = snprintf(buf + n, cap - (size_t)n, "],\"source\":\"%s\"}", - source); + unsigned next = offset + (unsigned)emitted; + /* Rows remain after this page: the table holds more than `next`. */ + int more_rows = total > next; + /* A row that cannot fit even on an empty page can never be delivered: + * say so and stop, rather than send the caller round in a loop. */ + int stuck = more_rows && emitted == 0 && count > 0; + int truncated = capped || stuck; + if (stuck) more_rows = 0; + + int end2 = snprintf(buf + n, cap - (size_t)n, + "],\"source\":\"%s\",\"offset\":%u,\"next\":%u," + "\"more\":%s,\"truncated\":%s}", + source, offset, next, more_rows ? "true" : "false", + truncated ? "true" : "false"); if (end2 < 0 || (size_t)(n + end2) >= cap) return -1; n += end2; if (written) *written = (size_t)n; return 0; } +int content_db_apps_json(char *buf, size_t cap, size_t *written) { + return content_db_apps_page_json(0, 0, buf, cap, written); +} + /* ── appinfo.db ──────────────────────────────────────────────────────── */ int content_db_appinfo_json(const char *title_id, const char *keys, diff --git a/payload/src/mgmt_table.def b/payload/src/mgmt_table.def index c21166fc..2b7e9766 100644 --- a/payload/src/mgmt_table.def +++ b/payload/src/mgmt_table.def @@ -57,7 +57,7 @@ MGMT_H0(AVA1_METHOD_APP_LAUNCH_BROWSER, MGMT_FRAME_APP_LAUNCH_BROWSER, MGMT_FRAM MGMT_H1(AVA1_METHOD_APP_LIFECYCLE, MGMT_FRAME_APP_LIFECYCLE, MGMT_FRAME_APP_LIFECYCLE_ACK, MGMT_SONY, handle_app_lifecycle, mgmt_call_text) MGMT_H1(AVA1_METHOD_APP_INFO_QUERY, MGMT_FRAME_APPINFO_QUERY, MGMT_FRAME_APPINFO_QUERY_ACK, 0, handle_appinfo_query, mgmt_call_text) MGMT_H1(AVA1_METHOD_APP_INFO_SET, MGMT_FRAME_APPINFO_SET, MGMT_FRAME_APPINFO_SET_ACK, MGMT_SONY, handle_appinfo_set, mgmt_call_text) -MGMT_H0(AVA1_METHOD_APP_DB_QUERY, MGMT_FRAME_APPDB_QUERY, MGMT_FRAME_APPDB_QUERY_ACK, 0, handle_appdb_query, mgmt_call_text) +MGMT_H1(AVA1_METHOD_APP_DB_QUERY, MGMT_FRAME_APPDB_QUERY, MGMT_FRAME_APPDB_QUERY_ACK, 0, handle_appdb_query, mgmt_call_text) MGMT_H0(AVA1_METHOD_PROC_FOCUS, MGMT_FRAME_FOCUS_PROBE, MGMT_FRAME_FOCUS_PROBE_ACK, 0, handle_focus_probe, mgmt_call_text) MGMT_H1(AVA1_METHOD_PROC_LIST, MGMT_FRAME_PROC_LIST, MGMT_FRAME_PROC_LIST_ACK, 0, handle_proc_list, mgmt_call_text) MGMT_H1(AVA1_METHOD_PROC_KILL, MGMT_FRAME_PROCESS_KILL, MGMT_FRAME_PROCESS_KILL_ACK, 0, handle_process_kill, mgmt_call_text) diff --git a/payload/src/runtime.c b/payload/src/runtime.c index 285c0704..5f5709bb 100644 --- a/payload/src/runtime.c +++ b/payload/src/runtime.c @@ -6991,10 +6991,32 @@ static int handle_shell_exec(runtime_state_t *state, const char *body, * The scan survives inside content_db_apps_json as the fallback for a * database that will not open at all (locked by the shell, or damaged), * which is a genuine runtime condition rather than a permanent one. + * + * Paged (#473): the request may carry {"offset":N,"limit":M}; the reply + * says where the next page starts ("next") and whether one follows + * ("more"), see content_db_apps_page_json. An empty request -- what an + * older engine sends -- is page one. */ -static int handle_appdb_query(runtime_state_t *state) { +static int handle_appdb_query(runtime_state_t *state, const char *body, + uint64_t body_len) { if (!state) return -1; + uint64_t offset = 0, limit = 0; + if (body && body_len > 0) { + /* mgmt_json_u64 reads a C string; the body is not terminated. */ + char req[256]; + size_t n = body_len < sizeof(req) - 1 ? (size_t)body_len + : sizeof(req) - 1; + memcpy(req, body, n); + req[n] = '\0'; + if (mgmt_json_u64(req, "offset", &offset) < 0 || + mgmt_json_u64(req, "limit", &limit) < 0 || offset > UINT32_MAX || + limit > UINT32_MAX) { + const char *err = "{\"err\":\"bad_window\",\"apps\":[]}"; + return mgmt_reply(MGMT_FRAME_APPDB_QUERY_ACK, err, strlen(err)); + } + } + const size_t cap = 64 * 1024; char *resp = (char *)malloc(cap); if (!resp) { @@ -7004,7 +7026,9 @@ static int handle_appdb_query(runtime_state_t *state) { size_t written = 0; int rc; - if (content_db_apps_json(resp, cap, &written) != 0 || written == 0) { + if (content_db_apps_page_json((unsigned)offset, (unsigned)limit, resp, cap, + &written) != 0 || + written == 0) { const char *err = "{\"err\":\"appdb_unreadable\",\"apps\":[]}"; rc = mgmt_reply(MGMT_FRAME_APPDB_QUERY_ACK, err, strlen(err)); } else { diff --git a/payload/tests/content_db_selftest.c b/payload/tests/content_db_selftest.c index 1a1cbbb7..0ec3649d 100644 --- a/payload/tests/content_db_selftest.c +++ b/payload/tests/content_db_selftest.c @@ -129,6 +129,115 @@ static int contains(const char *hay, const char *needle) { return strstr(hay, needle) != NULL; } + +/* ── app.db pagination fixtures (#473) ──────────────────────────────── + * + * A big app table: `n` rows CUSA00000.. in title order, plus (when + * `late` is set) one late-sorting row ZZZZ99999 -- the shape of a title + * that used to fall off the end of the fixed 256-row window. */ +static void make_big_app_db(const char *path, int n, int late, + int long_names) { + unlink(path); + sqlite3 *db = NULL; + assert(sqlite3_open(path, &db) == SQLITE_OK); + exec_or_die(db, + "CREATE TABLE tbl_info_1000000 (" + " titleId TEXT, appId INTEGER, titleName TEXT);"); + exec_or_die(db, "BEGIN"); + sqlite3_stmt *st = NULL; + assert(sqlite3_prepare_v2(db, + "INSERT INTO tbl_info_1000000 VALUES (?,?,?)", + -1, &st, NULL) == SQLITE_OK); + for (int i = 0; i < n; i++) { + char tid[16], name[300]; + snprintf(tid, sizeof tid, "CUSA%05d", i); + if (long_names) { + /* 200+ characters with quotes, backslashes and a control + * byte: every one of them grows when escaped. */ + int o = snprintf(name, sizeof name, "Game \"%d\" \\ \t", i); + while (o < 240) name[o++] = 'x'; + name[o] = '\0'; + } else { + snprintf(name, sizeof name, "Game %d", i); + } + sqlite3_bind_text(st, 1, tid, -1, SQLITE_TRANSIENT); + sqlite3_bind_int(st, 2, 1000 + i); + sqlite3_bind_text(st, 3, name, -1, SQLITE_TRANSIENT); + assert(sqlite3_step(st) == SQLITE_DONE); + sqlite3_reset(st); + } + sqlite3_finalize(st); + if (late) + exec_or_die(db, "INSERT INTO tbl_info_1000000 VALUES" + " ('ZZZZ99999', 9, 'Late Title')"); + exec_or_die(db, "COMMIT"); + sqlite3_close(db); +} + +/* The unsigned number after "key": in `json`, or -1. */ +static long json_num(const char *json, const char *key) { + char needle[40]; + snprintf(needle, sizeof needle, "\"%s\":", key); + const char *p = strstr(json, needle); + if (!p) return -1; + return strtol(p + strlen(needle), NULL, 10); +} + +static int json_true(const char *json, const char *key) { + char needle[40]; + snprintf(needle, sizeof needle, "\"%s\":true", key); + return strstr(json, needle) != NULL; +} + +/* Count the title ids in one page and check each is the next expected + * one, so a skipped or repeated row fails, not just a short count. */ +static int page_ids(const char *json, int *seen_upto, int *saw_late) { + int n = 0; + const char *p = json; + while ((p = strstr(p, "\"title_id\":\"")) != NULL) { + p += strlen("\"title_id\":\""); + if (strncmp(p, "ZZZZ99999", 9) == 0) { + *saw_late = 1; + } else { + char want[16]; + snprintf(want, sizeof want, "CUSA%05d", *seen_upto); + assert(strncmp(p, want, 9) == 0); + (*seen_upto)++; + } + n++; + } + return n; +} + +/* Page through the whole table the way the engine does; returns how many + * rows arrived. Fails if a page claims more but makes no progress. */ +static int page_all(char *buf, size_t cap, unsigned limit, int *saw_late, + int *pages, int *truncated) { + int seen_upto = 0, total = 0; + unsigned offset = 0; + *saw_late = 0; + *pages = 0; + *truncated = 0; + for (;;) { + size_t written = 0; + assert(content_db_apps_page_json(offset, limit, buf, cap, &written) == + 0); + assert(written > 0 && written < cap); + assert(buf[written - 1] == '}'); + (*pages)++; + int got = page_ids(buf, &seen_upto, saw_late); + total += got; + assert(json_num(buf, "offset") == (long)offset); + assert(json_num(buf, "next") == (long)(offset + (unsigned)got)); + if (json_true(buf, "truncated")) *truncated = 1; + if (!json_true(buf, "more")) break; + assert(got > 0); + offset += (unsigned)got; + assert(*pages < 10000); + } + return total; +} + int main(void) { char buf[64 * 1024]; size_t written = 0; @@ -278,6 +387,134 @@ int main(void) { printf("✓ non-SELECT refused; table intact\n"); } + + /* ── app.db pagination (#473) ───────────────────────────────────── */ + { + static char pbuf[64 * 1024]; + int late, pages, trunc; + const int sizes[] = {255, 256, 257, 1000, 1500}; + for (size_t i = 0; i < sizeof sizes / sizeof *sizes; i++) { + make_big_app_db(CONTENT_DB_APP, sizes[i], 1, 0); + int got = page_all(pbuf, sizeof pbuf, 0, &late, &pages, &trunc); + assert(got == sizes[i] + 1); + assert(late); + assert(!trunc); + printf("✓ %d rows + a late-sorting title: all %d arrive in %d page(s)\n", + sizes[i], got, pages); + } + + /* The no-argument call (what an older engine sends) is page one: + * never a silent cap that looks complete. */ + make_big_app_db(CONTENT_DB_APP, 1000, 1, 0); + assert(content_db_apps_json(pbuf, sizeof pbuf, &written) == 0); + assert(json_num(pbuf, "offset") == 0); + assert(json_true(pbuf, "more")); + assert(!json_true(pbuf, "truncated")); + assert(json_num(pbuf, "next") > 256); + printf("✓ the unpaged call says more rows follow (next=%ld)\n", + json_num(pbuf, "next")); + + /* An earlier-sorting row added later shifts the window but does + * not push the late title out. */ + { + sqlite3 *db = NULL; + assert(sqlite3_open(CONTENT_DB_APP, &db) == SQLITE_OK); + exec_or_die(db, "INSERT INTO tbl_info_1000000 VALUES" + " ('AAAA00001', 1, 'Early Title')"); + sqlite3_close(db); + int seen = 0, saw_late = 0, saw_early = 0; + unsigned off = 0; + for (;;) { + assert(content_db_apps_page_json(off, 0, pbuf, sizeof pbuf, + &written) == 0); + const char *p = pbuf; + int got = 0; + while ((p = strstr(p, "\"title_id\":\"")) != NULL) { + p += 12; + if (!strncmp(p, "ZZZZ99999", 9)) saw_late = 1; + if (!strncmp(p, "AAAA00001", 9)) saw_early = 1; + got++; + } + seen += got; + if (!json_true(pbuf, "more")) break; + off += (unsigned)got; + } + assert(seen == 1002 && saw_late && saw_early); + printf("✓ an earlier-sorting row does not push the late title out\n"); + } + + /* An explicit limit is honoured. */ + make_big_app_db(CONTENT_DB_APP, 300, 0, 0); + assert(content_db_apps_page_json(100, 50, pbuf, sizeof pbuf, + &written) == 0); + assert(contains(pbuf, "\"title_id\":\"CUSA00100\"")); + assert(contains(pbuf, "\"title_id\":\"CUSA00149\"")); + assert(!contains(pbuf, "\"title_id\":\"CUSA00150\"")); + assert(json_num(pbuf, "next") == 150); + assert(json_true(pbuf, "more")); + assert(content_db_apps_page_json(300, 50, pbuf, sizeof pbuf, + &written) == 0); + assert(contains(pbuf, "\"apps\":[]")); + assert(!json_true(pbuf, "more")); + assert(json_num(pbuf, "next") == 300); + printf("✓ offset/limit select a window; past the end is empty and final\n"); + + /* A small buffer stops early and says where -- a continuation, + * not a complete-looking list. */ + make_big_app_db(CONTENT_DB_APP, 300, 1, 1); + { + static char small[2048]; + assert(content_db_apps_page_json(0, 0, small, sizeof small, + &written) == 0); + long next = json_num(small, "next"); + assert(next > 0 && next < 300); + assert(json_true(small, "more")); + assert(!json_true(small, "truncated")); + assert(contains(small, "Game \\\"0\\\" \\\\ \\t")); + int got = page_all(small, sizeof small, 0, &late, &pages, &trunc); + assert(got == 301 && late && !trunc); + printf("✓ a 2 KiB buffer pages long escaped names: 301 rows in %d pages\n", + pages); + } + + /* A buffer that cannot hold even one row reports truncation and + * stops, rather than looping or claiming an empty success. */ + { + static char tiny[160]; + assert(content_db_apps_page_json(0, 0, tiny, sizeof tiny, + &written) == 0); + assert(contains(tiny, "\"apps\":[]")); + assert(json_true(tiny, "truncated")); + assert(!json_true(tiny, "more")); + printf("✓ a buffer too small for one row is reported as truncated\n"); + } + + /* The row API is no longer clipped at 256 either. */ + make_big_app_db(CONTENT_DB_APP, 600, 0, 0); + { + content_db_app_t *big = + (content_db_app_t *)malloc(sizeof(content_db_app_t) * 700); + assert(big); + assert(content_db_apps(big, 700, NULL) == 600); + assert(strcmp(big[599].title_id, "CUSA00599") == 0); + free(big); + } + printf("✓ row API returns all 600 rows\n"); + + /* A query error is still the err envelope, and says it is final. */ + { + FILE *f = fopen(CONTENT_DB_APP, "wb"); + assert(f); + fputs("not a database", f); + fclose(f); + assert(content_db_apps_page_json(0, 0, pbuf, sizeof pbuf, + &written) == 0); + assert(contains(pbuf, "\"err\":") && contains(pbuf, "\"apps\":[]")); + assert(!json_true(pbuf, "more")); + printf("✓ unreadable app.db on a paged call still answers err+apps\n"); + } + } + /* ── home-screen icon hide / unhide (#407) ─────────────────────── */ { make_icon_db(CONTENT_DB_APP); diff --git a/protocol/ava1/MGMT_METHODS.md b/protocol/ava1/MGMT_METHODS.md index 12887447..0c1afcf4 100644 --- a/protocol/ava1/MGMT_METHODS.md +++ b/protocol/ava1/MGMT_METHODS.md @@ -59,7 +59,7 @@ ones (`ava1-ctest/tests/mgmt*.rs`, `ps5upload-ava1/tests/mgmt*.rs`). Methods 119 | ProcModules 114 | 61 `proc.modules` | `MgmtText` | `src/runtime.c` `handle_proc_modules` | `ps5upload-core/src/diagnostics.rs` | | | ShellExec 116 | 87 `shell.exec` | `MgmtText` | `src/runtime.c` `handle_shell_exec` | `ps5upload-core/src/diagnostics.rs` | | | Crc32File 118 | 20 `job.run (op CRC32)` | `JobRun` -> `Status` (ext `state`, `result`, `code`) | `src/fs_jobs.c` `op_crc32` | `ps5upload-core/src/diagnostics.rs` | `ava1-ctest/tests/job_run.rs` `crc32_of_a_large_file_is_a_job_that_can_be_cancelled`; engine `mgmt_job.rs` `recursive_chmod_hash_crc32_fsck_and_backup_run_as_jobs_with_their_reply_shapes` | -| AppDbQuery 120 | 56 `app.db_query` | `MgmtText` | `src/runtime.c` `handle_appdb_query` | `ps5upload-core/src/diagnostics.rs` | | +| AppDbQuery 120 | 56 `app.db_query` | `MgmtText`; request `{"offset":N,"limit":M}` (both optional, empty = page one), reply adds `offset`, `next`, `more`, `truncated` (#473) | `src/runtime.c` `handle_appdb_query` | `ps5upload-core/src/diagnostics.rs` (pages until `more` is false) | | | NetSpeedTest 122 | 11 `net.speedtest` | `MgmtText` | `src/runtime.c` `handle_net_speed_test` | `ps5upload-core/src/diagnostics.rs` | | | PkgDirectMount 124 | 42 `fs.mount_pkg` | `MgmtText` | `src/runtime.c` `handle_pkg_direct_mount` | `ps5upload-core/src/diagnostics.rs` | host-tested | | UfsFsck 126 | 20 `job.run (op FSCK)` | `JobRun` -> `Status` (ext `state`, `result`, `code`) | `src/runtime.c` `handle_ufs_fsck` | `ps5upload-core/src/diagnostics.rs` | `ava1-ctest/tests/job_run.rs` `fsck_keeps_its_ok_false_body_as_the_answer_and_a_frame_error_is_a_failure`; engine `mgmt_job.rs` (same shapes test) |