diff --git a/CHANGELOG.md b/CHANGELOG.md index 59dd48a1..d48af188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,9 @@ What's new in ps5upload, written for humans. only up to its first 256 entries, and a long list could be cut shorter still, with nothing saying so. On a console with many entries, a game installed by a homebrew installer (no home-screen metadata folder) that sorted past that point was missing from Games. The app now - reads the whole database page by page, also lists any game it registered whose folder is on - the console, and logs a warning when a list could not be read in full (#473). + reads the whole database page by page. If the database still can't be read in full, games + whose folder and registration are on the console are listed anyway, and the app logs a warning + that the list may be incomplete (#473). ## 7.0.34 diff --git a/FAQ.md b/FAQ.md index 10f182ab..7bf55462 100644 --- a/FAQ.md +++ b/FAQ.md @@ -164,8 +164,8 @@ zero means no cap. The pre-6.0 name of this variable still works for one release **Q: Can I run the engine on a different machine (self-hosted)?** Yes. Run it on a server or NAS and point the app at it in **Settings → Engine URL**. For a -step-by-step guide that ends with the app open on a phone, see "Use PS5Upload from your phone or -another computer" below. Images: +step-by-step guide that ends with the app open on a phone, see "How do I use PS5Upload from my phone +or another computer?" below. Images: `ghcr.io/phantomptr/ps5upload-engine` (engine only) and `ghcr.io/phantomptr/ps5upload-engine-webui` (full web UI), tagged `:latest` or `:`. @@ -199,8 +199,8 @@ in the compose file to the folder's owner (`ls -ln` shows the numbers), or run **Q: Can I use ps5upload from a web browser (no desktop app)?** Yes. The `webui` image (above) serves the full app over HTTP; open `http://:19113` from an -allowed IP. Same security rules: no password, trusted LAN only. The next question walks through -it from start to finish. +allowed IP. Same security rules: no password, trusted LAN only. "How do I use PS5Upload from my +phone or another computer?" below walks through it from start to finish. - **Upload works**, but the file picker browses the **engine's** disk, not your browser machine's. Mount your games into the container (`-v /host/games:/pkgs:ro`) and set @@ -212,7 +212,7 @@ it from start to finish. report. Everything that works on the PS5 itself is the same as on desktop. - The docker image can be built using `PS5UPLOAD_BASE_URL` to change the root path of the app. -**Q: Use PS5Upload from your phone or another computer** +**Q: How do I use PS5Upload from my phone or another computer?** Run PS5Upload on one machine, for example a Mac tucked away next to the router, and use the whole app from a browser anywhere on your home network, including Safari on an iPhone or iPad. This uses the web UI Docker image, not the desktop app. diff --git a/engine/crates/ps5upload-core/src/diagnostics.rs b/engine/crates/ps5upload-core/src/diagnostics.rs index 368a8d1b..9c8f8417 100644 --- a/engine/crates/ps5upload-core/src/diagnostics.rs +++ b/engine/crates/ps5upload-core/src/diagnostics.rs @@ -256,6 +256,10 @@ pub struct AppDbList { pub next: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub more: Option, + /// The table's row count when the page was read (SQL reader only). A change between pages + /// means rows moved under the walk, so one may have been skipped: the list is truncated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total: Option, /// True when app.db holds rows this list does not: the helper could not deliver them, a /// page failed (`err` says why), the page bound was reached, or an older helper answered /// a full 256 rows (its old cap, which it could not report). A truncated list is never @@ -299,7 +303,12 @@ pub fn appdb_query(addr: &str) -> Result { let more = page.more.unwrap_or(false); let next = page.next.unwrap_or(offset + page.apps.len() as u64); let got = page.apps.len(); + let total_moved = match (list.as_ref().and_then(|l| l.total), page.total) { + (Some(a), Some(b)) => a != b, + _ => false, + }; let page_truncated = page.truncated + || total_moved || (!paged && got >= APPDB_LEGACY_CAP) // A page that claims more but does not move forward would loop. || (more && next <= offset); @@ -322,6 +331,7 @@ pub fn appdb_query(addr: &str) -> Result { l.offset = None; l.next = None; l.more = None; + l.total = None; return Ok(list.expect("set above")); } offset = next; @@ -332,6 +342,7 @@ pub fn appdb_query(addr: &str) -> Result { l.offset = None; l.next = None; l.more = None; + l.total = None; Ok(l) } diff --git a/engine/crates/ps5upload-core/src/pkg_install.rs b/engine/crates/ps5upload-core/src/pkg_install.rs index 4b4a07df..3e1b26ac 100644 --- a/engine/crates/ps5upload-core/src/pkg_install.rs +++ b/engine/crates/ps5upload-core/src/pkg_install.rs @@ -531,6 +531,7 @@ mod appdb_list_has_title_tests { offset: None, next: None, more: None, + total: None, truncated, } } diff --git a/engine/crates/ps5upload-core/tests/appdb_paging.rs b/engine/crates/ps5upload-core/tests/appdb_paging.rs index c164b8dd..bba10c3f 100644 --- a/engine/crates/ps5upload-core/tests/appdb_paging.rs +++ b/engine/crates/ps5upload-core/tests/appdb_paging.rs @@ -73,6 +73,7 @@ fn paged_helper(rows: usize, page: usize) -> impl Fn(&Value) -> Result( registered: impl IntoIterator, app_dirs: &std::collections::HashSet, seen: &std::collections::HashSet, + appdb_whole: bool, ) -> Vec { + if appdb_whole { + return Vec::new(); + } let mut out: Vec = registered .into_iter() .map(str::trim) - .filter(|tid| looks_like_title_id(tid) && app_dirs.contains(*tid) && !seen.contains(*tid)) + .filter(|tid| { + looks_like_title_id(tid) + && !tid.starts_with("NPXS") + && app_dirs.contains(*tid) + && !seen.contains(*tid) + }) .map(str::to_string) .collect(); out.sort(); @@ -5068,6 +5082,14 @@ fn registered_only_additions<'a>( out } +/// True when `/user/app//sce_sys` holds a param.json or param.sfo: the folder is a title, +/// not the leftovers of a failed install. +fn app_folder_has_param(addr: &str, title_id: &str) -> bool { + ["param.json", "param.sfo"] + .iter() + .any(|f| fs_read(addr, &format!("/user/app/{title_id}/sce_sys/{f}"), 0, 16).is_ok()) +} + /// GET /api/ps5/apps/installed?addr=IP:MGMT_PORT async fn ps5_apps_installed( State(state): State, @@ -5184,6 +5206,9 @@ async fn ps5_apps_installed( complete = complete && crate::console_snapshot::app_folders_whole(internal_listing, app_dirs.len()); if !app_dirs.is_empty() { + // Whether app.db answered in full: only then is a folder it does not list + // known not to be an installed title. + let mut appdb_whole = true; let rows: Option> = match appdb.get() { Ok(list) => { // `err` with no rows: the helper could not read app.db at all. @@ -5198,6 +5223,7 @@ async fn ps5_apps_installed( .unwrap_or_default() ); complete = false; + appdb_whole = false; } Some( list.apps @@ -5209,6 +5235,7 @@ async fn ps5_apps_installed( Err(e) => { crate::log_warn!("apps/installed: app.db query failed: {e}"); complete = false; + appdb_whole = false; None } }; @@ -5235,24 +5262,30 @@ async fn ps5_apps_installed( }); } } - // Registered titles whose folder is there but that neither listing above - // produced (#473): never let a registration vanish because app.db did not - // answer for it. + // When app.db could not be read in full, a title it did not get to still + // has its folder and its registration (#473): list it if the folder holds a + // title (our mount.lnk source, or a param file), never a bare leftover folder. let seen: std::collections::HashSet = titles.iter().map(|t| t.title_id.clone()).collect(); - for tid in - registered_only_additions(reg_map.keys().map(String::as_str), &app_dirs, &seen) - { + for tid in registered_only_additions( + reg_map.keys().map(String::as_str), + &app_dirs, + &seen, + appdb_whole, + ) { let Some(reg) = reg_map.get(&tid) else { continue; }; + if reg.src.trim().is_empty() && !app_folder_has_param(&addr, &tid) { + continue; + } let name = if reg.title_name.trim().is_empty() || reg.title_name == tid { appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone()) } else { reg.title_name.clone() }; titles.push(InstalledApp { - system: tid.starts_with("NPXS"), + system: false, title_id: tid, title_name: name, origin: "registered".to_string(), @@ -12672,26 +12705,43 @@ mod appdb_installed_additions_tests { ); } - /// #473: a registered title with its app folder but no appmeta and no - /// app.db row (a failed or partial app.db read) is still listed. + /// #473: when app.db could not be read in full, a registered title with its + /// app folder but no appmeta and no app.db row is a candidate. #[test] - fn registered_titles_with_a_folder_are_reconciled() { + fn an_incomplete_appdb_read_reconciles_registered_titles_with_a_folder() { let got = registered_only_additions( [ "PPSA00002", "ZZZZ99999", "CUSA00001", "PPSA00003", + "NPXS40056", "not-an-id", ], - &set(&["ZZZZ99999", "CUSA00001", "PPSA00002", "not-an-id"]), + &set(&[ + "ZZZZ99999", + "CUSA00001", + "PPSA00002", + "NPXS40056", + "not-an-id", + ]), // already listed by the appmeta scan or app.db &set(&["CUSA00001"]), + false, ); - // PPSA00003 has no folder: a stale registration is not an installed title. + // PPSA00003 has no folder; NPXS is a system id, never added this way. assert_eq!(got, vec!["PPSA00002".to_string(), "ZZZZ99999".to_string()]); } + /// A complete app.db read that does not list a folder means the folder is not an + /// installed title (a failed install, a partial uninstall): it is not added. + #[test] + fn a_complete_appdb_read_adds_no_folder_it_does_not_list() { + let got = + registered_only_additions(["PPSA00002"], &set(&["PPSA00002"]), &HashSet::new(), true); + assert!(got.is_empty()); + } + #[test] fn skips_titles_the_appmeta_scan_already_found() { let rows = vec![("PLDM00001".to_string(), "Payload Manager".to_string())]; diff --git a/payload/include/content_db.h b/payload/include/content_db.h index b0c3515a..cfc92b52 100644 --- a/payload/include/content_db.h +++ b/payload/include/content_db.h @@ -83,13 +83,17 @@ int content_db_select_text(const char *db_path, const char *sql, int max_rows, * * Emits {"apps":[{"title_id":"..","app_id":N,"name":".."}, ...], * "source":"sqlite"|"sqlite-immutable"|"scan", - * "offset":O,"next":N,"more":bool,"truncated":bool} + * "offset":O,"next":N,"more":bool,"truncated":bool,"total":T} * + * `total` is the table's row count, sent only from SQL (the scanner's is a + * lower bound); a caller that sees it change between pages knows rows + * moved under it. * `next` is the row the following page starts at; `more` is true while * rows remain after it. A page that fills `buf` before `limit` rows stops * early with `more` set, so a small buffer pages rather than silently * dropping the rest. `truncated` is true only when rows exist that no - * page can deliver: one row larger than `buf`, or the byte scanner (the + * page can deliver: one row larger than `buf`, a SQL walk that stopped + * before the end (SQLITE_BUSY and the like), or the byte scanner (the * fallback for a database that will not open) hitting its own bounds. * * `limit` 0 (or above CONTENT_DB_APPS_PAGE_MAX) means diff --git a/payload/src/content_db.c b/payload/src/content_db.c index ba7b6ed6..6cd98121 100644 --- a/payload/src/content_db.c +++ b/payload/src/content_db.c @@ -325,7 +325,8 @@ static int cdb_apps_window(content_db_app_t *out, unsigned offset, int max, continue; /* Every row is counted (so the caller knows whether more * follow); only the window is copied. */ - while (sqlite3_step(st) == SQLITE_ROW) { + int step; + while ((step = sqlite3_step(st)) == SQLITE_ROW) { const unsigned char *tid = sqlite3_column_text(st, 0); if (!tid) continue; unsigned idx = rows_seen++; @@ -335,6 +336,9 @@ static int cdb_apps_window(content_db_app_t *out, unsigned offset, int max, (const char *)nm, sqlite3_column_int(st, 1)); count++; } + /* A walk that ended on anything but DONE (BUSY under the + * shell's write, a damaged page) did not see every row. */ + if (rows_seen > 0 && step != SQLITE_DONE && capped) *capped = 1; sqlite3_finalize(st); } sqlite3_close(db); @@ -499,11 +503,17 @@ int content_db_apps_page_json(unsigned offset, unsigned limit, char *buf, int truncated = capped || stuck; if (stuck) more_rows = 0; + /* `total` is exact only from SQL: the scanner's is a lower bound that + * grows with the window, so it is left out rather than mislead a + * caller comparing pages. */ + char total_field[32] = ""; + if (strcmp(source, "scan") != 0) + snprintf(total_field, sizeof(total_field), ",\"total\":%u", total); int end2 = snprintf(buf + n, cap - (size_t)n, "],\"source\":\"%s\",\"offset\":%u,\"next\":%u," - "\"more\":%s,\"truncated\":%s}", + "\"more\":%s,\"truncated\":%s%s}", source, offset, next, more_rows ? "true" : "false", - truncated ? "true" : "false"); + truncated ? "true" : "false", total_field); if (end2 < 0 || (size_t)(n + end2) >= cap) return -1; n += end2; if (written) *written = (size_t)n; diff --git a/payload/tests/content_db_selftest.c b/payload/tests/content_db_selftest.c index 0ec3649d..50299d6f 100644 --- a/payload/tests/content_db_selftest.c +++ b/payload/tests/content_db_selftest.c @@ -411,6 +411,7 @@ int main(void) { assert(json_true(pbuf, "more")); assert(!json_true(pbuf, "truncated")); assert(json_num(pbuf, "next") > 256); + assert(json_num(pbuf, "total") == 1001); printf("✓ the unpaged call says more rows follow (next=%ld)\n", json_num(pbuf, "next"));