Review fixes for #473: reconcile folders only on a partial app.db read

- Registered-only titles are added only when app.db could not be read in
  full; the helper's registered list is every /user/app folder, so with a
  complete read a folder app.db does not list is a leftover, not a game.
  NPXS ids are excluded, and a candidate needs a mount.lnk source or a
  param.json/param.sfo in its sce_sys.
- app.db pages carry the table's row count (SQL reader only); the engine
  marks the list truncated when it changes between pages.
- A SQL walk that ends on anything but SQLITE_DONE reports truncated.
- FAQ: the phone guide is phrased as a question; references updated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RxCGb8Asp1A1AQYiv1kbgS

CHANGELOG.md

@@ -10,8 +10,9 @@ What's new in ps5upload, written for humans.
   only up to its first 256 entries, and a long list could be cut shorter still, with nothing
   saying so. On a console with many entries, a game installed by a homebrew installer (no
   home-screen metadata folder) that sorted past that point was missing from Games. The app now
-  reads the whole database page by page, also lists any game it registered whose folder is on
-  the console, and logs a warning when a list could not be read in full (#473).
+  reads the whole database page by page. If the database still can't be read in full, games
+  whose folder and registration are on the console are listed anyway, and the app logs a warning
+  that the list may be incomplete (#473).
 
 ## 7.0.34
 

FAQ.md

@@ -164,8 +164,8 @@ zero means no cap. The pre-6.0 name of this variable still works for one release
 
 **Q: Can I run the engine on a different machine (self-hosted)?**
 Yes. Run it on a server or NAS and point the app at it in **Settings → Engine URL**. For a
-step-by-step guide that ends with the app open on a phone, see "Use PS5Upload from your phone or
-another computer" below. Images:
+step-by-step guide that ends with the app open on a phone, see "How do I use PS5Upload from my phone
+or another computer?" below. Images:
 `ghcr.io/phantomptr/ps5upload-engine` (engine only) and `ghcr.io/phantomptr/ps5upload-engine-webui`
 (full web UI), tagged `:latest` or `:<version>`.
 
@@ -199,8 +199,8 @@ in the compose file to the folder's owner (`ls -ln` shows the numbers), or run
 
 **Q: Can I use ps5upload from a web browser (no desktop app)?**
 Yes. The `webui` image (above) serves the full app over HTTP; open `http://<host>:19113` from an
-allowed IP. Same security rules: no password, trusted LAN only. The next question walks through
-it from start to finish.
+allowed IP. Same security rules: no password, trusted LAN only. "How do I use PS5Upload from my
+phone or another computer?" below walks through it from start to finish.
 
 - **Upload works**, but the file picker browses the **engine's** disk, not your browser machine's.
   Mount your games into the container (`-v /host/games:/pkgs:ro`) and set
@@ -212,7 +212,7 @@ it from start to finish.
   report. Everything that works on the PS5 itself is the same as on desktop.
 - The docker image can be built using `PS5UPLOAD_BASE_URL` to change the root path of the app.
 
-**Q: Use PS5Upload from your phone or another computer**
+**Q: How do I use PS5Upload from my phone or another computer?**
 Run PS5Upload on one machine, for example a Mac tucked away next to the router, and use the whole
 app from a browser anywhere on your home network, including Safari on an iPhone or iPad. This
 uses the web UI Docker image, not the desktop app.

engine/crates/ps5upload-core/src/diagnostics.rs

@@ -256,6 +256,10 @@ pub struct AppDbList {
     pub next: Option<u64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub more: Option<bool>,
+    /// The table's row count when the page was read (SQL reader only). A change between pages
+    /// means rows moved under the walk, so one may have been skipped: the list is truncated.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub total: Option<u64>,
     /// True when app.db holds rows this list does not: the helper could not deliver them, a
     /// page failed (`err` says why), the page bound was reached, or an older helper answered
     /// a full 256 rows (its old cap, which it could not report). A truncated list is never
@@ -299,7 +303,12 @@ pub fn appdb_query(addr: &str) -> Result<AppDbList> {
         let more = page.more.unwrap_or(false);
         let next = page.next.unwrap_or(offset + page.apps.len() as u64);
         let got = page.apps.len();
+        let total_moved = match (list.as_ref().and_then(|l| l.total), page.total) {
+            (Some(a), Some(b)) => a != b,
+            _ => false,
+        };
         let page_truncated = page.truncated
+            || total_moved
             || (!paged && got >= APPDB_LEGACY_CAP)
             // A page that claims more but does not move forward would loop.
             || (more && next <= offset);
@@ -322,6 +331,7 @@ pub fn appdb_query(addr: &str) -> Result<AppDbList> {
             l.offset = None;
             l.next = None;
             l.more = None;
+            l.total = None;
             return Ok(list.expect("set above"));
         }
         offset = next;
@@ -332,6 +342,7 @@ pub fn appdb_query(addr: &str) -> Result<AppDbList> {
     l.offset = None;
     l.next = None;
     l.more = None;
+    l.total = None;
     Ok(l)
 }
 

engine/crates/ps5upload-core/src/pkg_install.rs

@@ -531,6 +531,7 @@ mod appdb_list_has_title_tests {
             offset: None,
             next: None,
             more: None,
+            total: None,
             truncated,
         }
     }

engine/crates/ps5upload-core/tests/appdb_paging.rs

@@ -73,6 +73,7 @@ fn paged_helper(rows: usize, page: usize) -> impl Fn(&Value) -> Result<Option<Ve
                 "next": end,
                 "more": end < all.len(),
                 "truncated": false,
+                "total": all.len(),
             }))
             .unwrap(),
         ))
@@ -208,3 +209,24 @@ fn errors_keep_their_meaning() {
     assert!(d.truncated);
     assert!(d.err.as_deref().unwrap_or("").contains("connection reset"));
 }
+
+#[test]
+fn a_row_count_that_changes_between_pages_marks_the_list_truncated() {
+    // A row deleted between pages shifts the window: the first row of page two was skipped.
+    let (_t, _g) = fake(|req| {
+        let off = req["offset"].as_u64().unwrap_or(0);
+        Ok(Some(
+            serde_json::to_vec(&if off == 0 {
+                json!({"apps":[{"title_id":"CUSA00001","app_id":1,"name":"a"}],
+                       "offset":0,"next":1,"more":true,"truncated":false,"total":3})
+            } else {
+                json!({"apps":[{"title_id":"CUSA00003","app_id":1,"name":"c"}],
+                       "offset":1,"next":2,"more":false,"truncated":false,"total":2})
+            })
+            .unwrap(),
+        ))
+    });
+    let d = appdb_query(A).unwrap();
+    assert_eq!(d.apps.len(), 2);
+    assert!(d.truncated);
+}

engine/crates/ps5upload-engine/src/lib.rs

@@ -5048,19 +5048,33 @@ fn appdb_installed_additions(
     out
 }
 
-/// Titles we registered whose app folder exists but that neither the /user/appmeta scan
-/// nor app.db produced (#473): a homebrew-installed title has no appmeta folder, and app.db
-/// may not have answered for it (a failed or partial read). The registration and the folder
-/// on disk are evidence enough. Sorted, so the list does not reshuffle between refreshes.
+/// Candidates for titles whose app folder exists but that neither the /user/appmeta scan
+/// nor app.db produced (#473), taken from the helper's registered list.
+///
+/// Only when app.db could not be read in full (`appdb_whole` false): with a complete app.db, a
+/// folder it does not list is not an installed title (a failed install, a partial uninstall),
+/// and the helper's list holds every /user/app folder, not only titles we registered. System
+/// ids (NPXS) are left out, as in [`appdb_installed_additions`]. The caller still checks each
+/// candidate for evidence it is a title (a mount.lnk source or a param file). Sorted, so the
+/// list does not reshuffle between refreshes.
 fn registered_only_additions<'a>(
     registered: impl IntoIterator<Item = &'a str>,
     app_dirs: &std::collections::HashSet<String>,
     seen: &std::collections::HashSet<String>,
+    appdb_whole: bool,
 ) -> Vec<String> {
+    if appdb_whole {
+        return Vec::new();
+    }
     let mut out: Vec<String> = registered
         .into_iter()
         .map(str::trim)
-        .filter(|tid| looks_like_title_id(tid) && app_dirs.contains(*tid) && !seen.contains(*tid))
+        .filter(|tid| {
+            looks_like_title_id(tid)
+                && !tid.starts_with("NPXS")
+                && app_dirs.contains(*tid)
+                && !seen.contains(*tid)
+        })
         .map(str::to_string)
         .collect();
     out.sort();
@@ -5068,6 +5082,14 @@ fn registered_only_additions<'a>(
     out
 }
 
+/// True when `/user/app/<id>/sce_sys` holds a param.json or param.sfo: the folder is a title,
+/// not the leftovers of a failed install.
+fn app_folder_has_param(addr: &str, title_id: &str) -> bool {
+    ["param.json", "param.sfo"]
+        .iter()
+        .any(|f| fs_read(addr, &format!("/user/app/{title_id}/sce_sys/{f}"), 0, 16).is_ok())
+}
+
 /// GET /api/ps5/apps/installed?addr=IP:MGMT_PORT
 async fn ps5_apps_installed(
     State(state): State<AppState>,
@@ -5184,6 +5206,9 @@ async fn ps5_apps_installed(
             complete = complete
                 && crate::console_snapshot::app_folders_whole(internal_listing, app_dirs.len());
             if !app_dirs.is_empty() {
+                // Whether app.db answered in full: only then is a folder it does not list
+                // known not to be an installed title.
+                let mut appdb_whole = true;
                 let rows: Option<Vec<(String, String)>> = match appdb.get() {
                     Ok(list) => {
                         // `err` with no rows: the helper could not read app.db at all.
@@ -5198,6 +5223,7 @@ async fn ps5_apps_installed(
                                     .unwrap_or_default()
                             );
                             complete = false;
+                            appdb_whole = false;
                         }
                         Some(
                             list.apps
@@ -5209,6 +5235,7 @@ async fn ps5_apps_installed(
                     Err(e) => {
                         crate::log_warn!("apps/installed: app.db query failed: {e}");
                         complete = false;
+                        appdb_whole = false;
                         None
                     }
                 };
@@ -5235,24 +5262,30 @@ async fn ps5_apps_installed(
                         });
                     }
                 }
-                // Registered titles whose folder is there but that neither listing above
-                // produced (#473): never let a registration vanish because app.db did not
-                // answer for it.
+                // When app.db could not be read in full, a title it did not get to still
+                // has its folder and its registration (#473): list it if the folder holds a
+                // title (our mount.lnk source, or a param file), never a bare leftover folder.
                 let seen: std::collections::HashSet<String> =
                     titles.iter().map(|t| t.title_id.clone()).collect();
-                for tid in
-                    registered_only_additions(reg_map.keys().map(String::as_str), &app_dirs, &seen)
-                {
+                for tid in registered_only_additions(
+                    reg_map.keys().map(String::as_str),
+                    &app_dirs,
+                    &seen,
+                    appdb_whole,
+                ) {
                     let Some(reg) = reg_map.get(&tid) else {
                         continue;
                     };
+                    if reg.src.trim().is_empty() && !app_folder_has_param(&addr, &tid) {
+                        continue;
+                    }
                     let name = if reg.title_name.trim().is_empty() || reg.title_name == tid {
                         appmeta_title_name(&addr, &tid, &mut appdb).unwrap_or_else(|| tid.clone())
                     } else {
                         reg.title_name.clone()
                     };
                     titles.push(InstalledApp {
-                        system: tid.starts_with("NPXS"),
+                        system: false,
                         title_id: tid,
                         title_name: name,
                         origin: "registered".to_string(),
@@ -12672,26 +12705,43 @@ mod appdb_installed_additions_tests {
         );
     }
 
-    /// #473: a registered title with its app folder but no appmeta and no
-    /// app.db row (a failed or partial app.db read) is still listed.
+    /// #473: when app.db could not be read in full, a registered title with its
+    /// app folder but no appmeta and no app.db row is a candidate.
     #[test]
-    fn registered_titles_with_a_folder_are_reconciled() {
+    fn an_incomplete_appdb_read_reconciles_registered_titles_with_a_folder() {
         let got = registered_only_additions(
             [
                 "PPSA00002",
                 "ZZZZ99999",
                 "CUSA00001",
                 "PPSA00003",
+                "NPXS40056",
                 "not-an-id",
             ],
-            &set(&["ZZZZ99999", "CUSA00001", "PPSA00002", "not-an-id"]),
+            &set(&[
+                "ZZZZ99999",
+                "CUSA00001",
+                "PPSA00002",
+                "NPXS40056",
+                "not-an-id",
+            ]),
             // already listed by the appmeta scan or app.db
             &set(&["CUSA00001"]),
+            false,
         );
-        // PPSA00003 has no folder: a stale registration is not an installed title.
+        // PPSA00003 has no folder; NPXS is a system id, never added this way.
         assert_eq!(got, vec!["PPSA00002".to_string(), "ZZZZ99999".to_string()]);
     }
 
+    /// A complete app.db read that does not list a folder means the folder is not an
+    /// installed title (a failed install, a partial uninstall): it is not added.
+    #[test]
+    fn a_complete_appdb_read_adds_no_folder_it_does_not_list() {
+        let got =
+            registered_only_additions(["PPSA00002"], &set(&["PPSA00002"]), &HashSet::new(), true);
+        assert!(got.is_empty());
+    }
+
     #[test]
     fn skips_titles_the_appmeta_scan_already_found() {
         let rows = vec![("PLDM00001".to_string(), "Payload Manager".to_string())];

payload/include/content_db.h

@@ -83,13 +83,17 @@ int content_db_select_text(const char *db_path, const char *sql, int max_rows,
  *
  * Emits {"apps":[{"title_id":"..","app_id":N,"name":".."}, ...],
  *        "source":"sqlite"|"sqlite-immutable"|"scan",
- *        "offset":O,"next":N,"more":bool,"truncated":bool}
+ *        "offset":O,"next":N,"more":bool,"truncated":bool,"total":T}
  *
+ * `total` is the table's row count, sent only from SQL (the scanner's is a
+ * lower bound); a caller that sees it change between pages knows rows
+ * moved under it.
  * `next` is the row the following page starts at; `more` is true while
  * rows remain after it. A page that fills `buf` before `limit` rows stops
  * early with `more` set, so a small buffer pages rather than silently
  * dropping the rest. `truncated` is true only when rows exist that no
- * page can deliver: one row larger than `buf`, or the byte scanner (the
+ * page can deliver: one row larger than `buf`, a SQL walk that stopped
+ * before the end (SQLITE_BUSY and the like), or the byte scanner (the
  * fallback for a database that will not open) hitting its own bounds.
  *
  * `limit` 0 (or above CONTENT_DB_APPS_PAGE_MAX) means

payload/src/content_db.c

@@ -325,7 +325,8 @@ static int cdb_apps_window(content_db_app_t *out, unsigned offset, int max,
                 continue;
             /* Every row is counted (so the caller knows whether more
              * follow); only the window is copied. */
-            while (sqlite3_step(st) == SQLITE_ROW) {
+            int step;
+            while ((step = sqlite3_step(st)) == SQLITE_ROW) {
                 const unsigned char *tid = sqlite3_column_text(st, 0);
                 if (!tid) continue;
                 unsigned idx = rows_seen++;
@@ -335,6 +336,9 @@ static int cdb_apps_window(content_db_app_t *out, unsigned offset, int max,
                              (const char *)nm, sqlite3_column_int(st, 1));
                 count++;
             }
+            /* A walk that ended on anything but DONE (BUSY under the
+             * shell's write, a damaged page) did not see every row. */
+            if (rows_seen > 0 && step != SQLITE_DONE && capped) *capped = 1;
             sqlite3_finalize(st);
         }
         sqlite3_close(db);
@@ -499,11 +503,17 @@ int content_db_apps_page_json(unsigned offset, unsigned limit, char *buf,
     int truncated = capped || stuck;
     if (stuck) more_rows = 0;
 
+    /* `total` is exact only from SQL: the scanner's is a lower bound that
+     * grows with the window, so it is left out rather than mislead a
+     * caller comparing pages. */
+    char total_field[32] = "";
+    if (strcmp(source, "scan") != 0)
+        snprintf(total_field, sizeof(total_field), ",\"total\":%u", total);
     int end2 = snprintf(buf + n, cap - (size_t)n,
                         "],\"source\":\"%s\",\"offset\":%u,\"next\":%u,"
-                        "\"more\":%s,\"truncated\":%s}",
+                        "\"more\":%s,\"truncated\":%s%s}",
                         source, offset, next, more_rows ? "true" : "false",
-                        truncated ? "true" : "false");
+                        truncated ? "true" : "false", total_field);
     if (end2 < 0 || (size_t)(n + end2) >= cap) return -1;
     n += end2;
     if (written) *written = (size_t)n;

payload/tests/content_db_selftest.c

@@ -411,6 +411,7 @@ int main(void) {
         assert(json_true(pbuf, "more"));
         assert(!json_true(pbuf, "truncated"));
         assert(json_num(pbuf, "next") > 256);
+        assert(json_num(pbuf, "total") == 1001);
         printf("✓ the unpaged call says more rows follow (next=%ld)\n",
                json_num(pbuf, "next"));