From c472f52c84be64ca4f7925a0d9d6fc00ae2fad8f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:51:26 +0000 Subject: [PATCH] Lightroom import: raise catalog/WAL size limit from 1 GiB to 4 GiB Reserve the exact file size before reading so a large catalog does not need twice its size in memory while the buffer grows. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_013TGi6EFawAr2RPfqXsoCrQ --- crates/engine/src/lightroom_catalog.rs | 11 ++++++++--- docs/lightroom-catalog-import.md | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs index 2519057a3..31300812f 100644 --- a/crates/engine/src/lightroom_catalog.rs +++ b/crates/engine/src/lightroom_catalog.rs @@ -16,7 +16,8 @@ use serde_json::{Value, json}; type Row = BTreeMap; const COMMAND: &str = "library.importLightroom"; -const MAX_FILE: u64 = 1 << 30; +/// Largest catalog database or WAL read into memory (4 GiB). The whole file is held in RAM, so the cap stays finite. +const MAX_FILE: u64 = 1 << 32; fn error(why: impl Into) -> crate::EngineError { crate::EngineError::BadParams { cmd: COMMAND.into(), msg: why.into() } @@ -234,10 +235,14 @@ pub(crate) fn validate_for_job(data: &CatalogImport) -> Result<(), String> { fn bytes(path: &Path) -> Result, String> { use std::io::Read; let f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; - if f.metadata().map_err(|e| e.to_string())?.len() > MAX_FILE { - return Err("catalog or WAL exceeds 1 GiB import limit".into()); + let len = f.metadata().map_err(|e| e.to_string())?.len(); + if len > MAX_FILE { + return Err("catalog or WAL exceeds 4 GiB import limit".into()); } let mut out = Vec::new(); + // Reserve the exact size up front: growing by doubling would briefly need about twice the catalog's size. + let wanted = usize::try_from(len).map_err(|_| "catalog is too large for this platform".to_string())?; + out.try_reserve_exact(wanted).map_err(|_| "not enough memory to read the catalog".to_string())?; f.take(MAX_FILE + 1).read_to_end(&mut out).map_err(|e| e.to_string())?; if out.len() as u64 > MAX_FILE { return Err("catalog grew beyond import limit".into()); diff --git a/docs/lightroom-catalog-import.md b/docs/lightroom-catalog-import.md index 1f9deeea4..dbdd92049 100644 --- a/docs/lightroom-catalog-import.md +++ b/docs/lightroom-catalog-import.md @@ -32,7 +32,7 @@ retained. Oldest managed archives are retired; legacy `.json` archives remain un `Interop/lightroom-index.json` records imported identities. Source catalogs and originals are never overwritten. -Native import is bounded to a 1 GiB database/WAL, 16 MiB XMP packets and one million rows per +Native import is bounded to a 4 GiB database/WAL (each file is read fully into memory), 16 MiB XMP packets and one million rows per table. The in-tree, read-only Rust SQLite B-tree reader follows SQLite’s public file-format specification; it introduces no dependency or C runtime. Table payloads are capped at 256 MiB. Optional history, keyword and collection tables that exceed limits are skipped with warnings;