diff --git a/crates/engine/examples/raw_frame.rs b/crates/engine/examples/raw_frame.rs new file mode 100644 index 000000000..ce6c63e85 --- /dev/null +++ b/crates/engine/examples/raw_frame.rs @@ -0,0 +1,499 @@ +//! Measure where a camera's own JPEG sits inside its raw sensor data, and which edge bands of that data hold no +//! picture: `cargo xtask raw-frame [--json] FILE…` (or `cargo run --release -p lightcraft-engine --example raw_frame -- +//! [--json] FILE…`). It checks the crop any decoder applies against the camera's own framing, and measures the entries +//! of Nikon `MEASURED_BODIES` (`crates/raw/src/vendor/nef.rs`) for bodies that record +//! no crop. It works for any colour-filter raw that embeds a full-size camera JPEG (checked on Nikon, Sony, Canon CR2 / +//! CR3, Pentax and Samsung files, where it finds the decoder's crop); bodies with a smaller embedded preview (most +//! Fujifilm, Panasonic and Olympus files) get the edge bands only. +//! +//! **Frame.** For each raw it aligns the largest embedded JPEG with the sensor data: both are reduced to half-size +//! luminance (the raw by summing each 2×2 colour-filter block), and every offset that keeps the JPEG inside the data is +//! scored by the correlation of their gradient magnitudes. The best offset is refined to sub-pixel precision; a peak +//! that is weak, ambiguous or between two pixels is flagged. Files are grouped by model and data size (bit depths +//! pooled: both read out the same area); when 3 clean files agree, the group prints its table entries (siblings of the +//! same readout that agree on the same frame count towards the 3; see `MIN_FILES`). Use several +//! files with detail across the frame: a blank sky, fog or a lens cap has nothing to align on (low score). The JPEG +//! must be the camera's full-size one, unrotated and without in-camera geometry corrections (distortion control off); +//! a smaller preview is not aligned. +//! +//! **Edge bands.** From each edge inwards (at most 128 lines) it counts lines that hold no picture: *empty or dark* +//! (mean below 1% of the range over black) or *masked* (flat, patternless and above black: even and odd samples +//! alike within 4 standard errors or within 0.4% of white, since some filler holds a small fixed offset between the +//! two, as on the D90 and D5000; spread no more than the line's own neighbour noise and under 0.4% of white). A +//! band that the decoder's crop still includes is developed as picture: the coloured bars of issue #976. This is a +//! survey aid: a dark band can also be deep shadow, so read it with the frame. +use std::collections::BTreeMap; + +use lightcraft_codecs::DecodeOptions; +use lightcraft_raw::RawData; + +/// Smallest correlation peak worth trusting, and how far it must stand above the best offset outside its 3×3 +/// neighbourhood (measured: 0.75–0.88 on detailed D5100 / D7000 frames, runners-up ≥ 0.05 lower). +const MIN_SCORE: f64 = 0.5; +const MIN_MARGIN: f64 = 0.02; +/// Largest sub-pixel distance from a whole pixel accepted as a clean alignment. +const MAX_FRACTION: f64 = 0.35; +/// Search at most this many half-size offsets in each direction (384 px of margin around the JPEG: Canon bodies keep +/// up to ~280 px of masked border). +const MAX_SEARCH: usize = 192; +/// Windows of up to this many offsets are searched exhaustively (every Nikon body); larger ones coarse to fine. +const FULL_SEARCH: usize = 2048; +/// Edge bands are counted up to this many lines from each edge. +const MAX_BAND: usize = 128; +/// Clean, agreeing files an entry needs: of one model and data size (bit depths pooled: both read out the same area), +/// or, with at least one of its own, together with sibling models of the same readout (data and JPEG size) whose own +/// files agree on the same frame. +const MIN_FILES: usize = 3; + +struct Frame { + jpeg: (usize, usize), + /// Sub-pixel offset of the JPEG's top left corner in sensor pixels. + at: (f64, f64), + score: f64, + margin: f64, +} + +impl Frame { + fn whole(&self) -> (usize, usize) { + (self.at.0.round().max(0.0) as usize, self.at.1.round().max(0.0) as usize) + } + fn clean(&self) -> bool { + let frac = |v: f64| (v - v.round()).abs(); + self.score >= MIN_SCORE && self.margin >= MIN_MARGIN && frac(self.at.0) <= MAX_FRACTION && frac(self.at.1) <= MAX_FRACTION + } +} + +/// Lines without picture at one edge: empty or dark ones, then masked ones (or the other way round), from the edge. +#[derive(Clone, Copy, Default)] +struct Band { + dark: usize, + masked: usize, +} + +impl Band { + fn total(self) -> usize { + self.dark + self.masked + } +} + +struct Measured { + model: String, + size: (usize, usize), + bits: u32, + black: f64, + white: f64, + /// The crop the decoder applies, in sensor-data coordinates: x, y, width, height. + crop: (usize, usize, usize, usize), + /// Left, right, top, bottom. + bands: [Band; 4], + frame: Result, + /// With a clean frame: what the crop holds beyond it. + beyond: Option<[Beyond; 4]>, +} + +impl Measured { + /// Lines of each edge band (left, right, top, bottom) inside the decoder's crop: developed as picture. + fn in_crop(&self) -> [usize; 4] { + let (w, h) = self.size; + let (x, y, cw, ch) = self.crop; + let [l, r, t, b] = self.bands.map(Band::total); + [l.saturating_sub(x), (x + cw).saturating_sub(w - r.min(w)), t.saturating_sub(y), (y + ch).saturating_sub(h - b.min(h))] + } +} + +/// What the decoder's crop holds outside the camera's JPEG frame, per side (left, right, top, bottom): lines that are +/// masked (flat above black: a coloured bar), dark (empty or optical black: a dark edge) or picture (trimmed by the +/// camera, harmless). +#[derive(Clone, Copy, Default)] +struct Beyond { + masked: usize, + dark: usize, + picture: usize, +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let json = args.iter().any(|a| a == "--json"); + let files: Vec<&String> = args.iter().filter(|a| *a != "--json").collect(); + if files.is_empty() { + eprintln!("usage: raw_frame [--json] FILE… (raw files whose largest embedded JPEG is the camera's full-size one)"); + std::process::exit(2); + } + // model, data size, JPEG size -> (bit depth, frame) of each file + let mut groups: BTreeMap<(String, (usize, usize), (usize, usize)), Vec<(u32, Frame)>> = BTreeMap::new(); + for f in files { + let m = match measure(f) { + Ok(m) => m, + Err(e) => { + if json { + println!("{}", serde_json::json!({ "file": f, "error": e })); + } else { + println!("{f}: skipped: {e}"); + } + continue; + } + }; + let in_crop = m.in_crop(); + if json { + let band = |b: Band| serde_json::json!({ "dark": b.dark, "masked": b.masked }); + let frame = match &m.frame { + Ok(fr) => serde_json::json!({ + "jpeg": [fr.jpeg.0, fr.jpeg.1], "at": [fr.at.0, fr.at.1], "score": fr.score, "margin": fr.margin, "clean": fr.clean() + }), + Err(e) => serde_json::json!({ "error": e }), + }; + println!( + "{}", + serde_json::json!({ + "file": f, "model": m.model, "size": [m.size.0, m.size.1], "bits": m.bits, "black": m.black, "white": m.white, + "crop": [m.crop.0, m.crop.1, m.crop.2, m.crop.3], + "bands": { "left": band(m.bands[0]), "right": band(m.bands[1]), "top": band(m.bands[2]), "bottom": band(m.bands[3]) }, + "in_crop": { "left": in_crop[0], "right": in_crop[1], "top": in_crop[2], "bottom": in_crop[3] }, + "beyond_frame": m.beyond.map(|b| { + let side = |b: Beyond| serde_json::json!({ "masked": b.masked, "dark": b.dark, "picture": b.picture }); + serde_json::json!({ "left": side(b[0]), "right": side(b[1]), "top": side(b[2]), "bottom": side(b[3]) }) + }), + "frame": frame, + }) + ); + } else { + let frame = match &m.frame { + Ok(fr) => format!( + "JPEG {}×{} at ({:.2}, {:.2}), score {:.3}, margin {:.3}{}", + fr.jpeg.0, + fr.jpeg.1, + fr.at.0, + fr.at.1, + fr.score, + fr.margin, + if fr.clean() { "" } else { " (not clean: weak, ambiguous or between pixels)" } + ), + Err(e) => format!("no frame: {e}"), + }; + println!("{f}: {} {}×{} {}-bit, {frame}", m.model, m.size.0, m.size.1, m.bits); + let side = |name: &str, b: Band| match (b.dark, b.masked) { + (0, 0) => None, + (d, 0) => Some(format!("{name} {d} dark")), + (0, k) => Some(format!("{name} {k} masked")), + (d, k) => Some(format!("{name} {d} dark + {k} masked")), + }; + let bands: Vec = ["left", "right", "top", "bottom"].iter().zip(m.bands).filter_map(|(n, b)| side(n, b)).collect(); + let shown: Vec = + ["left", "right", "top", "bottom"].iter().zip(in_crop).filter(|(_, n)| *n > 0).map(|(s, n)| format!("{s} {n}")).collect(); + println!( + " edge bands: {}; decoder crop ({}, {}, {}, {}){}", + if bands.is_empty() { "none".to_string() } else { bands.join(", ") }, + m.crop.0, + m.crop.1, + m.crop.2, + m.crop.3, + match m.beyond { + Some(b) => { + let parts: Vec = ["left", "right", "top", "bottom"] + .iter() + .zip(b) + .filter(|(_, b)| b.masked + b.dark + b.picture > 0) + .map(|(s, b)| format!("{s} {} masked / {} dark / {} picture", b.masked, b.dark, b.picture)) + .collect(); + if parts.is_empty() { + "; nothing beyond the camera frame".to_string() + } else { + format!("; beyond the camera frame: {}", parts.join(", ")) + } + } + None if shown.is_empty() => String::new(), + None => format!(" still includes edge bands: {} lines", shown.join(", ")), + } + ); + } + if let Ok(fr) = m.frame { + groups.entry((m.model.clone(), m.size, fr.jpeg)).or_default().push((m.bits, fr)); + } + } + if json { + return; + } + println!(); + // each group's agreed frame: (model, data size, JPEG size) -> (offset, clean files), when its clean files agree + let agreed: BTreeMap<&(String, (usize, usize), (usize, usize)), ((usize, usize), usize)> = groups + .iter() + .filter_map(|(key, frames)| { + let offsets: Vec<(usize, usize)> = frames.iter().filter(|(_, f)| f.clean()).map(|(_, f)| f.whole()).collect(); + let first = *offsets.first()?; + offsets.iter().all(|o| *o == first).then_some((key, (first, offsets.len()))) + }) + .collect(); + for (key, frames) in &groups { + let (model, size, jpeg) = key; + let clean = frames.iter().filter(|(_, f)| f.clean()).count(); + let mut depths: Vec = frames.iter().map(|(b, _)| *b).collect(); + depths.sort_unstable(); + depths.dedup(); + let label = format!("{model} {}×{}", size.0, size.1); + let Some(&((x, y), _)) = agreed.get(key) else { + if clean == 0 { + println!("{label}: no clean alignment in {} files; no entry", frames.len()); + } else { + let offsets: Vec<(usize, usize)> = frames.iter().filter(|(_, f)| f.clean()).map(|(_, f)| f.whole()).collect(); + println!("{label}: files disagree: {offsets:?}; no entry"); + } + continue; + }; + let note = if clean < frames.len() { format!(", {} not clean", frames.len() - clean) } else { String::new() }; + println!("{label}: {clean} of {} files agree on ({x}, {y}){note}", frames.len()); + // siblings: other models whose sensor readout matches (same data and JPEG size) and whose own files agree on + // the same frame + let siblings: Vec<(&str, usize)> = agreed + .iter() + .filter(|((m, s, j), (o, _))| m != model && s == size && j == jpeg && *o == (x, y)) + .map(|((m, _, _), (_, n))| (m.as_str(), *n)) + .collect(); + let pooled = clean + siblings.iter().map(|(_, n)| n).sum::(); + if clean < MIN_FILES { + if pooled < MIN_FILES { + println!(" pending: {} more clean file(s) needed for an entry", MIN_FILES - pooled); + continue; + } + let names: Vec = siblings.iter().map(|(m, n)| format!("{m} ({n})")).collect(); + println!(" with siblings of the same readout and frame: {}", names.join(", ")); + } + for bits in depths { + println!( + " MeasuredBody {{ model: {model:?}, size: ({}, {}), bits: {bits}, crop: ({x}, {y}, {}, {}), white: None }},", + size.0, size.1, jpeg.0, jpeg.1 + ); + } + } +} + +fn measure(path: &str) -> Result { + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + let raw = lightcraft_raw::decode(&bytes).map_err(|e| e.to_string())?; + let RawData::U16(d) = &raw.data else { return Err("not 16-bit integer sensor data".into()) }; + if raw.cpp != 1 || raw.cfa.is_none() { + return Err("not colour-filter sensor data".into()); + } + let (w, h) = (raw.width, raw.height); + if w < 4 || h < 4 || d.len() < w.saturating_mul(h) { + return Err("sensor data shorter than its size".into()); + } + let black = f64::from(raw.black.values.iter().sum::() / raw.black.values.len().max(1) as f32).max(0.0); + let white = f64::from(raw.white.first().copied().unwrap_or(((1u32 << raw.bits.min(16)) - 1) as f32)).max(black + 1.0); + let at = |x: usize, y: usize| f64::from(d.get(y * w + x).copied().unwrap_or(0)); + let bands = [ + band((0..w.min(MAX_BAND)).map(|x| line(h, |i| at(x, i))), black, white), + band((0..w.min(MAX_BAND)).map(|k| line(h, |i| at(w - 1 - k, i))), black, white), + band((0..h.min(MAX_BAND)).map(|y| line(w, |i| at(i, y))), black, white), + band((0..h.min(MAX_BAND)).map(|k| line(w, |i| at(i, h - 1 - k))), black, white), + ]; + // the decoder's crop is relative to the active area: in sensor-data coordinates here + let a = raw.active_area; + let c = lightcraft_raw::Rect::new(a.x + raw.crop.x, a.y + raw.crop.y, raw.crop.width, raw.crop.height); + let frame = frame(&bytes, d, w, h); + let beyond = frame.as_ref().ok().filter(|f| f.clean()).map(|f| { + let ((fx, fy), (fw, fh)) = (f.whole(), f.jpeg); + let kinds = |lines: Vec| { + lines.iter().fold(Beyond::default(), |mut b, l| { + match kind(l, black, white) { + Kind::Dark => b.dark += 1, + Kind::Masked => b.masked += 1, + Kind::Picture => b.picture += 1, + } + b + }) + }; + let col = |x: usize| line(h, |i| at(x, i)); + let row = |y: usize| line(w, |i| at(i, y)); + [ + kinds((c.x..fx.min(c.x + c.width)).map(col).collect()), + kinds(((fx + fw).max(c.x)..c.x + c.width).map(col).collect()), + kinds((c.y..fy.min(c.y + c.height)).map(row).collect()), + kinds(((fy + fh).max(c.y)..c.y + c.height).map(row).collect()), + ] + }); + Ok(Measured { + model: raw.metadata.model.clone().unwrap_or_default().trim().to_string(), + size: (w, h), + bits: raw.bits, + black, + white, + crop: (c.x, c.y, c.width, c.height), + bands, + frame, + beyond, + }) +} + +/// Statistics of one line of sensor samples: mean, |even − odd| mean, spread, neighbour noise (same colour, 2 apart) +/// and the number of sample pairs. +struct Line { + mean: f64, + pattern: f64, + spread: f64, + noise: f64, + pairs: f64, +} + +fn line(n: usize, at: impl Fn(usize) -> f64) -> Line { + let stride = ((n / 512).max(1)).div_ceil(2) * 2; + let (mut k, mut even, mut odd, mut var, mut noise) = (0f64, 0f64, 0f64, 0f64, 0f64); + let idx: Vec = (0..n.saturating_sub(2)).step_by(stride).collect(); + for &i in &idx { + even += at(i); + odd += at(i + 1); + k += 1.0; + } + let (even, odd) = (even / k.max(1.0), odd / k.max(1.0)); + for &i in &idx { + var += (at(i) - even).powi(2); + noise += (at(i + 2) - at(i)).powi(2) / 2.0; + } + Line { mean: (even + odd) / 2.0, pattern: (even - odd).abs(), spread: (var / k.max(1.0)).sqrt(), noise: (noise / k.max(1.0)).sqrt(), pairs: k } +} + +enum Kind { + Dark, + Masked, + Picture, +} + +fn kind(l: &Line, black: f64, white: f64) -> Kind { + let range = white - black; + let slack = white / 4096.0; + if l.mean < black + range * 0.01 { + Kind::Dark + } else if l.mean > black + range * 0.02 + && l.mean < white * 0.25 + // patternless within the noise, or with a small fixed offset between the CFA phases (D90 / D5000 filler) + && (l.pattern < 4.0 * l.noise * (2.0 / l.pairs.max(1.0)).sqrt() + slack || l.pattern < white * 0.004) + && l.spread < 1.5 * l.noise + slack + && l.spread < white * 0.004 + { + Kind::Masked + } else { + Kind::Picture + } +} + +fn band(lines: impl Iterator, black: f64, white: f64) -> Band { + let mut b = Band::default(); + for l in lines { + match kind(&l, black, white) { + Kind::Dark => b.dark += 1, + Kind::Masked => b.masked += 1, + Kind::Picture => break, + } + } + b +} + +fn frame(bytes: &[u8], d: &[u16], w: usize, h: usize) -> Result { + let jpeg = lightcraft_raw::embedded_preview(bytes).ok_or("no embedded preview")?; + let jpeg = lightcraft_codecs::decode(&jpeg, DecodeOptions::default()).map_err(|e| e.to_string())?; + let (jw, jh) = (jpeg.image.width, jpeg.image.height); + if jw > w || jh > h || jw * 10 < w * 9 || jh * 10 < h * 9 { + return Err(format!("the largest preview ({jw}×{jh}) is not a full-size JPEG of the {w}×{h} data")); + } + // half-size luminance, square-rooted so the linear raw and the JPEG weigh their tones alike + let (rw, rh) = (w / 2, h / 2); + let raw_y: Vec = (0..rh * rw) + .map(|i| { + let (x, y) = (2 * (i % rw), 2 * (i / rw)); + let s: f64 = + [(x, y), (x + 1, y), (x, y + 1), (x + 1, y + 1)].iter().map(|&(x, y)| f64::from(d.get(y * w + x).copied().unwrap_or(0))).sum(); + s.sqrt() + }) + .collect(); + let (pw, ph) = (jw / 2, jh / 2); + let px = &jpeg.image.data; + let jpg_y: Vec = (0..ph * pw) + .map(|i| { + let (x, y) = (2 * (i % pw), 2 * (i / pw)); + let s: f64 = [(x, y), (x + 1, y), (x, y + 1), (x + 1, y + 1)] + .iter() + .map(|&(x, y)| px.get(y * jw + x).map_or(0.0, |p| f64::from(0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2]).max(0.0))) + .sum(); + s.sqrt() + }) + .collect(); + let (sx, sy) = ((rw - pw).min(MAX_SEARCH), (rh - ph).min(MAX_SEARCH)); + let grad = |img: &[f64], stride: usize, x: usize, y: usize| { + let at = |x: usize, y: usize| img.get(y * stride + x).copied().unwrap_or(0.0); + (at(x + 1, y) - at(x, y)).hypot(at(x, y + 1) - at(x, y)) + }; + let normalise = |v: Vec| { + let n = v.len().max(1) as f64; + let mean = v.iter().sum::() / n; + let sd = (v.iter().map(|a| (a - mean).powi(2)).sum::() / n).sqrt().max(1e-12); + v.into_iter().map(|a| (a - mean) / sd).collect::>() + }; + // gradient magnitudes on every `step`-th half-size pixel, normalised, and the correlation at one offset + let sample = |step: usize| { + let points: Vec<(usize, usize)> = + (0..ph.saturating_sub(1)).step_by(step).flat_map(|y| (0..pw.saturating_sub(1)).step_by(step).map(move |x| (x, y))).collect(); + let gj = normalise(points.iter().map(|&(x, y)| grad(&jpg_y, pw, x, y)).collect()); + (points, gj) + }; + let score = |(points, gj): &(Vec<(usize, usize)>, Vec), dx: usize, dy: usize| { + let gr = normalise(points.iter().map(|&(x, y)| grad(&raw_y, rw, x + dx, y + dy)).collect()); + gr.iter().zip(gj).map(|(a, b)| a * b).sum::() / gj.len().max(1) as f64 + }; + let fine = sample(2); + // every offset when the window is small (every Nikon body); else every 4th on sparser points, then each offset + // around the best of those + let mut scores = vec![vec![f64::NEG_INFINITY; sx + 1]; sy + 1]; + let mut coarse_best = f64::NEG_INFINITY; + let (mut cx, mut cy) = (0, 0); + let window = if (sx + 1) * (sy + 1) <= FULL_SEARCH { + (0..=sx, 0..=sy) + } else { + let coarse = sample(8); + for dy in (0..=sy).step_by(4) { + for dx in (0..=sx).step_by(4) { + let v = score(&coarse, dx, dy); + if let Some(s) = scores.get_mut(dy).and_then(|r| r.get_mut(dx)) { + *s = v; + } + if v > coarse_best { + (coarse_best, cx, cy) = (v, dx, dy); + } + } + } + (cx.saturating_sub(5)..=(cx + 5).min(sx), cy.saturating_sub(5)..=(cy + 5).min(sy)) + }; + for dy in window.1.clone() { + for dx in window.0.clone() { + if let Some(s) = scores.get_mut(dy).and_then(|r| r.get_mut(dx)) { + *s = score(&fine, dx, dy); + } + } + } + let at = |dx: usize, dy: usize| scores.get(dy).and_then(|r| r.get(dx)).copied().unwrap_or(f64::NEG_INFINITY); + let (mut bx, mut by) = (0, 0); + for dy in 0..=sy { + for dx in 0..=sx { + if at(dx, dy) > at(bx, by) { + (bx, by) = (dx, dy); + } + } + } + let best = at(bx, by); + let runner_up = (0..=sy) + .flat_map(|dy| (0..=sx).map(move |dx| (dx, dy))) + .filter(|&(dx, dy)| dx.abs_diff(bx) > 1 || dy.abs_diff(by) > 1) + .map(|(dx, dy)| at(dx, dy)) + .fold(f64::NEG_INFINITY, f64::max); + // parabola through the peak and its neighbours (none at the window's edge) + let refine = |a: f64, b: f64, c: f64| { + let k = a - 2.0 * b + c; + if a.is_finite() && c.is_finite() && k < 0.0 { (0.5 * (a - c) / k).clamp(-0.5, 0.5) } else { 0.0 } + }; + let fx = if bx > 0 { refine(at(bx - 1, by), best, at(bx + 1, by)) } else { 0.0 }; + let fy = if by > 0 { refine(at(bx, by - 1), best, at(bx, by + 1)) } else { 0.0 }; + Ok(Frame { + jpeg: (jw, jh), + at: (2.0 * (bx as f64 + fx), 2.0 * (by as f64 + fy)), + score: best, + margin: if runner_up.is_finite() { best - runner_up } else { best }, + }) +} diff --git a/crates/raw/src/vendor/nef.rs b/crates/raw/src/vendor/nef.rs index 7d5b24a4f..271f9c71f 100644 --- a/crates/raw/src/vendor/nef.rs +++ b/crates/raw/src/vendor/nef.rs @@ -87,11 +87,88 @@ struct MeasuredBody { /// the masked-column test keeps them and they develop as a pink band on the right. The camera's 4288 × 2848 JPEG /// sits at (10, 10). Green saturates at 3840–3851 while red and blue run on to 4095; one level of 3830 keeps /// clipped highlights white. +/// * **NIKON D7000** and **NIKON D5100** (and the Coolpix A, the same readout): the data is 4992 × 3280, the picture its +/// first 4948 columns, then 36 masked ones at one level (~600, with a black level of 0) and 8 of 0. The masked +/// columns sit above the dark-column trim's threshold, so without this area they show as a coloured bar on the right +/// edge (the top in portrait; issue #976). The camera's 4928 × 3264 JPEG sits at (8, 8): in the D5100's 3 CC0 +/// samples, and in the D7000's one clean one, with the D5100 and Coolpix A as siblings (see below). +/// * **The other bodies** below keep masked or dark lines inside the whole data (a survey of every Nikon raw on raw.pixls.us): +/// each crop is its camera JPEG's frame as `cargo xtask raw-frame` measures it on raw.pixls.us CC0 samples (the +/// D200, D3200 and Nikon 1 V1 also on other publicly posted sample files), with at least 3 clean +/// files agreeing: of the model (bit depths pooled: both read out the same area), or, with at least +/// one of its own, together with models of the same readout (data and JPEG size) that agree on the same frame +/// (*siblings*). The D4's 12-bit mode reads out the 14-bit area measured above; the Df, the same readout, agrees on +/// (8, 6) in 6 files. Bodies whose crop holds only picture beyond the camera frame have no entry (no change). const MEASURED_BODIES: &[MeasuredBody] = &[ MeasuredBody { model: "NIKON D4", size: (4992, 3292), bits: 14, crop: (8, 6, 4928, 3280), white: Some(15520.0) }, + MeasuredBody { model: "NIKON D4", size: (4992, 3292), bits: 12, crop: (8, 6, 4928, 3280), white: None }, MeasuredBody { model: "NIKON D7100", size: (6036, 4020), bits: 14, crop: (18, 10, 6000, 4000), white: None }, MeasuredBody { model: "NIKON D7100", size: (4812, 3212), bits: 14, crop: (6, 6, 4800, 3200), white: None }, MeasuredBody { model: "NIKON D90", size: (4352, 2868), bits: 12, crop: (10, 10, 4288, 2848), white: Some(3830.0) }, + MeasuredBody { model: "NIKON D7000", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + MeasuredBody { model: "NIKON D7000", size: (4992, 3280), bits: 12, crop: (8, 8, 4928, 3264), white: None }, + MeasuredBody { model: "NIKON D5100", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + // siblings: D7000 (4), D5100 (3); masked: right 36 + MeasuredBody { model: "COOLPIX A", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + // 6 files each; masked: left 2, right 2 + MeasuredBody { model: "NIKON D3", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3S", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3S", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D700", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D700", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + // 6 and 4 files; masked: right up to 32 + MeasuredBody { model: "NIKON D300", size: (4352, 2868), bits: 12, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300", size: (4352, 2868), bits: 14, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300S", size: (4352, 2868), bits: 12, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300S", size: (4352, 2868), bits: 14, crop: (15, 10, 4288, 2848), white: None }, + // D2X (1) and D2Xs (2) as siblings; dark: right 1 + MeasuredBody { model: "NIKON D2Xs", size: (4320, 2868), bits: 12, crop: (16, 10, 4288, 2848), white: None }, + // D3000, D40X, D60, D80: one file each, siblings; masked: right 4, bottom 3 + MeasuredBody { model: "NIKON D3000", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D40X", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D60", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D80", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + // 3 files; masked: left 1, right 3 (the D80's data size, framed elsewhere) + MeasuredBody { model: "NIKON D200", size: (3904, 2616), bits: 12, crop: (16, 12, 3872, 2592), white: None }, + // 22 files; masked: right 46 + MeasuredBody { model: "NIKON D3200", size: (6080, 4012), bits: 12, crop: (8, 6, 6016, 4000), white: None }, + // 8 clean of 19, the rest ~0.3 px between pixels on the same frame; dark: bottom 2 + MeasuredBody { model: "NIKON 1 V1", size: (3904, 2606), bits: 12, crop: (12, 6, 3872, 2592), white: None }, + // Best guesses, short of the evidence rule; every frame below keeps the masked lines out by several pixels. + // D3100: 3 of 4 clean files at (24, 6), the 4th at (22, 7); masked: left 6, right 22 + MeasuredBody { model: "NIKON D3100", size: (4672, 3084), bits: 12, crop: (24, 6, 4608, 3072), white: None }, + // D5000: 4 of 5 clean files at (11, 10), the 5th at (11, 12); the D90's sensor, whose own files also align at + // x = 11; masked: right 42 (the same filler as the D90's) + MeasuredBody { model: "NIKON D5000", size: (4352, 2868), bits: 12, crop: (11, 10, 4288, 2848), white: None }, + // D40: 2 of 2 clean files at (16, 9); masked: right 1. The D50 / D70 / D70s share its data size but not its frame + MeasuredBody { model: "NIKON D40", size: (3040, 2014), bits: 12, crop: (16, 9, 3008, 2000), white: None }, + // D50, D70, D70s (one readout): one clean D50 and one clean D70 file at (15, 7); the D70s's 2 files align + // between pixels next to it, at (15.4, 6.9) and (15.6, 6.4); masked: right 1 + MeasuredBody { model: "NIKON D50", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + MeasuredBody { model: "NIKON D70", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + MeasuredBody { model: "NIKON D70s", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + // 3 files; dark: bottom 4 + MeasuredBody { model: "NIKON D4S", size: (4936, 3288), bits: 12, crop: (4, 4, 4928, 3280), white: None }, + MeasuredBody { model: "NIKON D4S", size: (4936, 3288), bits: 14, crop: (4, 4, 4928, 3280), white: None }, + // 4 files; dark: bottom up to 4 + MeasuredBody { model: "NIKON D5600", size: (6016, 4016), bits: 12, crop: (8, 8, 6000, 4000), white: None }, + MeasuredBody { model: "NIKON D5600", size: (6016, 4016), bits: 14, crop: (8, 8, 6000, 4000), white: None }, + // 4 files each; masked: right 8 + MeasuredBody { model: "NIKON D600", size: (6080, 4028), bits: 12, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D600", size: (6080, 4028), bits: 14, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D610", size: (6080, 4028), bits: 12, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D610", size: (6080, 4028), bits: 14, crop: (8, 6, 6016, 4016), white: None }, + // D800: 6 files; D800E: 2, with the D800 as sibling; masked: right 14 (issue #914, the top in portrait) + MeasuredBody { model: "NIKON D800", size: (7424, 4924), bits: 12, crop: (8, 6, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D800", size: (7424, 4924), bits: 14, crop: (8, 6, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D800E", size: (7424, 4924), bits: 14, crop: (8, 6, 7360, 4912), white: None }, + // 3 files; dark: bottom 1 + MeasuredBody { model: "NIKON D810", size: (7380, 4928), bits: 12, crop: (12, 8, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D810", size: (7380, 4928), bits: 14, crop: (12, 8, 7360, 4912), white: None }, + // 6 files; masked: left 2, right 42 (the D4's readout) + MeasuredBody { model: "NIKON Df", size: (4992, 3292), bits: 12, crop: (8, 6, 4928, 3280), white: None }, + MeasuredBody { model: "NIKON Df", size: (4992, 3292), bits: 14, crop: (8, 6, 4928, 3280), white: None }, ]; fn measured_body(model: &str, w: usize, h: usize, bits: u32) -> Option<&'static MeasuredBody> { @@ -297,6 +374,22 @@ mod tests { /// [`nef_with_note`] in either byte order (file and maker note). #[allow(clippy::too_many_arguments)] fn nef_in(order: ByteOrder, compression: u16, bits: u16, strips: Vec>, w: u32, h: u32, rps: u32, note: Option) -> Vec { + nef_model_in(order, "NIKON TEST", compression, bits, strips, w, h, rps, note) + } + + /// [`nef_in`] from camera `model`. + #[allow(clippy::too_many_arguments)] + fn nef_model_in( + order: ByteOrder, + model: &str, + compression: u16, + bits: u16, + strips: Vec>, + w: u32, + h: u32, + rps: u32, + note: Option, + ) -> Vec { let mut raw = IfdBuilder::new(); raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0])); raw.set(t::IMAGE_WIDTH, Value::Long(vec![w])); @@ -309,7 +402,7 @@ mod tests { raw.set_image(ImageData::Strips { rows_per_strip: rps, strips }); let mut ifd0 = IfdBuilder::new(); ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into())); - ifd0.set(t::MODEL, Value::Ascii("NIKON TEST".into())); + ifd0.set(t::MODEL, Value::Ascii(model.into())); if let Some(mn) = note { let mut bytes = b"Nikon\0\x02\x10\0\0".to_vec(); bytes.extend(TiffWriter::new(order, false).write(&[mn]).unwrap()); @@ -457,9 +550,76 @@ mod tests { assert_eq!((d4.metadata.width, d4.metadata.height), (Some(4928), Some(3280))); let developed = d4.develop(crate::Method::Bilinear).unwrap(); assert_eq!((developed.width, developed.height), (4928, 3280)); - for other in [decode_as("NIKON D5", 14), decode_as("NIKON D4", 12)] { - assert_eq!(other.crop, Rect::new(0, 0, w, h)); - assert_ne!(other.white, vec![15520.0]); + // the 12-bit mode reads out the same area but keeps the white level from its data + let d4_12 = decode_as("NIKON D4", 12); + assert_eq!(d4_12.crop, Rect::new(8, 6, 4928, 3280)); + assert_ne!(d4_12.white, vec![15520.0]); + let other = decode_as("NIKON D5", 14); + assert_eq!(other.crop, Rect::new(0, 0, w, h)); + assert_ne!(other.white, vec![15520.0]); + } + + /// Issue #976: the D7000 and D5100 record no CropArea, and their 36 masked columns at ~600 were developed as a + /// coloured bar. Their measured area applies to the measured models, data sizes and depths only. + #[test] + fn measured_d7000_and_d5100_image_area() { + let (w, h) = (4992usize, 3280usize); + let decode_as = |model: &str, bits: u16| { + let nef = nef_model_in(ByteOrder::Big, model, 1, bits, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32, None); + crate::decode(&nef).unwrap() + }; + for (model, bits) in [("NIKON D7000", 14), ("NIKON D7000", 12), ("NIKON D5100", 14)] { + let raw = decode_as(model, bits); + assert_eq!(raw.crop, Rect::new(8, 8, 4928, 3264), "{model} {bits}-bit"); + assert_eq!((raw.metadata.width, raw.metadata.height), (Some(4928), Some(3264))); + let developed = raw.develop(crate::Method::Bilinear).unwrap(); + assert_eq!((developed.width, developed.height), (4928, 3264)); + } + // another model, or the D5100's unmeasured 12-bit mode, keeps the whole data + for (model, bits) in [("NIKON D7100", 14), ("NIKON D5100", 12), ("NIKON D7000 ", 16)] { + assert_eq!(decode_as(model, bits).crop, Rect::new(0, 0, w, h), "{model} {bits}-bit"); + } + } + + /// The D200, D3200 and Nikon 1 V1 keep masked or dark lines in their whole data (the D200 shares the D80's data + /// size but frames it elsewhere), measured on public sample files beyond the CC0 ones; the D3100, D5000, D40, D50, + /// D70 and D70s get best-guess frames (the D40 shares the D50 / D70 / D70s data size but not their frame). + #[test] + fn measured_older_dx_bodies_image_area() { + let decode_as = |model: &str, (w, h): (usize, usize)| { + let nef = nef_model_in(ByteOrder::Big, model, 1, 12, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32, None); + crate::decode(&nef).unwrap() + }; + for (model, size, crop) in [ + ("NIKON D200", (3904, 2616), Rect::new(16, 12, 3872, 2592)), + ("NIKON D80", (3904, 2616), Rect::new(14, 9, 3872, 2592)), + ("NIKON D3200", (6080, 4012), Rect::new(8, 6, 6016, 4000)), + ("NIKON 1 V1", (3904, 2606), Rect::new(12, 6, 3872, 2592)), + ("NIKON D3100", (4672, 3084), Rect::new(24, 6, 4608, 3072)), + ("NIKON D5000", (4352, 2868), Rect::new(11, 10, 4288, 2848)), + ("NIKON D40", (3040, 2014), Rect::new(16, 9, 3008, 2000)), + ("NIKON D50", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ("NIKON D70", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ("NIKON D70s", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ] { + assert_eq!(decode_as(model, size).crop, crop, "{model}"); + } + // the V1's area on the V2's data size: no entry, the whole data + assert_eq!(decode_as("NIKON 1 V1", (4620, 3084)).crop, Rect::new(0, 0, 4620, 3084)); + } + + /// Every measured entry is a crop inside its data, once per model, data size and depth, with a white level (if + /// any) inside its bit range. + #[test] + fn measured_bodies_are_consistent() { + let mut seen = std::collections::HashSet::new(); + for b in MEASURED_BODIES { + let (x, y, w, h) = b.crop; + assert!(w > 0 && h > 0 && x + w <= b.size.0 && y + h <= b.size.1, "{}: crop outside its data", b.model); + assert!(seen.insert((b.model, b.size, b.bits)), "{} {:?} {}-bit listed twice", b.model, b.size, b.bits); + assert!(matches!(b.bits, 12 | 14), "{}: {}-bit", b.model, b.bits); + assert!(b.white.is_none_or(|v| v > 0.0 && v < (1u32 << b.bits) as f32), "{}: white level", b.model); + assert_eq!(b.model.trim(), b.model, "{}: model names are matched trimmed", b.model); } } diff --git a/docs/gaps.md b/docs/gaps.md index e63fcbf6b..2e9d66a04 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,6 +1,6 @@ # Gaps: where LightCraft falls short of Lightroom -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-14 · **Change:** minor (catalog import result, offline Local promotion and reduced copies; gap 8: Nikon Z 6II / Z 8 / Z f in-file distortion; gaps 10, 22 and 23 listed once again: merges had left two copies of each, and `cargo xtask parity` now fails on a gap listed twice) +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-14 · **Change:** minor (Nikon edge bars on bodies without CropArea fixed, #976 / #914 / #1236; gap 10 notes the survey; catalog import result, offline Local promotion and reduced copies; gap 8: Nikon Z 6II / Z 8 / Z f in-file distortion; gaps 10, 22 and 23 listed once again: merges had left two copies of each, and `cargo xtask parity` now fails on a gap listed twice) Every known shortfall, one entry each, ranked by user impact (how many Lightroom users hit it, and how hard it blocks them), then by effort. **Agents pick work from the top** (the first entry nobody is @@ -24,7 +24,7 @@ Hours are Opus 5.5 agent wall-clock hours (see [target-app-parity.md](target-app | 7 | **Nikon High Efficiency NEF** (HE / HE★ on Z 8, Z 9, Z 6III, Z f, Z50II, Z5II when set to HE) | named and preview only (#193); needs a pure-Rust JPEG XS decoder | +1.1 points (only shots taken in HE; lossless works on the same bodies); low gain per hour (~0.04), so not a beta blocker | 20–40 | [raw-parity](raw-parity.md) | | 8 B | **No lens profile database** (LR-EDIT-OPTICS-PROFILE): only DNG opcodes, RW2 in-file distortion, Sony ILCE-7RM4A distortion and Nikon Z 6II / Z 8 / Z f distortion (`docs/nikon-lens-corrections.md`) | Adobe ships 3,635 lens profiles; we never use them | distortion / vignetting correction is a default step for many | 40–80 + calibration shots | [raw-parity](raw-parity.md#lens-profiles) | | 9 B | **Tests don't run in CI on macOS, Windows or Linux x64**: only FreeBSD runs `cargo test`; the others build packages | `.github/workflows/` (freebsd, packaging-lint, release, windows-arm64); the linux-riscv64 release job added in 0.6.0 (#844) runs only `lightcraft-cli --version` under QEMU | regressions reach users on the main platforms; `cargo xtask ci` runs only locally | 6–12 | [target-app-parity](target-app-parity.md#by-dimension) | -| 10 B | **Per-model verification thin, plus cheap per-body decode fixes**: 236 of 1,446 Adobe models verified (16%), ~42% of photographers on a verified body; Nikon 5 bodies verified, Canon 12, Leica 14 (the Q3 by a recorded check of 38 private DNGs). Cheap fixes: Z 6II packed 14-bit NEF (#769, PR #671), Sony A100 / F828 regression (#702), DSLR-A200…A390 / A850 12-bit ARW (#535) | measured: [raw/coverage.tsv](raw/coverage.tsv) | per-body bugs (levels, crops, WB) surface only from users | 25–45 | [raw-parity](raw-parity.md#by-maker-measured) | +| 10 B | **Per-model verification thin, plus cheap per-body decode fixes**: 236 of 1,446 Adobe models verified (16%), ~42% of photographers on a verified body; Nikon 5 bodies verified, Canon 12, Leica 14 (the Q3 by a recorded check of 38 private DNGs). Cheap fixes: Z 6II packed 14-bit NEF (#769, PR #671), Sony A100 / F828 regression (#702), DSLR-A200…A390 / A850 12-bit ARW (#535) | measured: [raw/coverage.tsv](raw/coverage.tsv) | per-body bugs (levels, crops, WB) surface only from users; Nikon crops of bodies without CropArea now surveyed and measured ([raw-parity](raw-parity.md#variant-matrix-depth-per-format), D3100 / D5000 / D40 / D50 / D70 / D70s on best-guess frames until more samples confirm them) | 25–45 | [raw-parity](raw-parity.md#by-maker-measured) | | 11 | **Enhance: Raw Details, Super Resolution; AI denoise without shippable weights or X-Trans** | ENH rows 0%; denoise Bayer only, RawNIND weights GPL and opt-in | Fujifilm users get no AI denoise; nobody gets it out of the box | 30–60 + model decision | [raw-parity](raw-parity.md#demosaic-levels-crops-and-lens-metadata) | | 12 | **Content-aware / generative remove, detect distractions, reflections removal** | REM rows 75%; patch synthesis missing | cleanup work goes back to Photoshop | 25–45 (+ generative model) | [target-app-parity](target-app-parity.md#ai) | | 13 | **Lightroom catalog migration is one-way and approximate**: unsupported Adobe profiles / AI settings archived, history kept as data; offline record counts and destination are now shown in a lasting import result | lightroom-catalog-import.md | switching users lose some looks | 8–12 | [file-format-parity](file-format-parity.md) | @@ -59,6 +59,7 @@ Beta-blocking gaps (B) total **~230–430 h**; with the stability and platform w ## Closed recently (kept for calibration) +- 2026-10-14: Nikon edge bars on bodies without CropArea (#976, #914, #1236): 44 more model / size / depth groups measured (6 on best-guess frames). - 2026-10-10: Nikon lossy-after-split NEF (#633), ORF 12.8 bpp (#651), HDR gain map / PQ AVIF / float TIFF export (#646), activity stack (#345), JPEG XL export (#247). - 2026-10-09: EOS R7 C-RAW with an odd-height QP map (a45c2b66; part of gap 2), HEIC in release builds (#609), Samsung uncompressed SRW (#459), contact-sheet PDF (#563), @@ -81,6 +82,7 @@ Beta-blocking gaps (B) total **~230–430 h**; with the stability and platform w | 2026-10-12 | minor | Gap 25: Render to DNG lands as an export format; lossy DNG and AVIF import remain | | 2026-10-12 | minor | Gap 10 shrinks: the Hasselblad Lunar (a Sony NEX-7 ARW under its own Make) decodes and is verified, 236 models (#1275) | | 2026-10-12 | minor | Gap 14: HLG AVIF export (`hdrTransfer`) and a tested SDR/HDR display decision | +| 2026-10-14 | minor | Nikon edge bars on bodies without CropArea fixed for 44 more model / size / depth groups (#976, #914, #1236); gap 10 notes the survey and the best-guess bodies; gap not closed | | 2026-10-11 | minor | Reviewed against main after 0.6.0: CR3 entry names the CRX variants `crx.rs` still refuses and the R7 / R6 Mark II C-RAW state (decoding, not verified, so the measured numbers don't move); CI and Windows / Linux entries note the riscv64 release build (#844); R7 fix added to Closed recently. Panel docking (UI.9, #869) closes no listed gap | | 2026-10-11 | minor | Camera colour entry: raws with a black-and-white camera JPEG take its tone curve (#577); gap not closed | | 2026-10-11 | minor | Gap 4: Compare and Reference views get zoom windows per pane (1:1 was a 2,560 px render magnified); overlays and soft proofing remain, 8–15 h → 4–8 h | diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md index a434d28cf..8e50083c0 100644 --- a/docs/parity-checklist.md +++ b/docs/parity-checklist.md @@ -81,7 +81,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts). | LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts (album commands skip them and the album menus say to Add to My Photos first, issue #684); folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) | | LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars | | LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs`, `crates/raw/src/jxl.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP (lossy wide-gamut JXL decodes to the right colours: jxl-oxide desaturated XYB rendered to P3/Rec. 2020, worked around in `crates/codecs/src/jxl.rs`); DNG (incl. lossy / Smart Preview DNG and DNG 1.7 JPEG XL tiles/strips: lossless ones decode bit-exactly on synthetic CFA and LinearRaw files, lossy ones are restored to the declared linear primaries without clipping or gamut mapping, checked on one real file with 108 lossy tiles, and any tile that fails to decode makes the file preview-only; DNG 1.7 row/column-interleaved rasters (`RowInterleaveFactor` / `ColumnInterleaveFactor`) are undone before the mosaic is read, so a file from Adobe's converter with a 2×2 interleave no longer decodes as four scaled-down copies in the wrong colours; a dimension the factor does not divide decodes with short trailing fields; verified on 4 real ILCE-6400 JPEG XL DNGs rendering bit-identical to their non-interleaved twins, factors above 2 rest on the plain reading of the tag; a single-chunk JPEG XL preview IFD is offered as the embedded preview; a DNG whose samples pile up at one value well below its stated `WhiteLevel`, as a camera app's 12-bit Bayer DNG that states 4095 and clips at 3567, has those samples raised to the white level on decode, so highlight reconstruction sees the clipped areas instead of white balance tinting them magenta, issue #548, `crates/raw/src/saturation.rs`), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from the encrypted `SR2SubIFD`, issue #148, its `WB_RGGBLevels` being the white balance the camera applied, issue #535; the DSLR-A900's packed 12-bit strip, two pixels per three bytes, read only when the strip is exactly 1.5 bytes per pixel, with its camera-JPEG window centred, checked on one CC0 file (A850 unmeasured); downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export; Sony DSC-R1 .SR2 decodes, its big-endian 16-bit words read in the order that fits the sample range, checked on the CC0 3221 sample; its crop and white balance live in the SR2SubIFD and are not read yet; ARW6 Compressed / Compressed (HQ) of the ILCE-7RM6 (`crates/raw/src/vendor/arw6.rs`, `crates/raw/src/llvc.rs`; 12-bit codes exact against a black-box oracle on 8 CC0 files, FF over the full frame, APS-C in the interior (LibRaw's own edge rows differ); output through our own companding curve, measured from CC0 lossless / ARW6 pairs; geometry checked against the camera JPEG, `docs/arw6-compression.md`; the ILCE-7M5 writes the same codec, unverified)), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`; 12.8 bit/px 16-byte blocks of ten 12-bit samples + a zero pad byte, E-300 / E-330 / E-500 verified on CC0 files, E-M5 Mark II / PEN-F high-res files decode smooth but CFA and black unverified), Kodak DCS520C TIFF (a two-component 12-bit lossless JPEG strip, the two components the even and odd columns; the file's `GrayResponseCurve` table linearises the companded samples to 12 bits, black measured per CFA position, white balance from the `KodakIFD`; `RawFormat::KodakDcs`, `crates/raw/src/vendor/kodak.rs`, checked on the one CC0 sample against its IFD0 thumbnail), Hasselblad 3FR and FFF (a TIFF without a DNG version tag that describes its CFA IFD, levels, crop and white balance with DNG's tags; `RawFormat::CfaTiff`, read by `dng::decode_as`, RGGB when the file states no `CFAPattern`): uncompressed (X1D, X1D II, X2D, CFV 100C) and Hasselblad's lossless JPEG with scan selection value 8 (every FFF; H3D, H4D and CFV-50c 3FR), `ljpeg::decode_hasselblad`, its rule measured on 14 CC0 files with the ljpeg-explorer tool, Samsung SRW without compression (NX5, NX10, NX11 and the NX20-generation bodies NX20, NX200, NX210, NX1000 and NX1100 as 12-bit packed samples in either bit order, EX1 and WB2000 as 16-bit words; layout, camera-JPEG framing, levels and white balance from the maker note, `crates/raw/src/vendor/srw.rs`; median ΔE vs the camera JPEG 2.2–7.6 on 11 of the 12 files from 10 bodies checked, 10.7 (5.7 registered) on the backlit WB2000 sample). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. Import probes (`probe_info`) of compressed NEFs decode only the first 128 rows, the ones the masked-trailing-column test reads in both the probe and the full decode (the D5100 and D7000 write 8 masked columns at 0 with no maker-note tag for them, compressed and uncompressed alike), instead of the whole Huffman stream: ~3 ms instead of the full decode time per file (386 ms on a 24 MP D750 file), issue #708. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop (EOS R6 Mark III `ColorData` version 66: as-shot white balance and black level from the file, white level from the data, `docs/cr3.md`) (the shot's aspect ratio from the maker note's `AspectInfo`); full sensor samples match the reference on M50/R100/R8/RP, incl. RP 1.6× crop mode (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). EOS R5 Mark II C-RAW and RAW decode too (the QP map's first-symbol literal and 23-zero escape, odd-height Bayer planes): 4 CC0 files pinned, every QP map of 33,665 private C-RAW files decodes exactly, and full-frame images match the camera JPEG. Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, compressed Samsung SRW (NX1, NX30, NX300, NX500, NX2000, NX3000, NX3300, NX mini: preview only with the reason), NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), Nikon High Efficiency NEF (HE / HE★ from the Z 8, Z 9, Z 6III and Z f: compression 34713 with a JPEG XS codestream and no `0x0096` table; no public specification, so preview only with the reason "Nikon High Efficiency NEF", issue #193; Lossless compressed NEFs from the same bodies decode), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), TIFF-based raws whose raw data sits in a private block (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and the Kodak DCS Pro / DCS760C TIFFs (compression 65000): IFD0 is only a reduced RGB image next to a much larger declared picture) are recognised from that structure and import as `preview_only` with the reason, instead of opening as a thumbnail-sized ordinary image (`thumbnail_shell` in `crates/raw/src/lib.rs`), Recognised raws LightCraft cannot decode open as `preview_only` with any usable embedded JPEG, also Canon CRW, Minolta MRW and Sigma X3F (containers recognised by magic, preview found by a structural JPEG scan), Kodak KDC (IFD0 without `ImageWidth`), Leaf MOS and Epson ERF (scan; ones with no JPEG still fail with "without an embedded preview", AVIF decode (HEIC decodes only behind the non-default `heif` feature, which the release packages enable: `crates/heif`, heic-rs for the container and HEVC (bit-exact against libde265 on iPhone tiles) plus our own colour conversion, matching libheif within 1 code value on iPhone-style grids (VUI full range, Display P3 ICC, nclx, 10-bit, irot/imir/clap, gain map/depth ignored); 4:2:0/monochrome only, 4:2:2/4:4:4 Canon/Sony HIF refused; same policy as PhotoCraft), palette (indexed-colour) TIFF (refused by the `tiff` crate); transparency is flattened onto white, in linear light, where the working image is built (`to_working` in `crates/codecs/src/lib.rs`): renders, previews, exports and contact sheets no longer turn transparent areas black, issue #1212; median ΔE vs the camera JPEG 2.2–7.6 on 11 of the 12 files from 10 bodies checked, 10.7 (5.7 registered) on the backlit WB2000 sample). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. Import probes (`probe_info`) of compressed NEFs decode only the first 128 rows, the ones the masked-trailing-column test reads in both the probe and the full decode (the D5100 and D7000 write 8 masked columns at 0 with no maker-note tag for them, compressed and uncompressed alike), instead of the whole Huffman stream: ~3 ms instead of the full decode time per file (386 ms on a 24 MP D750 file), issue #708. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop (the shot's aspect ratio from the maker note's `AspectInfo`); the Hasselblad Lunar, a Sony NEX-7 under Make HASSELBLAD, decodes as an ARW: a HASSELBLAD file with Sony's raw coding (compression 32767 or tag `0x7010`) goes to the ARW reader, and the body for its `Tag2010` layout comes from the maker note's `SonyModelID`, checked on the CC0 4051 file, issue #1275; an uncompressed MOS, whose IFD0 is the 16-bit mosaic with no Make tag, is recognised by Leaf's data block `0x8606` instead of opening as a greyscale image of its mosaic, issue #1275), and any decode failure of a recognised raw, not only an unsupported variant | -| LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files plus 6 exact-sensor CR3 cases and 4 pinned EOS R5 Mark II files (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; CR2 black levels were read from border columns light already reaches, left of `SensorInfo`'s border, so CR2s rendered dark with crushed shadows (EOS 6D: 2591 instead of 2047), now measured from the masked columns only, checked on 8 corpus bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies; their white balance now comes from the encrypted `SR2SubIFD`'s `WB_RGGBLevels`, the gains the camera applied, which 14 bodies (DSLR-A450 … A700, NEX-3/5/C3, SLT-A33/A35/A55, ILCE-6001, UMC-R10C) had no other source for and which the maker-note estimate read before missed by up to 100 % on preset shots, issue #535, checked against ExifTool on 100 decoded ARWs from 87 Sony bodies; the encrypted SR2 black level is now read from its directory entry (the DSLR-A450/A500/A550/A700 read 354–975 instead of 512), checked on 126 ARWs from 97 Sony bodies, and downsized lossless M/S ARWs use their measured black level 1024 instead of 512, checked on 12 files from 4 bodies, issue #535, and on a fifth, the ILCE-7CM2 (issue #697: 0.5.0 rendered its lossless M file with that 512 pedestal, the darkest half of a dusk scene +11 L* and blue, the lamplight white; its darkest 0.1 % per channel sits at 1017/1020/1024, and with 1024 the render is within +3 L* of the camera JPEG; the body's compressed and uncompressed "M-size" files are Bayer at 4736×3132 with black 512, two public ones in the corpus); pre-2017 Sony bodies shot in the camera's 16:9 mode (ILCE-7SM2, DSLR-A580) now get the 16:9 crop of their camera JPEG instead of the whole 3:2 frame with padding columns, issue #535; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked; private Z 8 crop verified; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone); the D7100, whose data holds picture beyond the camera's frame, develops at Lightroom's frame (the camera JPEG's size) in full frame and 1.3x crop, so Lightroom crops line up (#1248).; CR2 white levels came from masked rows holding samples at 16383, so on the EOS 50D, 60D, 450D (Rebel XSi), 500D (Rebel T1i), 550D (Rebel T2i), 1200D (Rebel T5) and 100D (Rebel SL1) clipped highlights sat at 81–98% of the range and were not treated as clipped, now taken from the active area only, issue #1272). Compressed RAF sensor arrays verified on 13 Fujifilm bodies (`docs/raf-compression.md`). ILCE-7RM6 ARW6 12-bit codes verified on 8 CC0 files (FF and APS-C, both compression modes). Sony ILCE-7CR (61 MP): lossless compressed L and M and compressed ARW2 samples of one scene in the corpus (black level, default crop, as-shot gains, lossless vs compressed mosaic agreement, fitted camera look), and all nine public codings (also uncompressed, small and APS-C crop) decode and fit. Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone). Pentax PEFs without area tags (KP, KF, K-3 Mark III, K-70, K-1, K-1 Mark II, K-S1, K-S2, 645Z) no longer keep their masked border: black, empty and saturated lines are trimmed against the maker-note BlackPoint, issue #1271, measured on 38 CC0 PEFs from 30 bodies.; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked). Compressed RAF sensor arrays verified on 13 Fujifilm bodies (`docs/raf-compression.md`). ILCE-7RM6 ARW6 12-bit codes verified on 8 CC0 files (FF and APS-C, both compression modes). Sony ILCE-7CR (61 MP): lossless compressed L and M and compressed ARW2 samples of one scene in the corpus (black level, default crop, as-shot gains, lossless vs compressed mosaic agreement, fitted camera look), and all nine public codings (also uncompressed, small and APS-C crop) decode and fit. Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone). The Canon PowerShot G9, which writes no `ColorData`, takes its as-shot white balance from the WBInfo row (`0x0029`) of the preset ShotInfo names, issue #1273, checked on 1 CC0 file against neutral areas of the camera JPEG (within 0.1 %).; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone). The Olympus E-10 (no ImageProcessing directory) takes its as-shot gains and its black level (`0x1012` in `0x102c` ValidBits units: 128 at 12 bits) from the main maker note, issue #1273: its render against the camera thumbnail goes from ΔE76 18.5 with a rejected fit to 11.5 with the fit accepted (measured by AlexChambers).; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone), and the Nikon D90 likewise on 10 of 550 NEFs (42 filler columns on the right that developed as a pink band; green clips below red and blue).; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone). | +| LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files plus 6 exact-sensor CR3 cases and 4 pinned EOS R5 Mark II files (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; CR2 black levels were read from border columns light already reaches, left of `SensorInfo`'s border, so CR2s rendered dark with crushed shadows (EOS 6D: 2591 instead of 2047), now measured from the masked columns only, checked on 8 corpus bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies; their white balance now comes from the encrypted `SR2SubIFD`'s `WB_RGGBLevels`, the gains the camera applied, which 14 bodies (DSLR-A450 … A700, NEX-3/5/C3, SLT-A33/A35/A55, ILCE-6001, UMC-R10C) had no other source for and which the maker-note estimate read before missed by up to 100 % on preset shots, issue #535, checked against ExifTool on 100 decoded ARWs from 87 Sony bodies; the encrypted SR2 black level is now read from its directory entry (the DSLR-A450/A500/A550/A700 read 354–975 instead of 512), checked on 126 ARWs from 97 Sony bodies, and downsized lossless M/S ARWs use their measured black level 1024 instead of 512, checked on 12 files from 4 bodies, issue #535, and on a fifth, the ILCE-7CM2 (issue #697: 0.5.0 rendered its lossless M file with that 512 pedestal, the darkest half of a dusk scene +11 L* and blue, the lamplight white; its darkest 0.1 % per channel sits at 1017/1020/1024, and with 1024 the render is within +3 L* of the camera JPEG; the body's compressed and uncompressed "M-size" files are Bayer at 4736×3132 with black 512, two public ones in the corpus); pre-2017 Sony bodies shot in the camera's 16:9 mode (ILCE-7SM2, DSLR-A580) now get the 16:9 crop of their camera JPEG instead of the whole 3:2 frame with padding columns, issue #535; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked; private Z 8 crop verified; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone); the D7100, whose data holds picture beyond the camera's frame, develops at Lightroom's frame (the camera JPEG's size) in full frame and 1.3x crop, so Lightroom crops line up (#1248).; CR2 white levels came from masked rows holding samples at 16383, so on the EOS 50D, 60D, 450D (Rebel XSi), 500D (Rebel T1i), 550D (Rebel T2i), 1200D (Rebel T5) and 100D (Rebel SL1) clipped highlights sat at 81–98% of the range and were not treated as clipped, now taken from the active area only, issue #1272). Compressed RAF sensor arrays verified on 13 Fujifilm bodies (`docs/raf-compression.md`). ILCE-7RM6 ARW6 12-bit codes verified on 8 CC0 files (FF and APS-C, both compression modes). Sony ILCE-7CR (61 MP): lossless compressed L and M and compressed ARW2 samples of one scene in the corpus (black level, default crop, as-shot gains, lossless vs compressed mosaic agreement, fitted camera look), and all nine public codings (also uncompressed, small and APS-C crop) decode and fit. Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone). Pentax PEFs without area tags (KP, KF, K-3 Mark III, K-70, K-1, K-1 Mark II, K-S1, K-S2, 645Z) no longer keep their masked border: black, empty and saturated lines are trimmed against the maker-note BlackPoint, issue #1271, measured on 38 CC0 PEFs from 30 bodies.; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked). Compressed RAF sensor arrays verified on 13 Fujifilm bodies (`docs/raf-compression.md`). ILCE-7RM6 ARW6 12-bit codes verified on 8 CC0 files (FF and APS-C, both compression modes). Sony ILCE-7CR (61 MP): lossless compressed L and M and compressed ARW2 samples of one scene in the corpus (black level, default crop, as-shot gains, lossless vs compressed mosaic agreement, fitted camera look), and all nine public codings (also uncompressed, small and APS-C crop) decode and fit. Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone). The Canon PowerShot G9, which writes no `ColorData`, takes its as-shot white balance from the WBInfo row (`0x0029`) of the preset ShotInfo names, issue #1273, checked on 1 CC0 file against neutral areas of the camera JPEG (within 0.1 %).; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone). The Olympus E-10 (no ImageProcessing directory) takes its as-shot gains and its black level (`0x1012` in `0x102c` ValidBits units: 128 at 12 bits) from the main maker note, issue #1273: its render against the camera thumbnail goes from ΔE76 18.5 with a rejected fit to 11.5 with the fit accepted (measured by AlexChambers).; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone), and the Nikon D90 likewise on 10 of 550 NEFs (42 filler columns on the right that developed as a pink band; green clips below red and blue). Every Nikon body without CropArea on raw.pixls.us is surveyed with `cargo xtask raw-frame`: 44 more model / size / depth groups whose data keeps masked or dark lines (D7000 and D5100 issue #976, D800 issue #914, D600 / D610 issue #1236, Df, D3, D3S, D700, D300, D300S, D810, D4S, D5600, Coolpix A, D3000, D40X, D60, D80, D2Xs, D800E, D4 12-bit, D200, D3200, Nikon 1 V1) crop to their camera JPEG's frame measured on raw.pixls.us CC0 samples and other public sample files; the D3100, D5000, D40, D50, D70 and D70s crop to best-guess frames, short of the evidence rule, until more samples confirm them.; the Nikon D4, which records no CropArea, gets its image area and white level measured on 148 NEFs against the camera JPEG (optical-black stripes and magenta clipped highlights gone). | | LR-IMP-PREVIEW-READ | Preview reads the header, not the raw | P0 | ✅ | `crates/raw/src/preview.rs` (`bounded_preview`, `embedded_preview_from_range`, `embedded_preview_covering`), `crates/engine/src/files.rs` (`preview_window`), `docs/raw-preview-reads.md`, `crates/engine/src/files.rs` (`preview_window`, `planned_preview`), `crates/raw/src/sparse.rs` (`preview_plan`) | a grid thumbnail and a loupe preview of an unedited raw show the camera's embedded JPEG, which sits near the start of the file (a 24 MP Fuji RAF keeps its 5.6 MB JPEG in a 56 MB file), so the preview loader reads the header and that JPEG instead of the whole file: 56.5 MB → 5.9 MB and ~10 s → 0.96 s measured on one real RAF over a GVFS/SMB Wi-Fi share (byte-identical preview pixels; `LIGHTCRAFT_PROFILE=1` reports the bytes); the RAF pointer is read from its fixed header fields, a CR3's boxes and a TIFF raw whose preview tag sits past the prefix are read through `lightcraft_raw::sparse::preview_plan`, which names the ranges `embedded_preview` reads and nothing else; a cached render as stand-in, an edited photo and a raw without an embedded JPEG still render; the formats it cannot describe (CRW/MRW/X3F are found by scanning, NEF/ORF/PEF probe from samples) keep the whole-file path, and a lying pointer, a short read or a file that changes underneath us falls back to it rather than losing the preview. An unedited raw's **grid thumbnail is that camera JPEG itself**, so no sensor decode is scheduled behind it (`Session::embedded_thumb_is_final`, `grid::skips_the_real_render`): drawing a 512 px thumbnail used to decode the whole raw — 45 s a photo at 1.2 MB/s, 8.3 GB for a 149-raw folder — and the queue it built delayed the render the user was waiting for. Stand-ins and grid thumbnails decode the smallest embedded JPEG that covers the requested size and is a copy of the largest (aspect ratio within 1 %, same component count), not always the largest, on the whole-file and the plan route alike: an A7R VI's 1616 × 1080 JPEG (0.3 MB) instead of its 9984 × 6656 one (5.6 MB), and mid-sized NEF / RW2 JPEGs | | LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic | | LR-IMP-MOVE | Move on import [Classic] | P1 | ✅ | `cmd:library.import` (`mode` move), `crates/engine/src/import_move.rs`, `crates/engine/src/tests_import_move.rs`, `crates/ui-egui/src/import.rs` | Transfer ▸ Move with Copy's destination, folders (day / month / one folder / custom template), rename template and example destination; XMP sidecars (both namings) move along; each source is removed only after its destination is written (hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate, unchecked files keep their sources, taken names get -1, -2…, never overwritten; files already in the destination/library are added in place; sources that can't be removed (read-only card) are kept and reported (`kept`). Undo removes the photos from the library but leaves the moved files at the destination. No Copy as DNG while moving | diff --git a/docs/raw-parity.md b/docs/raw-parity.md index 811353a70..ba4195923 100644 --- a/docs/raw-parity.md +++ b/docs/raw-parity.md @@ -1,6 +1,6 @@ # RAW parity: cameras, formats, demosaic, colour and lens profiles -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-13 · **Change:** minor (Nikon Z 6II / Z 8 / Z f in-file distortion correction; Leica Q3 verified by a recorded check of 38 private DNGs; EOS R5 Mark II decoded and verified: CRX QP-map escape, odd-height planes) · **Target:** Adobe Lightroom Classic 15.5 / Lightroom 9.6 (Camera Raw 18.7) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-14 · **Change:** minor (Nikon image areas of bodies without CropArea surveyed and measured, issues #976 / #914 / #1236; NEF decode gaps listed; Nikon Z 6II / Z 8 / Z f in-file distortion correction; Leica Q3 verified by a recorded check of 38 private DNGs; EOS R5 Mark II decoded and verified: CRX QP-map escape, odd-height planes) · **Target:** Adobe Lightroom Classic 15.5 / Lightroom 9.6 (Camera Raw 18.7) Raw processing is the deepest single dimension of Lightroom parity: a photographer whose camera does not decode, or decodes with the wrong colour, cannot switch. This document measures it model by @@ -93,7 +93,7 @@ committed table is stale. |---|---:|---:|---:|---| | Canon | 33% | 73% | 19% | 18 of the 22 popular bodies write CR3; 4 of them verified (M50, R8, R100, R5 Mark II), the rest at 0.6 | | Sony | 27% | 98% | 91% | bulk-checked | -| Nikon | 16% | 91% | 12% | decoders cover every NEF coding but HE; few bodies verified | +| Nikon | 16% | 91% | 12% | decoders cover NEF codings but HE and the compressed NEFs without a linearization table (D100, D1X, Coolpix P330 / P7700 / P7800, D4S and D810 small raw); few bodies verified; image area of every body without `CropArea` surveyed | | Fujifilm | 10% | 91% | 45% | | | Panasonic | 4% | 100% | 98% | bulk-checked | | OM / Olympus | 3% | 3% | 1% | compressed ORF | @@ -125,6 +125,8 @@ checked, and Canon, the largest bucket, is the weakest of the big four. | NEF | uncompressed, lossless, lossy type 1 / 2, lossy after split | ✅ | D5100, D7000, D7500, Z f corpus; ~60 NEFs by hand; #633; D4 and D90 image area and white level measured against the camera JPEG | | NEF | packed 14-bit stored under a compressed tag (Z 6II uncompressed) | ⬜ | #769; fix in PR #671 (not merged) | | NEF | High Efficiency / HE★ (JPEG XS) | ⬜ | named, preview (#193); needs a JPEG XS decoder | +| NEF | compressed without a linearization table (D100, D1X, Coolpix P330 / P7700 / P7800 NRW, D4S and D810 small raw) | ⬜ | refused as unsupported; one raw.pixls.us sample each | +| NEF | image area of bodies without `CropArea` | 🟡 | 47 model / size / depth groups cropped to their measured camera frame (D800 #914, D7000 #976, Df, D3, D300, D600, D3200 …); the D3100, D5000, D40, D50, D70 and D70s on best-guess frames until more samples confirm them (evidence in each entry's comment, `MEASURED_BODIES` in `crates/raw/src/vendor/nef.rs`). Checked and needing no entry, their data holding only picture beyond the camera frame: D750, D5300, D5500, D3300, D2X, D2Hs, D3X, D5200, Coolpix P6000, Nikon 1 AW1 / J3 / V2 / V3 and the D800's DX / 1.2x / 5:4 modes. `cargo xtask raw-frame FILE…` measures a body | | ARW | uncompressed, cRAW (ARW2), lossless LJ92 (L / M / S) | ✅ | 86 bodies (#535); M/S black level 1024; the Hasselblad Lunar, a NEX-7 under Make HASSELBLAD, routed by Sony's raw coding (#1275) | | ARW | packed 12-bit (A900) | ✅ | one CC0 file; A850 unmeasured | | ARW | ARW6 Compressed / HQ (compression 32766; ILCE-7RM6) | ✅ | 8 CC0 files; 12-bit mode unsupported; `docs/arw6-compression.md` | @@ -278,6 +280,7 @@ Beyond decoding: | 2026-10-12 | minor | Olympus E-10: as-shot white balance (`0x1017`/`0x1018`) and black level (`0x1012` in `0x102c` ValidBits units) from the main maker note, issue #1273; ΔE76 against its thumbnail 18.5 → 11.5, camera-look fit now accepted; counts unchanged | | 2026-10-13 | minor | Nikon D90 image area (4288 × 2848 at 10, 10) and white level (3830) measured on 10 of 550 NEFs from one body: its 42 filler columns no longer develop as a pink band | | 2026-10-12 | minor | Hasselblad Lunar verified (a Sony NEX-7 ARW under Make HASSELBLAD, now decoded): 236 verified, 232 preview; an uncompressed Leaf MOS is recognised as a raw (preview only) instead of opening as a greyscale image of its mosaic (#1275) | +| 2026-10-14 | minor | Nikon: every body without `CropArea` on raw.pixls.us surveyed; 44 more model / size / depth groups cropped to their measured camera frame (#976, #914, #1236), 6 of them on best-guess frames; NEF row no longer claims every coding but HE decodes (compressed NEFs without a linearization table listed) | | 2026-10-10 | minor | Compressed ORF: pointer to the two Olympus patents that describe the method, as a clean-room starting point | | 2026-10-12 | minor | PEF without area tags: masked border rows and columns trimmed against the BlackPoint (issue #1271; KP, KF, K-3 Mark III, K-70, K-1, K-1 Mark II, K-S1, K-S2, 645Z); counts unchanged | | 2026-10-13 | minor | Lens profiles: Nikon in-file distortion (NikonNEFInfo `0xc7d5`) for the Z 6II, Z 8 and Z f, validated on 83 frames from three lenses and the CC0 Z f sample (`docs/nikon-lens-corrections.md`); lens score unchanged | diff --git a/docs/whats-new.md b/docs/whats-new.md index 1bf44c3f4..285b00521 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -2,6 +2,13 @@ ## 2026-10-14 +### Nikon raws without coloured edge bars +- Raws from older Nikon bodies no longer show a thin coloured or dark bar along one edge (the right edge, or the top + of a portrait photo): masked sensor columns were being developed as part of the picture (issues #976, #914, + #1236). The D7000, D5100, D800, D800E, Df, D4, D3, D3S, D700, D300, D300S, D600, D610, D810, D4S, D5600, + Coolpix A, D3000, D40X, D60, D80, D2Xs, D200, D3200, D3100, D5000, D40, D50, D70, D70s and Nikon 1 V1 now show + the area the camera itself frames, a few pixels smaller than before. + ### Hard-edged Heal and Clone spots - A spot with Feather at 0 no longer draws black pixels exactly on its radius. This also fixes spots at 0% Opacity changing pixels along their edge. The hard edge now leaves pixels on the radius unchanged. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c392d0890..b301a1180 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -37,6 +37,11 @@ commands: corpus [--download] show where test corpora live; --download fetches PngSuite and CC0 raw samples (raw.pixls.us) into corpus/ and checks their sha256 stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`) + raw-frame [--json] FILE… + measure where each raw's embedded camera JPEG sits in its sensor data, which edge bands hold no + picture and what the decoder's crop keeps beyond the camera frame (any make with a full-size + embedded JPEG); print image-area entries for Nikon bodies that record no crop (MEASURED_BODIES in + crates/raw/src/vendor/nef.rs) "; fn main() -> ExitCode { @@ -54,6 +59,7 @@ fn main() -> ExitCode { Some("ci") => cmd_ci(), Some("corpus") => cmd_corpus(rest.contains(&"--download")), Some("stats") => stats::run(&root(), rest.contains(&"--exact")), + Some("raw-frame") => cmd_raw_frame(&rest), Some("-h" | "--help" | "help") | None => { print!("{USAGE}"); Ok(()) @@ -69,6 +75,20 @@ fn main() -> ExitCode { } } +/// `cargo xtask raw-frame FILE…`: runs the `raw_frame` example of `lightcraft-engine` on the files. +fn cmd_raw_frame(files: &[&str]) -> Result<(), String> { + if files.is_empty() { + return Err("raw-frame needs one or more raw files (several per camera, with detail across the frame)".into()); + } + eprintln!("$ cargo run --release -p lightcraft-engine --example raw_frame -- {}", files.join(" ")); + let status = cargo() + .args(["run", "--release", "-q", "-p", "lightcraft-engine", "--example", "raw_frame", "--"]) + .args(files) + .status() + .map_err(|e| format!("raw_frame: {e}"))?; + if status.success() { Ok(()) } else { Err("raw_frame failed".into()) } +} + /// Workspace root (parent of the xtask crate). pub fn root() -> PathBuf { // `cargo run` sets it at run time; prefer that over the compile-time value, which goes stale