From ac677a26244b4ed00d6dfe4f47bbff66706e42a0 Mon Sep 17 00:00:00 2001 From: Alex Chambers Date: Mon, 12 Oct 2026 04:42:22 +0200 Subject: [PATCH 1/2] LR-EDIT-OPTICS-PROFILE: decode Sony shading, CA and distortion tables with settings and provenance; apply nothing new Sony ARWs record shading, lateral chromatic aberration and distortion as a camera setting plus a radial table in the raw image IFD (0x7031-0x7037), with copies in the encrypted SR2SubIFD (0x797c-0x7982). They are now decoded for every body that records them and carried through RawInfo and RawImage as `lens_metadata.sony` (types in crates/raw/src/lens_metadata.rs, decoder in crates/raw/src/vendor/arw_lens.rs): - Raw-IFD fields are the ones the TIFF reader parsed, which is what the image itself uses. A scan of the directory adds diagnostics only: repeated tags, and why an entry the reader skipped was dropped (values outside the file, an unknown type, past the reader's limits). SR2SubIFD fields are read from their first entry. Only the exact layouts are accepted: SHORT[1] or non-negative SSHORT[1] settings, SSHORT[17] headed by 16 for shading and distortion, SSHORT[33] headed by 32 for CA. Nothing is allocated from a count in the file. - The raw IFD's copy wins. The SR2SubIFD copy fills in only for a tag the raw IFD lacks, never for a malformed one, and the two copies' agreement is recorded. - Settings keep their number and their documented meaning, Unknown(value) otherwise (never Off). - Framing evidence (stored frame, Sony and default crops, reference rectangle, coding, 3:2) and a per-component status. Only Applied components become opcodes. Applied corrections are unchanged: the ILCE-7RM4A distortion conversion receives exactly the table it received before, and no other body or component gains a correction. The ILCE-7CR distortion misses its camera JPEG at 24 mm, and shading and CA semantics are not validated on any body; see docs/sony-lens-corrections.md. Locating the SR2SubIFD, now done for every ARW that has one, selects the SR2Private offset, length and key with the TIFF reader's own rules, which lightcraft-tiff now exposes and builds its reader on (directory_entries, entry_span, ParseOptions::budget_for, decode_value). Only each field's first value is decoded, so the directory's other entries cost no memory sized by the file: a 16 MiB file whose SR2Private holds eight fields aliasing it peaks at 37 kB. DirectoryEntries keeps its layout private, and its positions() iterator uses checked arithmetic. Tests: synthetic decoder tests; a differential test of the SR2Private selection against the TIFF reader's in both byte orders (0 to 8,193 filler entries, unreadable and unusual first occurrences of each field, a used-up value budget); a dhat peak-heap test; public-corpus checks against the values ExifTool prints, with pinned and anchored SHA-256s (LIGHTCRAFT_CORPUS_REQUIRED=1 makes absent files fail); and an opt-in local inventory check (crates/raw/tests/sony_lens_inventory.rs). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/engine/src/camera_preview.rs | 3 + crates/engine/src/tests_xmp.rs | 1 + crates/merge/src/output.rs | 1 + crates/raw/Cargo.toml | 2 + crates/raw/src/binned.rs | 1 + crates/raw/src/color.rs | 1 + crates/raw/src/dng.rs | 1 + crates/raw/src/lens_metadata.rs | 187 ++++ crates/raw/src/lib.rs | 12 + crates/raw/src/saturation.rs | 1 + crates/raw/src/vendor/arw.rs | 48 +- crates/raw/src/vendor/arw_lens.rs | 1101 ++++++++++++++++++++++- crates/raw/src/vendor/cr2.rs | 1 + crates/raw/src/vendor/cr2/sraw.rs | 1 + crates/raw/src/vendor/cr3.rs | 1 + crates/raw/src/vendor/kodak.rs | 1 + crates/raw/src/vendor/nef.rs | 1 + crates/raw/src/vendor/orf.rs | 1 + crates/raw/src/vendor/pef.rs | 1 + crates/raw/src/vendor/raf.rs | 1 + crates/raw/src/vendor/rw2.rs | 1 + crates/raw/src/vendor/sr2.rs | 270 +++++- crates/raw/src/vendor/srw.rs | 1 + crates/raw/tests/corpus.rs | 223 +++++ crates/raw/tests/dng.rs | 1 + crates/raw/tests/robust.rs | 1 + crates/raw/tests/sony_lens_inventory.rs | 351 ++++++++ crates/raw/tests/sr2_heap.rs | 79 ++ crates/tiff/README.md | 4 + crates/tiff/src/lib.rs | 2 +- crates/tiff/src/reader.rs | 178 +++- docs/parity-checklist.md | 3 +- docs/sony-lens-corrections.md | 206 ++++- 35 files changed, 2536 insertions(+), 154 deletions(-) create mode 100644 crates/raw/src/lens_metadata.rs create mode 100644 crates/raw/tests/sony_lens_inventory.rs create mode 100644 crates/raw/tests/sr2_heap.rs diff --git a/Cargo.lock b/Cargo.lock index bf2ac824f..65dffed24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3418,6 +3418,7 @@ dependencies = [ name = "lightcraft-raw" version = "0.7.1" dependencies = [ + "dhat", "jpeg-encoder", "jxl-oxide", "lightcraft-color", diff --git a/Cargo.toml b/Cargo.toml index 7effbe0d9..1bf57fec8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ bytemuck = { version = "1", features = ["derive"] } crc32fast = "1" siphasher = "1" libc = "0.2" # dev-only: process CPU time in benches -dhat = "0.3" # opt-in heap profiling (lightcraft-cli --features dhat-heap); never in default builds +dhat = "0.3" # opt-in heap profiling (lightcraft-cli --features dhat-heap) and heap tests; never in product builds # formats quick-xml = "0.38" diff --git a/crates/engine/src/camera_preview.rs b/crates/engine/src/camera_preview.rs index c4824997c..5e70b58a9 100644 --- a/crates/engine/src/camera_preview.rs +++ b/crates/engine/src/camera_preview.rs @@ -1659,6 +1659,7 @@ mod tests { linearized: true, opcodes: OpcodeLists::default(), metadata: lightcraft_meta::Metadata::default(), + lens_metadata: Default::default(), }; let proxy = sensor_proxy(&raw, 2, 384).unwrap(); assert_eq!((proxy.width, proxy.height), (32, 32)); @@ -2002,6 +2003,7 @@ mod tests { linearized: false, opcodes: Default::default(), metadata: lightcraft_raw::Metadata { make: Some(make.to_owned()), model: Some(model.to_owned()), ..Default::default() }, + lens_metadata: Default::default(), } } @@ -3128,6 +3130,7 @@ mod tests { linearized: true, opcodes: OpcodeLists::default(), metadata: lightcraft_meta::Metadata { make: Some("SONY".into()), model: Some("ILCE-7M4".into()), ..Default::default() }, + lens_metadata: Default::default(), }; assert!(crate::camera_profiles::get("ILCE-7M4").is_some()); // the camera's rendering: a bright curve on the scene's luminance, as a grey (or tinted) 8-bit sRGB JPEG diff --git a/crates/engine/src/tests_xmp.rs b/crates/engine/src/tests_xmp.rs index 58d0d7a14..41927181b 100644 --- a/crates/engine/src/tests_xmp.rs +++ b/crates/engine/src/tests_xmp.rs @@ -586,6 +586,7 @@ pub(crate) fn synthetic_dng_of(w: usize, h: usize, cfa: &str, data: Vec, xm linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), }; write_dng(&raw, &DngWriteOptions { xmp: xmp.map(str::to_string), ..Default::default() }).unwrap() } diff --git a/crates/merge/src/output.rs b/crates/merge/src/output.rs index f8b7e7173..e15448646 100644 --- a/crates/merge/src/output.rs +++ b/crates/merge/src/output.rs @@ -113,6 +113,7 @@ pub fn write_linear_dng( linearized: false, opcodes: OpcodeLists { list3: lens.to_vec(), ..Default::default() }, metadata: meta, + lens_metadata: Default::default(), }; let compression = DngCompression::Deflate { tile: 256, half: samples == DngSamples::Half }; lightcraft_raw::write_dng(&raw, &DngWriteOptions { compression, ..Default::default() }).map_err(|e| MergeError::Output(e.to_string())) diff --git a/crates/raw/Cargo.toml b/crates/raw/Cargo.toml index 2ca9ef246..7b32cfe03 100644 --- a/crates/raw/Cargo.toml +++ b/crates/raw/Cargo.toml @@ -33,3 +33,5 @@ proptest = { workspace = true } zune-jpegxl = { workspace = true } # Pinned CC0 corpus identities; not a product dependency. sha2 = { version = "0.10", default-features = false, features = ["std"] } +# Peak-heap regression tests (tests/sr2_heap.rs); not a product dependency. +dhat = { workspace = true } diff --git a/crates/raw/src/binned.rs b/crates/raw/src/binned.rs index 674651c90..5eb99422b 100644 --- a/crates/raw/src/binned.rs +++ b/crates/raw/src/binned.rs @@ -165,6 +165,7 @@ mod tests { linearized: false, opcodes: OpcodeLists::default(), metadata: Metadata::default(), + lens_metadata: Default::default(), } } diff --git a/crates/raw/src/color.rs b/crates/raw/src/color.rs index a7cea813d..12c509190 100644 --- a/crates/raw/src/color.rs +++ b/crates/raw/src/color.rs @@ -289,6 +289,7 @@ mod tests { linearized: false, opcodes: Default::default(), metadata: Default::default(), + lens_metadata: Default::default(), } } diff --git a/crates/raw/src/dng.rs b/crates/raw/src/dng.rs index 026aae90d..b076c8bc3 100644 --- a/crates/raw/src/dng.rs +++ b/crates/raw/src/dng.rs @@ -340,6 +340,7 @@ pub(crate) fn decode_as(bytes: &[u8], mode: Mode, format: RawFormat) -> Result, +} + +/// Sony's in-file lens-correction record: three independent components, each a camera setting and a radial table. +/// +/// Tag identities and storage types from the ExifTool EXIF / Sony tag-name documentation: raw-IFD `0x7031` / `0x7032` +/// (shading, "VignettingCorrection" / "VignettingCorrParams"), `0x7034` / `0x7035` (lateral chromatic aberration), +/// `0x7036` / `0x7037` (distortion), and the `SR2SubIFD` tables `0x797d`, `0x7980`, `0x7982`. The `SR2SubIFD` setting +/// tags `0x797c`, `0x797f`, `0x7981` are unnamed there; they sit next to the tables and equal the raw-IFD settings +/// in every file inspected. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SonyLensCorrections { + /// Shading (vignetting): 16 radial samples whose gain function, radius convention and processing stage are not + /// established. Opaque signed data. + pub shading: SonyComponent<[i16; 16]>, + /// Lateral chromatic aberration: two 16-sample radial curves. A public analysis reads them as red, then blue, + /// relative to green; that is not validated here. + pub chromatic_aberration: SonyComponent<[[i16; 16]; 2]>, + /// Distortion: 16 radial samples (see `docs/sony-lens-corrections.md` for the validated interpretation). + pub distortion: SonyComponent<[i16; 16]>, + /// Whether the encrypted `SR2SubIFD` was found and decrypted. `false`: no copies to fall back on or compare. + pub sr2_read: bool, + /// The framing and coding the tables have to be interpreted against. + pub geometry: SonyLensGeometry, +} + +/// One correction component: the camera's setting, its table, and whether LightCraft applies it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SonyComponent { + pub setting: SonyField, + /// The table's samples after the leading sample count, which must equal their number. + pub table: SonyField, + pub status: SonyLensStatus, +} + +/// One field read from the raw IFD and from the `SR2SubIFD`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SonyField { + /// The value in use: the raw IFD's when it holds a well-formed one, else the `SR2SubIFD`'s when the raw IFD + /// lacks the tag. `None` when no copy is usable, and when the raw IFD's copy is malformed: a corrupt field is + /// reported, not silently replaced by the other copy. The raw IFD's copy is the one the TIFF reader that decodes + /// the image parsed (its first occurrence, within the reader's limits). + pub value: Option, + pub source: Option, + pub raw_ifd: SonyCopy, + pub sr2: SonyCopy, + /// Whether the two copies are equal, when both are well-formed. The raw IFD's wins when they differ. + pub copies_agree: Option, + /// Diagnostics: the tag occurs more than once in the raw IFD / the `SR2SubIFD`. Only the first occurrence is used, + /// as the TIFF reader does (the raw IFD's: the first one whose values it could read). + #[serde(default)] + pub raw_ifd_repeated: bool, + #[serde(default)] + pub sr2_repeated: bool, +} + +/// Where a value was read. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SonySource { + /// The plain raw image IFD. + RawIfd, + /// The encrypted `SR2SubIFD`. + Sr2SubIfd, +} + +/// How one copy of a field was found. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SonyCopy { + /// Not in that directory (or the `SR2SubIFD` could not be read). + Absent, + /// Present with exactly the supported type and layout. + Valid, + /// Present, but refused. + Rejected(SonyFieldError), +} + +/// Why a present field was refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SonyFieldError { + /// Another TIFF field type than SSHORT (8) for a table, or SHORT (3) / SSHORT for a setting. + FieldType(u16), + /// Another number of values than the supported layout's: 17 (shading, distortion), 33 (CA) or 1 (setting). + Count(u64), + /// The leading sample count is not the layout's 16 (32 for CA). Older bodies store 11 (22) samples padded to the + /// same length, a layout that is not interpreted; a count of 0 goes with an "unavailable" setting. + CountPrefix(i16), + /// The values lie outside the file (raw IFD) or the decrypted block (`SR2SubIFD`). + OutsideData, + /// In the raw IFD but past what the TIFF reader that decodes the image reads (8,192 entries per directory, a + /// total value budget), so the image doesn't use it either. + PastReaderLimits, + /// A setting stored as a negative SSHORT. + NegativeSetting(i16), +} + +/// A camera correction setting: the stored number and its documented meaning. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct SonySetting { + pub value: u16, + pub meaning: SonySettingMeaning, +} + +/// The documented meaning of a setting value (ExifTool EXIF tag-name documentation). Values are compared whole; +/// they are not read as bit masks. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SonySettingMeaning { + /// Shading 256, CA 0, distortion 0. The table is still recorded: Off does not mean absent. + Off, + /// Shading 257 or 272, CA 1, distortion 1. + Auto, + /// Distortion 17: Auto, fixed by the lens. + ForcedAuto, + /// "No correction params available": shading 511, CA 255, distortion 255. + Unavailable, + /// Any other value (e.g. a shading "Low" setting, whose number is not documented). Never treated as Off. + Unknown(u16), +} + +/// Whether a component is applied, and if not, why. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SonyLensStatus { + /// Converted into the image's `OpcodeList3` (today: ILCE-7RM4A Bayer distortion in native 3:2 framing). + Applied, + /// No usable table. + NoTable, + /// Decoded only: no independent validation covers this component's rendering semantics on any body (shading, + /// lateral CA). + NotValidatedComponent, + /// Decoded only: the component is validated on other camera models, not on this one. + NotValidatedModel, + /// Decoded only: the model is validated, but not this coding or framing (linear YCbCr, a non-3:2 crop) or a + /// table found only in the `SR2SubIFD`. + NotValidatedMode, + /// Validated scope, but the table is all zeros: a valid no-op, nothing to apply. + Neutral, + /// Validated scope, but the conversion refused this table (values outside the validated range, a degenerate + /// crop, or a fitted map that misses the table or folds). + Rejected, +} + +/// The framing and coding evidence for interpreting the tables, all in raw-frame pixels. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SonyLensGeometry { + /// The stored raw frame, including any padding (lossless L files are padded). + pub storage_width: usize, + pub storage_height: usize, + /// Sony's crop tags (`0x74c7` / `0x74c8`) as recorded. + pub sony_crop: Option, + /// The raw IFD's `DefaultCropOrigin` / `DefaultCropSize` as recorded. + pub default_crop: Option, + /// The image's crop (the decoder's precedence: Sony's tags, then the default crop, then the recorded size). + pub image_crop: Rect, + /// The rectangle the validated distortion conversion is centred on and normalised to: the default crop when + /// the file has one, else the image crop. Clipped to the frame. + pub reference: Rect, + /// TIFF compression of the raw image (1 uncompressed, 7 lossless, 32767 ARW2, 32766 ARW6). + pub compression: u16, + /// Downsized lossless M/S data (linear YCbCr, not sensor CFA). + pub linear_ycbcr: bool, + /// Both the image crop and the reference are 3:2 within 0.5 %, the only framing validated. + pub native_3_2: bool, +} + +impl SonyField { + /// Whether either directory holds the tag (well-formed or not). + pub fn is_present(&self) -> bool { + self.raw_ifd != SonyCopy::Absent || self.sr2 != SonyCopy::Absent + } +} diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs index 0f366648c..29ace7025 100644 --- a/crates/raw/src/lib.rs +++ b/crates/raw/src/lib.rs @@ -35,6 +35,7 @@ pub mod highlight; #[cfg(feature = "jxl")] mod jxl; pub mod kelvin_presets; +pub mod lens_metadata; pub mod ljpeg; pub(crate) mod llvc; pub mod opcodes; @@ -50,6 +51,7 @@ mod vendor; pub use demosaic::{Method, demosaic}; pub use dngwrite::{DngCompression, DngWriteOptions, write_dng, write_rendered_dng}; +pub use lens_metadata::LensMetadata; pub use lightcraft_color::Mat3; pub use lightcraft_geom::Orientation; pub use lightcraft_meta::Metadata; @@ -415,6 +417,9 @@ pub struct RawInfo { pub wb_multipliers: Option<[f32; 3]>, pub opcodes: OpcodeLists, pub metadata: Metadata, + /// Lens-correction tables the file records, decoded but not necessarily applied (see [`crate::lens_metadata`]). + #[serde(default)] + pub lens_metadata: LensMetadata, } impl RawInfo { @@ -648,6 +653,10 @@ pub struct RawImage { pub linearized: bool, pub opcodes: OpcodeLists, pub metadata: Metadata, + /// Lens-correction tables the file records, decoded but not necessarily applied (see [`crate::lens_metadata`]); only + /// validated components are in `opcodes`. + #[serde(default)] + pub lens_metadata: LensMetadata, } /// Black-subtracted, white-normalised data covering the active area. @@ -679,6 +688,7 @@ impl RawImage { wb_multipliers: self.wb_multipliers, opcodes: self.opcodes.clone(), metadata: self.metadata.clone(), + lens_metadata: self.lens_metadata.clone(), } } @@ -698,6 +708,7 @@ impl RawImage { wb_multipliers: self.wb_multipliers, opcodes: self.opcodes, metadata: self.metadata, + lens_metadata: self.lens_metadata, } } @@ -862,6 +873,7 @@ mod tests { linearized: false, opcodes: OpcodeLists::default(), metadata: Metadata::default(), + lens_metadata: Default::default(), }; let direct = raw.develop(Method::Bilinear).unwrap(); let by_hand = raw.finish_demosaiced(demosaic(&raw.normalized().unwrap(), Method::Bilinear)); diff --git a/crates/raw/src/saturation.rs b/crates/raw/src/saturation.rs index aaa0a6dff..261d5023b 100644 --- a/crates/raw/src/saturation.rs +++ b/crates/raw/src/saturation.rs @@ -137,6 +137,7 @@ mod tests { linearized: false, opcodes: OpcodeLists::default(), metadata: Metadata::default(), + lens_metadata: Default::default(), } } diff --git a/crates/raw/src/vendor/arw.rs b/crates/raw/src/vendor/arw.rs index 59a9cf59e..7a4e0cc45 100644 --- a/crates/raw/src/vendor/arw.rs +++ b/crates/raw/src/vendor/arw.rs @@ -44,8 +44,9 @@ //! through the generic TIFF path. use super::arw6; +use crate::lens_metadata::SonyLensGeometry; use crate::tiffraw::{Packing, check_image, read_image_in}; -use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg}; +use crate::{BlackLevel, Cfa, ColorData, LensMetadata, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg}; use lightcraft_geom::Orientation; use lightcraft_tiff::image::{Chunk, ImageInfo, Layout, chunk_bytes}; use lightcraft_tiff::tags::{self as t, photometric}; @@ -552,6 +553,14 @@ fn dng_default_crop(raw: &Ifd, w: usize, h: usize) -> Option { None } +/// A rectangle exactly as an origin tag and a size tag record it (two values each), when both exist and fit `usize`. +fn recorded_rect(raw: &Ifd, origin: u16, size: u16) -> Option { + let (Some(origin), Some(size)) = (raw.u64s(origin), raw.u64s(size)) else { return None }; + let (&[x, y], &[w, h]) = (origin.as_slice(), size.as_slice()) else { return None }; + let n = |v: u64| usize::try_from(v).ok(); + Some(Rect::new(n(x)?, n(y)?, n(w)?, n(h)?)) +} + /// The image area for files without Sony's crop tags (`0x74c7/0x74c8`, written since about 2017): the DNG-style /// default crop when the raw IFD has one, else the image size the camera records (see [`recorded_size_crop`]). /// @@ -748,6 +757,25 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { let mut metadata = lightcraft_meta::from_tiff(&tiff); metadata.width = Some(crop.width as u32); metadata.height = Some(crop.height as u32); + // The lens-correction record. The distortion table is centred on the DNG-style default crop when the file has + // one: on the ILCE-7RM4A it starts at x = 32 where Sony's crop tags (the image crop, unchanged) start at 0, and + // the warp was validated against Sony's exports with that centre. The opcode's centre is in active-area + // coordinates, so it stays on the optical centre whichever crop frames the image. + let reference = dng_default_crop(raw, w, h).unwrap_or(crop); + let geometry = SonyLensGeometry { + storage_width: w, + storage_height: h, + sony_crop: recorded_rect(raw, CROP_TOP_LEFT, CROP_SIZE), + default_crop: recorded_rect(raw, t::DEFAULT_CROP_ORIGIN, t::DEFAULT_CROP_SIZE), + image_crop: crop, + reference, + compression: info.compression, + linear_ycbcr: linear_rgb, + native_3_2: super::arw_lens::is_3_2(crop) && super::arw_lens::is_3_2(reference), + }; + // decrypted for the copies' agreement check too, not only when a plain field is missing: a few dozen kB, found by + // reading only three `SR2Private` entries (see `sr2::SubIfd::read`) + let (sony_lens, warp) = super::arw_lens::corrections(bytes, tiff.order, (raw, tiff.bigtiff), sr2(), &model, geometry); let img = RawImage { format: RawFormat::Arw, width: w, @@ -767,22 +795,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { // makes a neutral surface magenta. WB edits remain relative to this as-shot RGB. wb_multipliers: if linear_rgb { Some([1.0; 3]) } else { wb }, linearized: false, - // The distortion table is centred on the DNG-style default crop when the file has one: on the ILCE-7RM4A - // it starts at x = 32 where Sony's crop tags (the image crop, unchanged) start at 0, and the warp was - // validated against Sony's exports with that centre. The opcode's centre is in active-area coordinates, so - // it stays on the optical centre whichever crop frames the image. - opcodes: OpcodeLists { - // DefaultCrop can retain native geometry while Sony's tags select an aspect crop. - // Both the image framing and the table geometry must stay within the validated 3:2 scope. - list3: if linear_rgb || crop.height == 0 || (crop.width as f64 / crop.height as f64 - 1.5).abs() > 0.005 { - Vec::new() - } else { - let geometry = dng_default_crop(raw, w, h).unwrap_or(crop); - super::arw_lens::distortion(&model, raw, Rect::new(0, 0, w, h), geometry).into_iter().collect() - }, - ..Default::default() - }, + // only validated components become opcodes (see `arw_lens`) + opcodes: OpcodeLists { list3: warp.into_iter().collect(), ..Default::default() }, metadata, + lens_metadata: LensMetadata { sony: sony_lens }, }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/arw_lens.rs b/crates/raw/src/vendor/arw_lens.rs index f4e02d69d..aa7313e23 100644 --- a/crates/raw/src/vendor/arw_lens.rs +++ b/crates/raw/src/vendor/arw_lens.rs @@ -1,28 +1,348 @@ -//! Sony's plain raw-IFD distortion table, expressed as a DNG warp for the shared optics pipeline. +//! Sony's in-file lens-correction records (shading, lateral chromatic aberration, distortion): bounded decoding of +//! the raw image IFD's fields and their encrypted `SR2SubIFD` copies into [`SonyLensCorrections`], and the one +//! validated conversion, the ILCE-7RM4A's distortion table, into a DNG warp for the shared optics pipeline. //! -//! Tag identity/layout: https://exiftool.org/TagNames/EXIF.html (`0x7037`, signed shorts). -//! Geometry inferred independently from files and Sony Imaging Edge exports; see docs/sony-lens-corrections.md. -//! No lens database or external decoder code. The unrelated MakerNote table has a different scale. +//! Tag identity/layout: https://exiftool.org/TagNames/EXIF.html (`0x7031` to `0x7037`, signed shorts) and +//! https://exiftool.org/TagNames/Sony.html#SR2SubIFD (`0x797d`, `0x7980`, `0x7982`). The `SR2SubIFD` setting tags +//! `0x797c`, `0x797f` and `0x7981` sit next to those tables, unnamed there; they equal the raw IFD's settings in every +//! file inspected. Geometry inferred independently from files and Sony Imaging Edge exports; see +//! docs/sony-lens-corrections.md. No lens database and no external decoder code. The MakerNote's similarly named +//! tables have another scale and layout and are never read here. +//! +//! A raw-IFD field is the one the TIFF reader parsed, the copy the image itself uses (its first occurrence, within +//! the reader's limits). The directory is also scanned, for diagnostics only: whether a tag is repeated, and why an +//! entry the reader left out was dropped (values outside the file, an unknown type, or past the reader's limits). +//! `SR2SubIFD` fields are read straight from their first entry. Only the exact layouts are accepted: SHORT\[1\] (or +//! nonnegative SSHORT\[1\]) settings, SSHORT\[17\] headed by 16 for shading and distortion, SSHORT\[33\] headed by +//! 32 for CA. Nothing is allocated from a count. The raw IFD's copy wins; the `SR2SubIFD`'s is used only when the raw +//! IFD lacks the tag, and a malformed raw-IFD copy is reported, not replaced. Decoding grants nothing: only components +//! whose interpretation was independently validated for the model and mode are converted ([`SonyLensStatus`]), and +//! the conversion receives exactly the table the decoder applied before this record existed. +use super::sr2::SubIfd; +use crate::lens_metadata::{ + SonyComponent, SonyCopy, SonyField, SonyFieldError, SonyLensCorrections, SonyLensGeometry, SonyLensStatus, SonySetting, SonySettingMeaning, + SonySource, +}; use crate::{Opcode, Rect}; -use lightcraft_tiff::{Ifd, Value}; +use lightcraft_tiff::{ByteOrder, FieldType, Ifd, Value}; -const DISTORTION: u16 = 0x7037; +/// Each component's setting and table tags, `(raw IFD, SR2SubIFD)`. +const SHADING: Tags = Tags { setting: (0x7031, 0x797c), table: (0x7032, 0x797d) }; +const CA: Tags = Tags { setting: (0x7034, 0x797f), table: (0x7035, 0x7980) }; +const DISTORTION: Tags = Tags { setting: (0x7036, 0x7981), table: (0x7037, 0x7982) }; +/// TIFF field types. +const SHORT: u16 = 3; +const SSHORT: u16 = 8; +/// Camera models whose raw-IFD distortion table was independently validated. Exact names: ILCE-7RM4 and ILCE-7RM4A +/// are distinct models, not interchangeable aliases. +const DISTORTION_MODELS: &[&str] = &["ILCE-7RM4A"]; +/// Most directory entries scanned: a classic TIFF directory's largest count, also bounding a BigTIFF one. +const MAX_ENTRIES: usize = 1 << 16; const SAMPLES: usize = 1024; const MAX_ERROR: f64 = 0.0005; -/// Only independently validated camera models and the 16-entry layout. Unknown models/layouts -/// and implausible/non-smooth maps are left uncorrected. See docs/sony-lens-corrections.md. -/// A camera setting of Off does not remove the lens data: the user can enable it in the developer. -pub(super) fn distortion(model: &str, ifd: &Ifd, active: Rect, crop: Rect) -> Option { +#[derive(Clone, Copy)] +struct Tags { + setting: (u16, u16), + table: (u16, u16), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Shading, + Ca, + Distortion, +} + +/// The documented meaning of a setting value (ExifTool EXIF tag-name documentation), compared whole. +fn meaning(kind: Kind, value: u16) -> SonySettingMeaning { + use Kind::{Ca, Distortion, Shading}; + use SonySettingMeaning::*; + match (kind, value) { + (Shading, 256) | (Ca | Distortion, 0) => Off, + (Shading, 257 | 272) | (Ca | Distortion, 1) => Auto, + (Distortion, 17) => ForcedAuto, + (Shading, 511) | (Ca | Distortion, 255) => Unavailable, + _ => Unknown(value), + } +} + +/// One copy of a field. +#[derive(Clone, Debug, PartialEq)] +enum Read { + Absent, + Valid(T), + Rejected(SonyFieldError), +} + +impl Read { + fn map(self, f: impl FnOnce(T) -> U) -> Read { + match self { + Read::Absent => Read::Absent, + Read::Valid(v) => Read::Valid(f(v)), + Read::Rejected(e) => Read::Rejected(e), + } + } + fn copy(&self) -> SonyCopy { + match self { + Read::Absent => SonyCopy::Absent, + Read::Valid(_) => SonyCopy::Valid, + Read::Rejected(e) => SonyCopy::Rejected(*e), + } + } +} + +/// One copy of a field, and whether its directory holds the tag more than once. +struct Found { + read: Read, + repeated: bool, +} + +impl Found { + fn map(self, f: impl FnOnce(T) -> U) -> Found { + Found { read: self.read.map(f), repeated: self.repeated } + } +} + +/// A TIFF directory whose out-of-line values sit at `offset - base` in `data`: the raw IFD in the file (`base` 0), or +/// the decrypted `SR2SubIFD`, whose offsets are file offsets into the block itself. +struct Directory<'a> { + /// From the entry count on. + entries: &'a [u8], + data: &'a [u8], + base: usize, + order: ByteOrder, + /// BigTIFF layout: 8-byte entry count, 20-byte entries with an 8-byte count and value field. + big: bool, +} + +impl<'a> Directory<'a> { + /// The first entry of `tag`, and whether another follows. Only the first counts, as in the TIFF reader and + /// `sr2.rs`. Reads at most the entries the data holds, and at most [`MAX_ENTRIES`]. + fn entry(&self, tag: u16) -> (Option<&'a [u8]>, bool) { + let o = self.order; + let (n, head, size) = if self.big { (o.read_u64(self.entries, 0), 8, 20) } else { (o.read_u16(self.entries, 0).map(u64::from), 2, 12) }; + let Some(n) = n else { return (None, false) }; + let mut found = None; + for i in 0..usize::try_from(n).unwrap_or(usize::MAX).min(MAX_ENTRIES) { + let at = i.checked_mul(size).and_then(|at| at.checked_add(head)); + let Some(e) = at.and_then(|at| self.entries.get(at..at.checked_add(size)?)) else { break }; + if o.read_u16(e, 0) == Some(tag) { + if found.is_some() { + return (found, true); + } + found = Some(e); + } + } + (found, false) + } + + /// Field type and count of entry `e`. + fn field(&self, e: &[u8]) -> (u16, u64) { + let o = self.order; + let count = if self.big { o.read_u64(e, 4) } else { o.read_u32(e, 4).map(u64::from) }; + (o.read_u16(e, 2).unwrap_or(0), count.unwrap_or(u64::MAX)) + } + + /// `len` value bytes of entry `e`: inline when they fit the value field, else at the entry's offset. + fn values(&self, e: &'a [u8], len: usize) -> Option<&'a [u8]> { + let (field, width) = if self.big { (12, 8) } else { (8, 4) }; + if len <= width { + return e.get(field..field + len); + } + let offset = if self.big { self.order.read_u64(e, field)? } else { u64::from(self.order.read_u32(e, field)?) }; + let at = usize::try_from(offset).ok()?.checked_sub(self.base)?; + self.data.get(at..at.checked_add(len)?) + } + + /// Why the TIFF reader left out entry `e`, which the directory holds: a field type it doesn't know, values outside + /// the data, or else its own limits (the entries it reads per directory, its value budget). + fn unread(&self, e: &'a [u8]) -> SonyFieldError { + let (kind, count) = self.field(e); + let Some(size) = FieldType::from_u16(kind).map(FieldType::size) else { return SonyFieldError::FieldType(kind) }; + match count.checked_mul(size as u64).and_then(|len| usize::try_from(len).ok()).and_then(|len| self.values(e, len)) { + Some(_) => SonyFieldError::PastReaderLimits, + None => SonyFieldError::OutsideData, + } + } + + /// A radial table of exactly `N` samples in entry `e`: SSHORT\[N + 1\], the first value `N`. + fn table(&self, e: &'a [u8]) -> Read<[i16; N]> { + let (kind, count) = self.field(e); + if kind != SSHORT { + return Read::Rejected(SonyFieldError::FieldType(kind)); + } + if count != (N + 1) as u64 { + return Read::Rejected(SonyFieldError::Count(count)); + } + let Some(bytes) = self.values(e, 2 * (N + 1)) else { return Read::Rejected(SonyFieldError::OutsideData) }; + let at = |i: usize| self.order.read_u16(bytes, 2 * i).map_or(0, |v| v as i16); + let prefix = at(0); + if usize::try_from(prefix).ok() != Some(N) { + return Read::Rejected(SonyFieldError::CountPrefix(prefix)); + } + Read::Valid(std::array::from_fn(|i| at(i + 1))) + } + + /// A setting in entry `e`: SHORT\[1\], or SSHORT\[1\] when not negative. + fn setting(&self, e: &'a [u8], kind: Kind) -> Read { + let (field_type, count) = self.field(e); + if field_type != SHORT && field_type != SSHORT { + return Read::Rejected(SonyFieldError::FieldType(field_type)); + } + if count != 1 { + return Read::Rejected(SonyFieldError::Count(count)); + } + let Some(value) = self.values(e, 2).and_then(|b| self.order.read_u16(b, 0)) else { + return Read::Rejected(SonyFieldError::OutsideData); + }; + if field_type == SSHORT && (value as i16) < 0 { + return Read::Rejected(SonyFieldError::NegativeSetting(value as i16)); + } + Read::Valid(SonySetting { value, meaning: meaning(kind, value) }) + } +} + +/// A table as the TIFF reader parsed it: SSHORT\[N + 1\], the first value `N`. +fn parsed_table(value: &Value) -> Read<[i16; N]> { + let Value::SShort(v) = value else { return Read::Rejected(SonyFieldError::FieldType(value.field_type() as u16)) }; + if v.len() != N + 1 { + return Read::Rejected(SonyFieldError::Count(v.len() as u64)); + } + let at = |i: usize| v.get(i).copied().unwrap_or(0); + if usize::try_from(at(0)).ok() != Some(N) { + return Read::Rejected(SonyFieldError::CountPrefix(at(0))); + } + Read::Valid(std::array::from_fn(|i| at(i + 1))) +} + +/// A setting as the TIFF reader parsed it: SHORT\[1\], or SSHORT\[1\] when not negative (the documented and the stored +/// type differ). +fn parsed_setting(value: &Value, kind: Kind) -> Read { + let number = match value { + Value::Short(v) if v.len() != 1 => return Read::Rejected(SonyFieldError::Count(v.len() as u64)), + Value::SShort(v) if v.len() != 1 => return Read::Rejected(SonyFieldError::Count(v.len() as u64)), + Value::Short(v) => v.first().copied(), + Value::SShort(v) => match v.first().copied() { + Some(n) if n < 0 => return Read::Rejected(SonyFieldError::NegativeSetting(n)), + n => n.map(|n| n as u16), + }, + other => return Read::Rejected(SonyFieldError::FieldType(other.field_type() as u16)), + }; + number.map_or(Read::Rejected(SonyFieldError::Count(0)), |value| Read::Valid(SonySetting { value, meaning: meaning(kind, value) })) +} + +/// The raw IFD's copy of `tag`: the one the TIFF reader parsed, which is what the image itself uses. The directory +/// itself only adds diagnostics: whether the tag is repeated, and why an entry the reader left out was dropped. +fn raw_copy(ifd: &Ifd, dir: Option<&Directory<'_>>, tag: u16, read: impl Fn(&Value) -> Read) -> Found { + let (entry, repeated) = dir.map_or((None, false), |d| d.entry(tag)); + let read = match (ifd.get(tag), entry.zip(dir)) { + (Some(parsed), _) => read(&parsed.value), + (None, Some((e, d))) => Read::Rejected(d.unread(e)), + (None, None) => Read::Absent, + }; + Found { read, repeated } +} + +/// The `SR2SubIFD`'s copy of `tag`, read straight from its first entry. +fn sr2_copy<'a, T>(dir: Option<&Directory<'a>>, tag: u16, read: impl Fn(&Directory<'a>, &'a [u8]) -> Read) -> Found { + let Some(d) = dir else { return Found { read: Read::Absent, repeated: false } }; + let (entry, repeated) = d.entry(tag); + Found { read: entry.map_or(Read::Absent, |e| read(d, e)), repeated } +} + +/// One field from its two copies: the raw IFD's when well-formed, else the `SR2SubIFD`'s when the raw IFD lacks the +/// tag. +fn combine(raw: Found, sr2: Found) -> SonyField { + let (raw_ifd, sr2_copy) = (raw.read.copy(), sr2.read.copy()); + let copies_agree = match (&raw.read, &sr2.read) { + (Read::Valid(a), Read::Valid(b)) => Some(a == b), + _ => None, + }; + let (value, source) = match (raw.read, sr2.read) { + (Read::Valid(v), _) => (Some(v), Some(SonySource::RawIfd)), + (Read::Absent, Read::Valid(v)) => (Some(v), Some(SonySource::Sr2SubIfd)), + _ => (None, None), + }; + SonyField { value, source, raw_ifd, sr2: sr2_copy, copies_agree, raw_ifd_repeated: raw.repeated, sr2_repeated: sr2.repeated } +} + +/// Whether `r` has the native 3:2 framing, within 0.5 %. +pub(super) fn is_3_2(r: Rect) -> bool { + r.height > 0 && (r.width as f64 / r.height as f64 - 1.5).abs() <= 0.005 +} + +/// Decode the file's record and the validated distortion warp. `raw` is the raw image IFD as the TIFF reader parsed +/// it (in a BigTIFF file when `big`), `sr2` the decrypted `SR2SubIFD`. The record is `None` when neither directory +/// holds any of the six fields. +pub(super) fn corrections( + bytes: &[u8], + order: ByteOrder, + (raw, big): (&Ifd, bool), + sr2: Option<&SubIfd>, + model: &str, + geometry: SonyLensGeometry, +) -> (Option, Option) { + let plain = usize::try_from(raw.offset).ok().and_then(|at| Some(Directory { entries: bytes.get(at..)?, data: bytes, base: 0, order, big })); + let encrypted = sr2.map(|s| { + let (block, start) = s.block(); + Directory { entries: block, data: block, base: start, order: s.order(), big: false } + }); + let (p, s) = (plain.as_ref(), encrypted.as_ref()); + // CA: two contiguous 16-sample curves after the count + let split = |v: [i16; 32]| -> [[i16; 16]; 2] { std::array::from_fn(|c| std::array::from_fn(|i| v[16 * c + i])) }; + let table16 = |tags: (u16, u16)| combine(raw_copy(raw, p, tags.0, parsed_table::<16>), sr2_copy(s, tags.1, |d, e| d.table::<16>(e))); + let shading_table = table16(SHADING.table); + let distortion_table = table16(DISTORTION.table); + let ca_table = combine(raw_copy(raw, p, CA.table.0, parsed_table::<32>).map(split), sr2_copy(s, CA.table.1, |d, e| d.table::<32>(e)).map(split)); + let setting = |tags: Tags, kind: Kind| { + combine(raw_copy(raw, p, tags.setting.0, |v| parsed_setting(v, kind)), sr2_copy(s, tags.setting.1, |d, e| d.setting(e, kind))) + }; + let settings = [setting(SHADING, Kind::Shading), setting(CA, Kind::Ca), setting(DISTORTION, Kind::Distortion)]; + if !shading_table.is_present() && !ca_table.is_present() && !distortion_table.is_present() && !settings.iter().any(SonyField::is_present) { + return (None, None); + } + let [shading_setting, ca_setting, distortion_setting] = settings; + // Only the validated scope reaches the conversion: the raw IFD's table, as the TIFF reader parsed it, of a Bayer + // image whose image crop and reference are native 3:2 (DefaultCrop can keep native geometry while Sony's tags + // select an aspect crop). + let validated_model = DISTORTION_MODELS.contains(&model); + let in_mode = !geometry.linear_ycbcr && geometry.native_3_2 && distortion_table.source == Some(SonySource::RawIfd); + let (w, h) = (geometry.storage_width, geometry.storage_height); + let warp = match &distortion_table.value { + Some(table) if in_mode => distortion(model, table, Rect::new(0, 0, w, h), geometry.reference), + _ => None, + }; + let distortion_status = match &distortion_table.value { + None => SonyLensStatus::NoTable, + Some(_) if warp.is_some() => SonyLensStatus::Applied, + Some(_) if !validated_model => SonyLensStatus::NotValidatedModel, + Some(_) if !in_mode => SonyLensStatus::NotValidatedMode, + Some(table) if table.iter().all(|v| *v == 0) => SonyLensStatus::Neutral, + Some(_) => SonyLensStatus::Rejected, + }; + // No independent validation of shading or CA semantics exists on any body: decoded, never applied. + let unvalidated = |present: bool| if present { SonyLensStatus::NotValidatedComponent } else { SonyLensStatus::NoTable }; + let record = SonyLensCorrections { + shading: SonyComponent { status: unvalidated(shading_table.value.is_some()), setting: shading_setting, table: shading_table }, + chromatic_aberration: SonyComponent { status: unvalidated(ca_table.value.is_some()), setting: ca_setting, table: ca_table }, + distortion: SonyComponent { setting: distortion_setting, table: distortion_table, status: distortion_status }, + sr2_read: sr2.is_some(), + geometry, + }; + (Some(record), warp) +} + +/// The validated conversion of a raw-IFD distortion table into a `WarpRectilinear`. Only independently validated +/// camera models and the 16-entry layout; unknown models and implausible or non-smooth maps are left uncorrected. +/// See docs/sony-lens-corrections.md. A camera setting of Off does not remove the lens data: the user can enable it +/// in the developer. +fn distortion(model: &str, table: &[i16; 16], active: Rect, crop: Rect) -> Option { // A matching table shape does not establish its scale or radial normalization on another body. - // Keep this exact: ILCE-7RM4 and ILCE-7RM4A are distinct models, not interchangeable aliases. - if model != "ILCE-7RM4A" { + if !DISTORTION_MODELS.contains(&model) { return None; } - let Value::SShort(values) = &ifd.get(DISTORTION)?.value else { return None }; - let [16, table @ ..] = values.as_slice() else { return None }; - let table: &[i16; 16] = table.try_into().ok()?; if table.iter().all(|v| *v == 0) || table.iter().any(|v| !(-4096..=4096).contains(v)) { return None; } @@ -122,22 +442,720 @@ fn solve4(mut a: [[f64; 4]; 4], mut b: [f64; 4]) -> Option<[f64; 4]> { #[cfg(test)] mod tests { use super::*; - use lightcraft_tiff::Entry; + use crate::lens_metadata::SonyCopy::{Absent, Rejected, Valid}; + use crate::lens_metadata::SonyFieldError::*; + use crate::{LensMetadata, Mode, RawImage}; + use lightcraft_tiff::ByteOrder::{Big, Little}; + use lightcraft_tiff::{IfdBuilder, ImageData, Tiff, TiffWriter, Value, tags as t}; - fn ifd(value: Value) -> Ifd { - Ifd { entries: vec![Entry { tag: DISTORTION, value, offset: 0 }], ..Default::default() } - } - fn table(values: &[i16]) -> Ifd { - ifd(Value::SShort(std::iter::once(16).chain(values.iter().copied()).collect())) - } const TELE: [i16; 16] = [-2, 1, 7, 17, 31, 48, 68, 93, 120, 152, 187, 225, 265, 309, 356, 403]; const WIDE: [i16; 16] = [14, 0, -27, -59, -106, -158, -223, -291, -367, -440, -519, -593, -667, -731, -792, -839]; const AREA: Rect = Rect { x: 0, y: 0, width: 9504, height: 6336 }; + /// The shading samples of the public ILCE-7CR samples (raw.pixls.us DSC00795/6/8). + const SHADING_T: [i16; 16] = [0, 64, 128, 224, 320, 480, 672, 896, 1184, 1472, 1824, 2240, 2816, 3552, 4512, 7328]; + const KEY: [u8; 4] = [0x11, 0x22, 0x33, 0x44]; + + /// A CA table with negative and positive samples and two different curves. + fn ca() -> [i16; 32] { + std::array::from_fn(|i| (i as i16 - 16) * 128) + } + fn ca_curves() -> [[i16; 16]; 2] { + let v = ca(); + std::array::from_fn(|c| std::array::from_fn(|i| v[16 * c + i])) + } + fn sshort(prefix: i16, samples: &[i16]) -> Value { + Value::SShort(std::iter::once(prefix).chain(samples.iter().copied()).collect()) + } + /// The six raw-IFD fields: shading 272 (Auto), CA 1 (Auto), and the distortion `setting` and `table`. + fn plain(table: &[i16; 16], setting: u16) -> Vec<(u16, Value)> { + vec![ + (SHADING.setting.0, Value::Short(vec![272])), + (SHADING.table.0, sshort(16, &SHADING_T)), + (CA.setting.0, Value::Short(vec![1])), + (CA.table.0, sshort(32, &ca())), + (DISTORTION.setting.0, Value::Short(vec![setting])), + (DISTORTION.table.0, sshort(16, table)), + ] + } + /// The same fields as `SR2SubIFD` entries `(tag, type, values)`. + fn encrypted(table: &[i16; 16], setting: u16) -> Vec<(u16, u16, Vec)> { + let t = |prefix: i16, v: &[i16]| std::iter::once(prefix).chain(v.iter().copied()).map(|x| x as u16).collect::>(); + vec![ + (SHADING.setting.1, SHORT, vec![272]), + (SHADING.table.1, SSHORT, t(16, &SHADING_T)), + (CA.setting.1, SHORT, vec![1]), + (CA.table.1, SSHORT, t(32, &ca())), + (DISTORTION.setting.1, SHORT, vec![setting]), + (DISTORTION.table.1, SSHORT, t(16, table)), + ] + } + fn set(fields: &mut Vec<(u16, Value)>, tag: u16, value: Value) { + fields.retain(|(t, _)| *t != tag); + fields.push((tag, value)); + } + fn word(order: ByteOrder, v: u32) -> Vec { + let mut out = Vec::new(); + order.put_u32(&mut out, v); + out + } + + /// A small uncompressed Sony CFA file (the raw IFD is IFD0) with the lens fields `plain` in its raw IFD and, when + /// `sr2` is set, an encrypted `SR2SubIFD` holding those entries, reached as Sony stores it: `DNGPrivateData` -> + /// `SR2Private` IFD -> block. + struct Arw { + model: &'static str, + order: ByteOrder, + bigtiff: bool, + size: (u32, u32), + sony_crop: Option<[u32; 4]>, + default_crop: Option<[u32; 4]>, + plain: Vec<(u16, Value)>, + sr2: Option)>>, + /// The key stored in `SR2SubIFDKey`; the block is always encrypted with the usual one. + stored_key: [u8; 4], + /// An edit to the decrypted `SR2SubIFD` before it is encrypted. + sr2_patch: Option>, + } + + impl Arw { + fn new(model: &'static str, order: ByteOrder) -> Arw { + Arw { + model, + order, + bigtiff: false, + size: (32, 24), + sony_crop: Some([2, 2, 30, 20]), + default_crop: Some([2, 2, 30, 20]), + plain: plain(&TELE, 0), + sr2: Some(encrypted(&TELE, 0)), + stored_key: KEY, + sr2_patch: None, + } + } + + fn build(&self) -> Vec { + let (w, h) = self.size; + let mut raw = IfdBuilder::new(); + raw.set(t::MAKE, Value::Ascii("SONY".into())); + raw.set(t::MODEL, Value::Ascii(self.model.into())); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16])); + raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA])); + raw.set(t::COMPRESSION, Value::Short(vec![1])); + raw.set(t::WHITE_LEVEL, Value::Long(vec![16383])); + if let Some([x, y, cw, ch]) = self.sony_crop { + raw.set(0x74c7, Value::Long(vec![x, y])); + raw.set(0x74c8, Value::Long(vec![cw, ch])); + } + if let Some([x, y, cw, ch]) = self.default_crop { + raw.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![x, y])); + raw.set(t::DEFAULT_CROP_SIZE, Value::Long(vec![cw, ch])); + } + if self.sr2.is_some() { + raw.set(t::DNG_PRIVATE_DATA, Value::Byte(vec![0; 4])); + } + for (tag, value) in &self.plain { + raw.set(*tag, value.clone()); + } + raw.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![vec![0; (w * h * 2) as usize]] }); + let mut file = TiffWriter::new(self.order, self.bigtiff).write(&[raw]).unwrap(); + let Some(entries) = &self.sr2 else { return file }; + let o = self.order; + let at = Tiff::parse(&file).unwrap().ifds[0].get(t::DNG_PRIVATE_DATA).unwrap().offset as usize; + let private = file.len(); + let start = private + 2 + 3 * 12 + 4; + let refs: Vec<(u16, u16, &[u16])> = entries.iter().map(|(tag, kind, v)| (*tag, *kind, v.as_slice())).collect(); + let mut block = super::super::sr2::tests::block(start, &refs, o); + if let Some(patch) = &self.sr2_patch { + super::super::sr2::tests::encrypt(&mut block); + patch(&mut block); + super::super::sr2::tests::encrypt(&mut block); + } + o.put_u16(&mut file, 3); + for (tag, value) in [(0x7200u16, start as u32), (0x7201, block.len() as u32)] { + o.put_u16(&mut file, tag); + o.put_u16(&mut file, 4); + o.put_u32(&mut file, 1); + o.put_u32(&mut file, value); + } + o.put_u16(&mut file, 0x7221); + o.put_u16(&mut file, 7); + o.put_u32(&mut file, 4); + file.extend(self.stored_key); + o.put_u32(&mut file, 0); + file.extend(block); + file[at..at + 4].copy_from_slice(&word(o, private as u32)); + file + } + } + + /// Byte position of IFD0's entry for `tag` in a classic TIFF `file`. + fn entry_at(file: &[u8], order: ByteOrder, tag: u16) -> usize { + let ifd = order.read_u32(file, 4).unwrap() as usize; + let n = usize::from(order.read_u16(file, ifd).unwrap()); + (0..n).map(|i| ifd + 2 + 12 * i).find(|&e| order.read_u16(file, e) == Some(tag)).unwrap() + } + + /// Header and full decode of `file`, which must describe the same image; the full one. + fn decode(file: &[u8]) -> RawImage { + let header = super::super::arw::decode(file, Mode::Header).unwrap(); + let full = super::super::arw::decode(file, Mode::Full).unwrap(); + assert_eq!(header.info(), full.info()); + full + } + fn sony(img: &RawImage) -> &SonyLensCorrections { + img.lens_metadata.sony.as_ref().expect("a Sony lens record") + } + /// (has a value, source, raw-IFD copy, SR2SubIFD copy, agreement) of one field. + fn prov(f: &SonyField) -> (bool, Option, SonyCopy, SonyCopy, Option) { + (f.value.is_some(), f.source, f.raw_ifd, f.sr2, f.copies_agree) + } + /// [`prov`] of the three settings, then the three tables. + fn provenance(s: &SonyLensCorrections) -> [(bool, Option, SonyCopy, SonyCopy, Option); 6] { + [ + prov(&s.shading.setting), + prov(&s.chromatic_aberration.setting), + prov(&s.distortion.setting), + prov(&s.shading.table), + prov(&s.chromatic_aberration.table), + prov(&s.distortion.table), + ] + } + + #[test] + fn decodes_all_three_components_with_settings_and_provenance() { + let raw = Some(SonySource::RawIfd); + for order in [Little, Big] { + for bigtiff in [false, true] { + for (model, warps, distortion_status) in + [("ILCE-7CR", 0, SonyLensStatus::NotValidatedModel), ("ILCE-7RM4A", 1, SonyLensStatus::Applied)] + { + let case = format!("{model} {order:?} bigtiff={bigtiff}"); + let mut arw = Arw::new(model, order); + arw.bigtiff = bigtiff; + let img = decode(&arw.build()); + let s = sony(&img); + assert_eq!(img.opcodes.list3.len(), warps, "{case}"); + assert!(s.sr2_read, "{case}"); + assert_eq!(provenance(s), [(true, raw, Valid, Valid, Some(true)); 6], "{case}"); + assert_eq!(s.shading.table.value, Some(SHADING_T), "{case}"); + assert_eq!(s.chromatic_aberration.table.value, Some(ca_curves()), "{case}"); + assert_eq!(s.distortion.table.value, Some(TELE), "{case}"); + assert_eq!(s.shading.setting.value, Some(SonySetting { value: 272, meaning: SonySettingMeaning::Auto })); + assert_eq!(s.chromatic_aberration.setting.value, Some(SonySetting { value: 1, meaning: SonySettingMeaning::Auto })); + assert_eq!(s.distortion.setting.value, Some(SonySetting { value: 0, meaning: SonySettingMeaning::Off })); + // readable shading and CA tables are not corrections + assert_eq!(s.shading.status, SonyLensStatus::NotValidatedComponent, "{case}"); + assert_eq!(s.chromatic_aberration.status, SonyLensStatus::NotValidatedComponent, "{case}"); + assert_eq!(s.distortion.status, distortion_status, "{case}"); + let crop = Rect::new(2, 2, 30, 20); + let geometry = SonyLensGeometry { + storage_width: 32, + storage_height: 24, + sony_crop: Some(crop), + default_crop: Some(crop), + image_crop: crop, + reference: crop, + compression: 1, + linear_ycbcr: false, + native_3_2: true, + }; + assert_eq!(s.geometry, geometry, "{case}"); + // the applied warp is the validated conversion of the raw IFD's table, as before + let reference: Vec = distortion(model, &TELE, Rect::new(0, 0, 32, 24), crop).into_iter().collect(); + assert_eq!(img.opcodes.list3, reference, "{case}"); + // DNG export keeps the warp and invents no Sony record + let dng = crate::write_dng(&img, &Default::default()).unwrap(); + let back = crate::decode(&dng).unwrap(); + assert_eq!(back.opcodes.list3, img.opcodes.list3, "{case}"); + assert_eq!(back.lens_metadata, LensMetadata::default(), "{case}"); + } + } + } + } + + #[test] + fn settings_keep_their_number_and_unknown_values_are_never_off() { + use Kind::{Ca, Distortion, Shading}; + use SonySettingMeaning::*; + for (kind, value, expected) in [ + (Shading, 256, Off), + (Shading, 257, Auto), + (Shading, 272, Auto), + (Shading, 511, Unavailable), + (Shading, 0, Unknown(0)), + (Shading, 1, Unknown(1)), + (Shading, 255, Unknown(255)), + (Shading, 258, Unknown(258)), + (Shading, 273, Unknown(273)), + (Ca, 0, Off), + (Ca, 1, Auto), + (Ca, 255, Unavailable), + (Ca, 2, Unknown(2)), + (Ca, 17, Unknown(17)), + (Ca, 256, Unknown(256)), + (Distortion, 0, Off), + (Distortion, 1, Auto), + (Distortion, 17, ForcedAuto), + (Distortion, 255, Unavailable), + (Distortion, 2, Unknown(2)), + (Distortion, 16, Unknown(16)), + (Distortion, 256, Unknown(256)), + (Distortion, u16::MAX, Unknown(u16::MAX)), + ] { + assert_eq!(meaning(kind, value), expected, "{kind:?} {value}"); + } + for order in [Little, Big] { + let read = |value: Value| { + let mut arw = Arw::new("ILCE-7CR", order); + arw.sr2 = None; + set(&mut arw.plain, DISTORTION.setting.0, value); + sony(&decode(&arw.build())).distortion.setting.clone() + }; + assert_eq!(read(Value::Short(vec![17])).value, Some(SonySetting { value: 17, meaning: ForcedAuto })); + assert_eq!(read(Value::Short(vec![9])).value, Some(SonySetting { value: 9, meaning: Unknown(9) })); + // the documented editor type (SSHORT) is accepted when not negative + assert_eq!(read(Value::SShort(vec![1])).value, Some(SonySetting { value: 1, meaning: Auto })); + for (value, error) in [ + (Value::SShort(vec![-1]), NegativeSetting(-1)), + (Value::SShort(vec![i16::MIN]), NegativeSetting(i16::MIN)), + (Value::Long(vec![1]), FieldType(4)), + (Value::Ascii("1".into()), FieldType(2)), + (Value::Short(vec![1, 1]), Count(2)), + (Value::Short(vec![0; 40]), Count(40)), + ] { + let f = read(value.clone()); + assert_eq!(prov(&f), (false, None, Rejected(error), Absent, None), "{value:?} {order:?}"); + } + } + } + + #[test] + fn malformed_raw_ifd_tables_are_refused_and_not_replaced_by_the_sr2_copy() { + let d = DISTORTION.table.0; + let cases = [ + (d, sshort(16, &[]), Count(1)), + (d, Value::SShort(vec![16, 0]), Count(2)), + (d, sshort(16, &TELE[..15]), Count(16)), + (d, Value::SShort(std::iter::once(16).chain(TELE).chain([0]).collect()), Count(18)), + (d, Value::SShort(vec![15; 17]), CountPrefix(15)), + (d, sshort(-16, &TELE), CountPrefix(-16)), + (d, sshort(0, &TELE), CountPrefix(0)), + (d, Value::Short(vec![16; 17]), FieldType(3)), + (d, Value::Long(vec![16; 17]), FieldType(4)), + (d, Value::Double(vec![f64::NAN; 17]), FieldType(12)), + (d, Value::Undefined(vec![16; 34]), FieldType(7)), + (SHADING.table.0, sshort(32, &[0; 32]), Count(33)), + (SHADING.table.0, Value::Short(std::iter::once(16).chain([0; 16]).collect()), FieldType(3)), + (CA.table.0, sshort(16, &ca()[..16]), Count(17)), + (CA.table.0, sshort(16, &ca()), CountPrefix(16)), + (CA.table.0, sshort(33, &ca()), CountPrefix(33)), + ]; + for order in [Little, Big] { + for (tag, value, error) in &cases { + let mut arw = Arw::new("ILCE-7RM4A", order); + set(&mut arw.plain, *tag, value.clone()); + let img = decode(&arw.build()); + let s = sony(&img); + let field = if *tag == SHADING.table.0 { + prov(&s.shading.table) + } else if *tag == CA.table.0 { + prov(&s.chromatic_aberration.table) + } else { + prov(&s.distortion.table) + }; + // the SR2SubIFD's good copy does not hide the corruption + assert_eq!(field, (false, None, Rejected(*error), Valid, None), "{tag:#x} {value:?} {order:?}"); + // the validated distortion needs its own raw-IFD table; a bad unrelated table doesn't remove it + assert_eq!(img.opcodes.list3.len(), usize::from(*tag != d), "{tag:#x} {value:?}"); + assert_eq!(s.distortion.status == SonyLensStatus::NoTable, *tag == d); + } + } + } + + #[test] + fn counts_and_offsets_beyond_the_data_are_reported_without_reading_them() { + for order in [Little, Big] { + let good = Arw::new("ILCE-7RM4A", order).build(); + let e = entry_at(&good, order, DISTORTION.table.0); + let len = good.len() as u32; + for (field, value, error) in [ + // counts: the TIFF reader drops an entry whose values would pass the end of the file + (4, 0x7fff_ffff, OutsideData), + (4, u32::MAX, OutsideData), + (4, 0, Count(0)), + (4, 16, Count(16)), + // value offsets past the end, straddling it, at it + (8, u32::MAX, OutsideData), + (8, len - 10, OutsideData), + (8, len, OutsideData), + ] { + let mut file = good.clone(); + file[e + field..e + field + 4].copy_from_slice(&word(order, value)); + let img = decode(&file); + let s = sony(&img); + assert_eq!(prov(&s.distortion.table), (false, None, Rejected(error), Valid, None), "{field} {value} {order:?}"); + assert!(img.opcodes.list3.is_empty()); + assert_eq!(s.distortion.status, SonyLensStatus::NoTable); + assert_eq!(s.shading.table.value, Some(SHADING_T)); + } + } + } + + #[test] + fn a_table_past_the_tiff_readers_entry_limit_is_not_applied() { + // The TIFF reader that decodes the image reads 8,192 entries of a directory; with 8,200 filler entries ahead + // of them the image never sees the lens fields, so they must not become a correction (main applies none). + for order in [Little, Big] { + let mut arw = Arw::new("ILCE-7RM4A", order); + (arw.size, arw.sony_crop, arw.default_crop, arw.sr2) = ((30, 20), None, None, None); + let reference = decode(&arw.build()); + assert_eq!(reference.opcodes.list3.len(), 1, "control: the same file without filler is corrected"); + arw.plain = (0..8200u16).map(|i| (0x5000 + i, Value::Byte(vec![0]))).chain(plain(&TELE, 0)).collect(); + let img = decode(&arw.build()); + assert!(img.opcodes.list3.is_empty(), "{order:?}: a table the TIFF reader never read was applied"); + // the directory scan still finds and reports it + let s = sony(&img); + assert_eq!(prov(&s.distortion.table), (false, None, Rejected(PastReaderLimits), Absent, None)); + assert_eq!(s.distortion.status, SonyLensStatus::NoTable); + } + } + + #[test] + fn a_repeated_tag_keeps_the_copy_the_tiff_reader_keeps() { + let crop = Rect::new(2, 2, 30, 20); + let warp = |table: &[i16; 16]| -> Vec { distortion("ILCE-7RM4A", table, Rect::new(0, 0, 32, 24), crop).into_iter().collect() }; + for order in [Little, Big] { + // a second distortion table (WIDE) right after the first (TELE), under a placeholder tag renamed to 0x7037 + let mut arw = Arw::new("ILCE-7RM4A", order); + arw.plain.push((0x7040, sshort(16, &WIDE))); + let good = arw.build(); + let mut tag = Vec::new(); + order.put_u16(&mut tag, DISTORTION.table.0); + let mut twice = good.clone(); + let e = entry_at(&twice, order, 0x7040); + twice[e..e + 2].copy_from_slice(&tag); + // both valid: the first one, as the TIFF reader keeps it and main applies it + let img = decode(&twice); + assert_eq!(img.opcodes.list3, warp(&TELE), "{order:?}: both copies valid"); + assert_eq!(sony(&img).distortion.table.value, Some(TELE)); + // reported as a diagnostic, which changes nothing else + let f = &sony(&img).distortion.table; + assert_eq!((f.raw_ifd_repeated, f.sr2_repeated, f.raw_ifd, f.copies_agree), (true, false, Valid, Some(true))); + assert_eq!(sony(&img).distortion.status, SonyLensStatus::Applied); + assert!(!sony(&decode(&good)).distortion.table.raw_ifd_repeated); + // the first one's values outside the file: the TIFF reader drops it and keeps the second + let mut dropped = twice.clone(); + let first = entry_at(&dropped, order, DISTORTION.table.0); + dropped[first + 8..first + 12].copy_from_slice(&word(order, u32::MAX)); + let img = decode(&dropped); + assert_eq!(img.opcodes.list3, warp(&WIDE), "{order:?}: first copy unreadable"); + assert_eq!(sony(&img).distortion.table.value, Some(WIDE)); + assert!(sony(&img).distortion.table.raw_ifd_repeated); + // the first one malformed (the setting's SHORT entry renamed): the TIFF reader keeps it, nothing applies + let mut short = good.clone(); + let e = entry_at(&short, order, DISTORTION.setting.0); + short[e..e + 2].copy_from_slice(&tag); + let img = decode(&short); + assert!(img.opcodes.list3.is_empty(), "{order:?}: first copy malformed"); + assert_eq!(sony(&img).distortion.table.raw_ifd, Rejected(FieldType(SHORT))); + } + } + + #[test] + fn sr2_copies_fill_in_only_missing_raw_ifd_fields_and_are_never_applied() { + let (raw, sr2) = (Some(SonySource::RawIfd), Some(SonySource::Sr2SubIfd)); + for order in [Little, Big] { + // nothing in the raw IFD: every field from the SR2SubIFD, decoded, not applied even on the validated + // model (only the raw IFD's table was validated) + let mut arw = Arw::new("ILCE-7RM4A", order); + arw.plain.clear(); + let img = decode(&arw.build()); + let s = sony(&img); + assert_eq!(provenance(s), [(true, sr2, Absent, Valid, None); 6]); + assert_eq!( + (s.shading.table.value, s.chromatic_aberration.table.value, s.distortion.table.value), + (Some(SHADING_T), Some(ca_curves()), Some(TELE)) + ); + assert_eq!(s.distortion.setting.value, Some(SonySetting { value: 0, meaning: SonySettingMeaning::Off })); + assert_eq!(s.distortion.status, SonyLensStatus::NotValidatedMode); + assert!(img.opcodes.list3.is_empty()); + // only the distortion table in the raw IFD: that one is used and applied, the rest comes from the SR2SubIFD + let mut arw = Arw::new("ILCE-7RM4A", order); + arw.plain.retain(|(tag, _)| *tag == DISTORTION.table.0); + let img = decode(&arw.build()); + let s = sony(&img); + let p = provenance(s); + assert_eq!(p[5], (true, raw, Valid, Valid, Some(true))); + assert!(p[..5].iter().all(|f| *f == (true, sr2, Absent, Valid, None)), "{p:?}"); + assert_eq!((s.distortion.status, img.opcodes.list3.len()), (SonyLensStatus::Applied, 1)); + // the raw IFD's tag numbers inside the SR2SubIFD are not its tables + let mut arw = Arw::new("ILCE-7CR", order); + arw.plain.clear(); + let plain_tags = [SHADING.setting.0, SHADING.table.0, CA.setting.0, CA.table.0, DISTORTION.setting.0, DISTORTION.table.0]; + arw.sr2 = Some(encrypted(&TELE, 0).into_iter().zip(plain_tags).map(|((_, kind, v), tag)| (tag, kind, v)).collect()); + assert_eq!(decode(&arw.build()).lens_metadata, LensMetadata::default()); + } + } + + #[test] + fn disagreeing_copies_are_reported_and_the_raw_ifd_wins() { + for order in [Little, Big] { + let mut arw = Arw::new("ILCE-7RM4A", order); + arw.sr2 = Some(encrypted(&WIDE, 17)); + let img = decode(&arw.build()); + let s = sony(&img); + let raw = Some(SonySource::RawIfd); + assert_eq!(prov(&s.distortion.table), (true, raw, Valid, Valid, Some(false))); + assert_eq!(prov(&s.distortion.setting), (true, raw, Valid, Valid, Some(false))); + assert_eq!(prov(&s.shading.table), (true, raw, Valid, Valid, Some(true))); + assert_eq!(s.distortion.table.value, Some(TELE)); + assert_eq!(s.distortion.setting.value.map(|v| v.value), Some(0)); + // the validated conversion is unchanged: the raw IFD's table + let reference: Vec = distortion("ILCE-7RM4A", &TELE, Rect::new(0, 0, 32, 24), Rect::new(2, 2, 30, 20)).into_iter().collect(); + assert_eq!(img.opcodes.list3, reference); + } + } + + #[test] + fn unreadable_or_hostile_sr2_subifds_give_no_copies() { + let raw = Some(SonySource::RawIfd); + for order in [Little, Big] { + // stored with another key: the block decrypts to noise, no copy is found, nothing falls back on it + let mut arw = Arw::new("ILCE-7CR", order); + arw.stored_key = [1, 2, 3, 4]; + let img = decode(&arw.build()); + let s = sony(&img); + assert!(provenance(s).iter().all(|&(value, source, plain, sr2, agree)| value + && source == raw + && plain == Valid + && sr2 != Valid + && agree.is_none())); + arw.plain.clear(); + let img = decode(&arw.build()); + assert!(img.lens_metadata.sony.as_ref().is_none_or(|s| provenance(s).iter().all(|f| !f.0 && f.3 != Valid))); + // no SR2SubIFD; DNGPrivateData pointing past the file or into the image + let mut arw = Arw::new("ILCE-7CR", order); + arw.sr2 = None; + let s = sony(&decode(&arw.build())).clone(); + assert!(!s.sr2_read); + assert_eq!(provenance(&s), [(true, raw, Valid, Absent, None); 6]); + let good = Arw::new("ILCE-7CR", order).build(); + let at = Tiff::parse(&good).unwrap().ifds[0].get(t::DNG_PRIVATE_DATA).unwrap().offset as usize; + for pointer in [u32::MAX, good.len() as u32, 8] { + let mut file = good.clone(); + file[at..at + 4].copy_from_slice(&word(order, pointer)); + let img = decode(&file); + assert!(sony(&img).distortion.table.sr2 != Valid, "{pointer}"); + assert_eq!(sony(&img).distortion.table.value, Some(TELE)); + } + // hostile entries in a well-encrypted block (the distortion table is its sixth entry): another type, huge + // count, a value offset before the block or past it + let sixth = 2 + 12 * 5; + let put = move |at: usize, bytes: Vec| move |b: &mut [u8]| b[at..at + bytes.len()].copy_from_slice(&bytes); + let mut short = Vec::new(); + order.put_u16(&mut short, SHORT); + for (patch, error) in [ + (Box::new(put(sixth + 2, short.clone())) as Box, FieldType(SHORT)), + (Box::new(put(sixth + 4, word(order, u32::MAX))), Count(u64::from(u32::MAX))), + (Box::new(put(sixth + 4, word(order, 1))), Count(1)), + (Box::new(put(sixth + 8, word(order, 0))), OutsideData), + (Box::new(put(sixth + 8, word(order, u32::MAX))), OutsideData), + ] { + let mut arw = Arw::new("ILCE-7CR", order); + arw.sr2_patch = Some(patch); + let s = sony(&decode(&arw.build())).clone(); + assert_eq!(prov(&s.distortion.table), (true, raw, Valid, Rejected(error), None), "{error:?}"); + assert_eq!(prov(&s.shading.table), (true, raw, Valid, Valid, Some(true))); + } + // an entry count past the end of the block: the scan stops where the block does + let mut count = Vec::new(); + order.put_u16(&mut count, u16::MAX); + let mut arw = Arw::new("ILCE-7CR", order); + arw.sr2_patch = Some(Box::new(put(0, count))); + assert_eq!(provenance(sony(&decode(&arw.build()))), [(true, raw, Valid, Valid, Some(true)); 6]); + } + } + + #[test] + fn framing_and_coding_evidence_gate_the_validated_distortion() { + use SonyLensStatus::*; + let r = Rect::new; + for order in [Little, Big] { + let run = |arw: &Arw| { + let img = decode(&arw.build()); + let s = sony(&img).clone(); + (s, img) + }; + // Sony's crop at x = 0 and the default crop at x = 2 (the A7R IVA's 0 versus 32): the image keeps Sony's + // crop, the table is centred on and normalised to the default crop + let mut arw = Arw::new("ILCE-7RM4A", order); + arw.sony_crop = Some([0, 2, 30, 20]); + let (s, img) = run(&arw); + let g = &s.geometry; + assert_eq!( + (g.sony_crop, g.default_crop, g.image_crop, g.reference), + (Some(r(0, 2, 30, 20)), Some(r(2, 2, 30, 20)), r(0, 2, 30, 20), r(2, 2, 30, 20)) + ); + assert_eq!((img.crop, s.distortion.status), (r(0, 2, 30, 20), Applied)); + let Some(Opcode::WarpRectilinear { center, .. }) = img.opcodes.list3.first() else { panic!("warp") }; + assert!((center[0] - 16.5 / 31.0).abs() < 1e-12 && (center[1] - 11.5 / 23.0).abs() < 1e-12, "{center:?}"); + // agreeing crops (the A7CR's case), in a frame padded beyond them (lossless L): the storage is recorded, + // the reference stays the crop + let mut arw = Arw::new("ILCE-7RM4A", order); + arw.size = (40, 28); + let (s, _) = run(&arw); + assert_eq!((s.geometry.storage_width, s.geometry.storage_height, s.geometry.reference), (40, 28, r(2, 2, 30, 20))); + assert_eq!(s.distortion.status, Applied); + // without a default crop the reference is the image crop + let mut arw = Arw::new("ILCE-7RM4A", order); + arw.default_crop = None; + let (s, _) = run(&arw); + assert_eq!((s.geometry.default_crop, s.geometry.reference, s.distortion.status), (None, r(2, 2, 30, 20), Applied)); + // non-3:2 framing: an aspect crop in Sony's tags, in the default crop, or a 4:3 frame without crop tags + for (sony_crop, default_crop) in + [(Some([2, 2, 30, 16]), Some([2, 2, 30, 20])), (Some([2, 2, 30, 20]), Some([2, 2, 30, 16])), (None, None)] + { + let mut arw = Arw::new("ILCE-7RM4A", order); + (arw.sony_crop, arw.default_crop) = (sony_crop, default_crop); + let (s, img) = run(&arw); + assert!(!s.geometry.native_3_2, "{sony_crop:?} {default_crop:?}"); + assert_eq!((s.distortion.status, img.opcodes.list3.len()), (NotValidatedMode, 0), "{sony_crop:?} {default_crop:?}"); + } + // neutral, out-of-range and folding tables in the validated scope; models outside it + for (model, table, status) in [ + ("ILCE-7RM4A", [0; 16], Neutral), + ("ILCE-7RM4A", [i16::MIN; 16], Rejected), + ("ILCE-7RM4A", [0, 4096, -4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], Rejected), + ("ILCE-7RM4", TELE, NotValidatedModel), + ("ILCE-7CR", [0; 16], NotValidatedModel), + ("", TELE, NotValidatedModel), + ] { + let mut arw = Arw::new(model, order); + arw.plain = plain(&table, 1); + let (s, img) = run(&arw); + assert_eq!((s.distortion.status, img.opcodes.list3.len()), (status, 0), "{model} {table:?}"); + assert_eq!(s.distortion.table.value, Some(table)); + } + } + } + + #[test] + fn out_of_range_crop_tags_are_recorded_and_never_applied() { + for order in [Little, Big] { + // past the frame, enormous, empty, partly outside + for crop in [[99999, 0, 30, 20], [u32::MAX; 4], [2, 2, 0, 0], [30, 2, 30, 20]] { + let mut arw = Arw::new("ILCE-7RM4A", order); + (arw.sony_crop, arw.default_crop) = (Some(crop), Some(crop)); + let img = decode(&arw.build()); + let s = sony(&img); + let [x, y, w, h] = crop.map(|v| v as usize); + assert_eq!((s.geometry.sony_crop, s.geometry.default_crop), (Some(Rect::new(x, y, w, h)), Some(Rect::new(x, y, w, h))), "{crop:?}"); + assert!(!s.geometry.native_3_2, "{crop:?}"); + assert_eq!((s.distortion.status, img.opcodes.list3.len()), (SonyLensStatus::NotValidatedMode, 0), "{crop:?}"); + assert_eq!(s.distortion.table.value, Some(TELE)); + } + } + } + + #[test] + fn exif_orientation_does_not_change_the_record() { + // the tables and the warp are in sensor coordinates; the engine orients the warp (`engine::files::embedded_lens`) + let reference = decode(&Arw::new("ILCE-7RM4A", Little).build()); + for orientation in 1..=8u16 { + let mut arw = Arw::new("ILCE-7RM4A", Little); + arw.plain.push((t::ORIENTATION, Value::Short(vec![orientation]))); + let img = decode(&arw.build()); + assert_eq!(img.orientation, crate::Orientation::from_exif(orientation)); + assert_eq!((&img.lens_metadata, &img.opcodes), (&reference.lens_metadata, &reference.opcodes), "{orientation}"); + } + } + + #[test] + fn linear_ycbcr_files_are_decoded_but_not_corrected() { + // a downsized lossless (M/S) coding, as in arw.rs's test, from the validated model with 3:2 crop tags + for order in [Little, Big] { + let mut raw = IfdBuilder::new(); + raw.set(t::MAKE, Value::Ascii("SONY".into())); + raw.set(t::MODEL, Value::Ascii("ILCE-7RM4A".into())); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![4])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![4])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![15, 15, 15])); + raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::YCBCR])); + raw.set(t::COMPRESSION, Value::Short(vec![7])); + raw.set(0x7010, Value::Short(vec![0; 4])); + raw.set(0x74c7, Value::Long(vec![0, 0])); + raw.set(0x74c8, Value::Long(vec![3, 2])); + for (tag, value) in plain(&TELE, 17) { + raw.set(tag, value); + } + raw.set_image(ImageData::Tiles { tile_width: 4, tile_height: 4, tiles: vec![crate::ljpeg::tests::fixture_420()] }); + let img = decode(&TiffWriter::new(order, false).write(&[raw]).unwrap()); + let s = sony(&img); + assert!(s.geometry.linear_ycbcr && s.geometry.native_3_2); + assert_eq!((s.geometry.compression, s.distortion.status, img.opcodes.list3.len()), (7, SonyLensStatus::NotValidatedMode, 0)); + assert_eq!(s.distortion.setting.value, Some(SonySetting { value: 17, meaning: SonySettingMeaning::ForcedAuto })); + assert_eq!(s.distortion.table.value, Some(TELE)); + } + } + + #[test] + fn files_without_lens_fields_have_no_record() { + for order in [Little, Big] { + let mut arw = Arw::new("ILCE-7RM4A", order); + arw.plain.clear(); + arw.sr2 = None; + let img = decode(&arw.build()); + assert_eq!((img.lens_metadata.clone(), img.opcodes.list3.len()), (LensMetadata::default(), 0)); + // an SR2SubIFD holding other tags only + arw.sr2 = Some(vec![(0x7310, SHORT, vec![512; 4])]); + assert_eq!(decode(&arw.build()).lens_metadata, LensMetadata::default()); + // one field anywhere is a record + arw.sr2 = Some(vec![(DISTORTION.setting.1, SHORT, vec![255])]); + let img = decode(&arw.build()); + let s = sony(&img); + assert_eq!(s.distortion.setting.value, Some(SonySetting { value: 255, meaning: SonySettingMeaning::Unavailable })); + assert_eq!( + (s.distortion.status, s.shading.status, s.chromatic_aberration.status), + (SonyLensStatus::NoTable, SonyLensStatus::NoTable, SonyLensStatus::NoTable) + ); + } + } + + #[test] + fn corrupt_files_never_panic() { + for order in [Little, Big] { + let good = Arw::new("ILCE-7RM4A", order).build(); + let tiff = Tiff::parse(&good).unwrap(); + let strip = tiff.ifds[0].image().unwrap().chunks(good.len() as u64)[0]; + let image = strip.offset as usize..(strip.offset + strip.len) as usize; + let check = |file: &[u8]| { + let header = super::super::arw::decode(file, Mode::Header); + let full = super::super::arw::decode(file, Mode::Full); + if let (Ok(h), Ok(f)) = (header, full) { + assert_eq!((h.lens_metadata, h.opcodes), (f.lens_metadata, f.opcodes)); + } + }; + for len in 0..good.len() { + check(&good[..len]); + } + for at in (0..good.len()).filter(|at| !image.contains(at)) { + for v in [0, 0xff, good[at] ^ 0x80, good[at] ^ 0x01] { + let mut file = good.clone(); + file[at] = v; + check(&file); + } + } + } + } #[test] fn matches_independent_sony_reference_geometry() { // On/off feature correspondences from Imaging Edge 4.1 at 600 mm, independently fitted. - let Opcode::WarpRectilinear { planes, center } = distortion("ILCE-7RM4A", &table(&TELE), AREA, AREA).unwrap() else { panic!("warp") }; + let Opcode::WarpRectilinear { planes, center } = distortion("ILCE-7RM4A", &TELE, AREA, AREA).unwrap() else { panic!("warp") }; assert_eq!(center, [0.5, 0.5]); let reference = [0.9760159044, 0.0254891644, -0.0005984764, -0.0010120638]; for i in 0..=100 { @@ -151,8 +1169,8 @@ mod tests { fn barrel_framing_and_offset_crop_keep_the_same_geometry() { let crop = Rect::new(32, 20, 9504, 6336); let active = Rect::new(0, 0, 9600, 6376); - let Opcode::WarpRectilinear { planes: a, .. } = distortion("ILCE-7RM4A", &table(&WIDE), AREA, AREA).unwrap() else { panic!("warp") }; - let Opcode::WarpRectilinear { planes: b, center } = distortion("ILCE-7RM4A", &table(&WIDE), active, crop).unwrap() else { panic!("warp") }; + let Opcode::WarpRectilinear { planes: a, .. } = distortion("ILCE-7RM4A", &WIDE, AREA, AREA).unwrap() else { panic!("warp") }; + let Opcode::WarpRectilinear { planes: b, center } = distortion("ILCE-7RM4A", &WIDE, active, crop).unwrap() else { panic!("warp") }; assert!(a[0][0] > 1.02 && a[0][0] < 1.03, "barrel correction expands the middle"); assert!((center[0] * 9599.0 - 4783.5).abs() < 1e-10); assert!((center[1] * 6375.0 - 3187.5).abs() < 1e-10); @@ -162,37 +1180,28 @@ mod tests { } } + /// The conversion's own refusals; missing and malformed layouts are refused by the decoder before it (see + /// `malformed_raw_ifd_tables_are_refused_and_not_replaced_by_the_sr2_copy`). #[test] - fn refuses_missing_unknown_malformed_or_unusable_tables() { - for bad in [ - Ifd::default(), - table(&[]), - table(&TELE[..15]), - table(&[0; 16]), - table(&[i16::MIN; 16]), - table(&[0, 4096, -4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - ifd(Value::SShort(vec![15; 17])), - ifd(Value::Short(vec![16; 17])), - ifd(Value::Double(vec![f64::NAN; 17])), - ] { + fn refuses_unusable_tables() { + for bad in [[0; 16], [i16::MIN; 16], [0, 4096, -4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] { assert!(distortion("ILCE-7RM4A", &bad, AREA, AREA).is_none()); } - assert!(distortion("ILCE-7RM4A", &table(&TELE), Rect::new(0, 0, 1, 1), AREA).is_none()); - assert!(distortion("ILCE-7RM4A", &table(&TELE), AREA, Rect::new(0, 0, 9504, 5346)).is_none()); - assert!(distortion("ILCE-7RM4A", &table(&TELE), AREA, Rect::new(99999, 0, 10, 10)).is_none()); + assert!(distortion("ILCE-7RM4A", &TELE, Rect::new(0, 0, 1, 1), AREA).is_none()); + assert!(distortion("ILCE-7RM4A", &TELE, AREA, Rect::new(0, 0, 9504, 5346)).is_none()); + assert!(distortion("ILCE-7RM4A", &TELE, AREA, Rect::new(99999, 0, 10, 10)).is_none()); } + #[test] fn refuses_unvalidated_models_even_with_a_valid_table() { - let valid = table(&TELE); - assert!(distortion("ILCE-7RM4A", &valid, AREA, AREA).is_some()); - for model in ["", "ILCE-7M3", "ILCE-7M4", "ILCE-7RM4", "ILCE-7RM5", "ILCE-9M2", "DSC-RX100M3", "ILCE-7RM4A unknown"] { - assert!(distortion(model, &valid, AREA, AREA).is_none(), "{model}"); + assert!(distortion("ILCE-7RM4A", &TELE, AREA, AREA).is_some()); + for model in ["", "ILCE-7M3", "ILCE-7M4", "ILCE-7RM4", "ILCE-7RM5", "ILCE-9M2", "DSC-RX100M3", "ILCE-7RM4A unknown", "ILCE-7CR"] { + assert!(distortion(model, &TELE, AREA, AREA).is_none(), "{model}"); } } #[test] fn header_and_full_decode_preserve_correction_through_dng() { - use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter, tags as t}; for (model, expected) in [("ILCE-7RM4A", 1), ("ILCE-7RM4", 0), ("ILCE-7M3", 0), ("", 0)] { for (height, expected) in [(20, expected), (16, 0)] { for order in [ByteOrder::Little, ByteOrder::Big] { @@ -211,7 +1220,7 @@ mod tests { raw.set(t::DEFAULT_CROP_SIZE, Value::Long(vec![30, 20])); // An aspect crop must stay excluded even when DefaultCrop still describes 3:2. raw.set(0x74c8, Value::Long(vec![30, height])); - raw.set(DISTORTION, Value::SShort(std::iter::once(16).chain(TELE).collect())); + raw.set(DISTORTION.table.0, Value::SShort(std::iter::once(16).chain(TELE).collect())); // Zero pixels have the same byte representation in both orders. raw.set_image(ImageData::Strips { rows_per_strip: 24, strips: vec![vec![0; 32 * 24 * 2]] }); let file = TiffWriter { order, ..Default::default() }.write(&[raw]).unwrap(); diff --git a/crates/raw/src/vendor/cr2.rs b/crates/raw/src/vendor/cr2.rs index 74d93534d..f6bc1798a 100644 --- a/crates/raw/src/vendor/cr2.rs +++ b/crates/raw/src/vendor/cr2.rs @@ -251,6 +251,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/cr2/sraw.rs b/crates/raw/src/vendor/cr2/sraw.rs index 68b406238..e1b2988e9 100644 --- a/crates/raw/src/vendor/cr2/sraw.rs +++ b/crates/raw/src/vendor/cr2/sraw.rs @@ -393,6 +393,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode) -> Result linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/cr3.rs b/crates/raw/src/vendor/cr3.rs index 7c93e3bac..2c37c99d3 100644 --- a/crates/raw/src/vendor/cr3.rs +++ b/crates/raw/src/vendor/cr3.rs @@ -255,6 +255,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), }; image.validate_for(mode)?; Ok(image) diff --git a/crates/raw/src/vendor/kodak.rs b/crates/raw/src/vendor/kodak.rs index f7f82cff8..61c238740 100644 --- a/crates/raw/src/vendor/kodak.rs +++ b/crates/raw/src/vendor/kodak.rs @@ -117,6 +117,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/nef.rs b/crates/raw/src/vendor/nef.rs index 7d5b24a4f..4343afe3f 100644 --- a/crates/raw/src/vendor/nef.rs +++ b/crates/raw/src/vendor/nef.rs @@ -219,6 +219,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { ..Default::default() }, metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/orf.rs b/crates/raw/src/vendor/orf.rs index ea7f30eb7..834be3c62 100644 --- a/crates/raw/src/vendor/orf.rs +++ b/crates/raw/src/vendor/orf.rs @@ -331,6 +331,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/pef.rs b/crates/raw/src/vendor/pef.rs index 86db33540..840761f30 100644 --- a/crates/raw/src/vendor/pef.rs +++ b/crates/raw/src/vendor/pef.rs @@ -326,6 +326,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/raf.rs b/crates/raw/src/vendor/raf.rs index 7b148929d..0ff69cd19 100644 --- a/crates/raw/src/vendor/raf.rs +++ b/crates/raw/src/vendor/raf.rs @@ -214,6 +214,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { linearized: false, opcodes: OpcodeLists { list3: super::raf_lens::corrections(&camera_name(bytes), &ifd, active), ..Default::default() }, metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/rw2.rs b/crates/raw/src/vendor/rw2.rs index b856fbd52..67f0ff10c 100644 --- a/crates/raw/src/vendor/rw2.rs +++ b/crates/raw/src/vendor/rw2.rs @@ -618,6 +618,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { linearized: false, opcodes: OpcodeLists { list3: distortion(ifd0, active).into_iter().collect(), ..Default::default() }, metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/src/vendor/sr2.rs b/crates/raw/src/vendor/sr2.rs index 212533add..10fc15074 100644 --- a/crates/raw/src/vendor/sr2.rs +++ b/crates/raw/src/vendor/sr2.rs @@ -45,7 +45,7 @@ //! of an accident 2⁻⁴⁰⁶⁴). For the DSC-R1's key `63 be 01 3e` the 11932-byte block becomes a well-formed 101-entry //! directory (black `0x7300`, white balance `0x7303`, crop `0x74c3`). -use lightcraft_tiff::{ByteOrder, Ifd, ParseOptions, tags as t}; +use lightcraft_tiff::{ByteOrder, EntrySpan, Ifd, ParseOptions, decode_value, directory_entries, entry_span, tags as t}; /// `SR2Private` tags. const SUBIFD_OFFSET: u16 = 0x7200; @@ -136,11 +136,8 @@ impl SubIfd { /// [`MAX_LEN`]. pub(crate) fn read(bytes: &[u8], ifd0: &Ifd, order: ByteOrder) -> Option { let at = order.read_u32(ifd0.bytes(t::DNG_PRIVATE_DATA)?, 0)?; - let opts = ParseOptions { max_ifds: 1, max_depth: 0, follow_children: false, ..Default::default() }; - let (private, _) = lightcraft_tiff::parse_ifd_at(bytes, u64::from(at), order, 0, false, &opts).ok()?; - let key: [u8; 4] = private.bytes(SUBIFD_KEY)?.try_into().ok()?; - let start = usize::try_from(private.u64(SUBIFD_OFFSET)?).ok()?; - let len = usize::try_from(private.u64(SUBIFD_LENGTH)?).ok()?; + let fields = private_fields(bytes, u64::from(at), order); + let (start, len, key) = (fields.offset?, fields.length?, fields.key?); if len > MAX_LEN { return None; } @@ -165,6 +162,13 @@ impl SubIfd { self.order } + /// The decrypted block (the directory from its entry count on) and the file offset it was stored at, which its + /// value offsets count from: for readers that check each entry's exact type and count themselves + /// (`arw_lens`). + pub(crate) fn block(&self) -> (&[u8], usize) { + (&self.plain, self.start) + } + /// The field type and value bytes of a SHORT or SSHORT entry; `None` when the tag is absent, has another type /// or its values lie outside the block. pub(crate) fn short_bytes(&self, tag: u16) -> Option<(u16, &[u8])> { @@ -193,9 +197,69 @@ impl SubIfd { } } +/// The `SR2Private` fields that locate the block. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct PrivateFields { + offset: Option, + length: Option, + key: Option<[u8; 4]>, +} + +/// The block's file offset, length and key from the `SR2Private` directory at `at`, exactly as the TIFF reader would +/// give them (`parse_ifd_at` with its default limits, then its `u64` and `bytes` lookups), without decoding the +/// directory. The selection uses the reader's own rules: its directory layout and entry limit +/// ([`directory_entries`]), which entries it can read ([`entry_span`]) and its value budget; each field is its first +/// readable occurrence. Of that entry only the first value is decoded (the key's first five bytes, which settle +/// whether it is four bytes), so a hostile directory can't make this allocate memory sized by its other entries, +/// which could each alias the whole file, before the block's own [`MAX_LEN`] is checked. +fn private_fields(bytes: &[u8], at: u64, order: ByteOrder) -> PrivateFields { + let opts = ParseOptions::default(); + let Ok(dir) = directory_entries(bytes, at, order, false, opts.max_entries) else { return PrivateFields::default() }; + let mut budget = opts.budget_for(bytes.len()); + let (mut offset, mut length, mut key) = (None, None, None); + for p in dir.positions() { + let Some(span) = entry_span(bytes, p, order, false, 0) else { continue }; + if span.len > budget { + continue; + } + budget -= span.len; + match span.tag { + SUBIFD_OFFSET if offset.is_none() => offset = Some(first_number(bytes, &span, order)), + SUBIFD_LENGTH if length.is_none() => length = Some(first_number(bytes, &span, order)), + SUBIFD_KEY if key.is_none() => key = Some(four_bytes(bytes, &span, order)), + _ => {} + } + } + PrivateFields { offset: offset.flatten(), length: length.flatten(), key: key.flatten() } +} + +/// The entry's first value as the reader's `u64` lookup reads it (an unsigned or non-negative integer), as a `usize`. +fn first_number(bytes: &[u8], span: &EntrySpan, order: ByteOrder) -> Option { + if span.count == 0 { + return None; + } + let start = usize::try_from(span.start).ok()?; + let first = bytes.get(start..start.checked_add(span.field_type.size())?)?; + usize::try_from(decode_value(order, span.field_type, first).get_u64(0)?).ok() +} + +/// The entry's bytes as the reader's `bytes` lookup gives them, when they are exactly four. Decoding the first five +/// value bytes settles it: BYTE and UNDEFINED give all their bytes, ASCII its text before the first NUL, which +/// decoding never makes shorter. +fn four_bytes(bytes: &[u8], span: &EntrySpan, order: ByteOrder) -> Option<[u8; 4]> { + let start = usize::try_from(span.start).ok()?; + let head = bytes.get(start..start.checked_add(usize::try_from(span.len).ok()?.min(5))?)?; + <[u8; 4]>::try_from(decode_value(order, span.field_type, head).as_bytes()?).ok() +} + #[cfg(test)] pub(crate) mod tests { use super::*; + /// TIFF field types. + const BYTE: u16 = 1; + const LONG: u16 = 4; + const UNDEFINED: u16 = 7; + const IFD: u16 = 13; use lightcraft_tiff::ByteOrder::{Big, Little}; /// An `SR2SubIFD` block for file offset `start`, encrypted: SHORT/SSHORT entries `(tag, type, values)`, values @@ -345,6 +409,200 @@ pub(crate) mod tests { } } + /// An `SR2Private` directory of `entries` `(tag, type, count, value field)` at file offset 0. + fn private(entries: &[(u16, u16, u32, [u8; 4])], order: ByteOrder) -> Vec { + let mut out = Vec::new(); + order.put_u16(&mut out, entries.len() as u16); + for &(tag, kind, count, value) in entries { + order.put_u16(&mut out, tag); + order.put_u16(&mut out, kind); + order.put_u32(&mut out, count); + out.extend(value); + } + order.put_u32(&mut out, 0); + out + } + + /// `(offset, length, key)` as one of the readers selects them. + type Selected = (Option, Option, Option<[u8; 4]>); + + /// What `SubIfd::read` on main selects: the TIFF reader's parse of the directory with main's options, then the + /// directory's `u64` and `bytes` lookups. + fn reader_selects(bytes: &[u8], at: usize, order: ByteOrder) -> Selected { + let opts = lightcraft_tiff::ParseOptions { max_ifds: 1, max_depth: 0, follow_children: false, ..Default::default() }; + let Ok((private, _)) = lightcraft_tiff::parse_ifd_at(bytes, at as u64, order, 0, false, &opts) else { return (None, None, None) }; + let n = |tag: u16| private.u64(tag).and_then(|v| usize::try_from(v).ok()); + (n(SUBIFD_OFFSET), n(SUBIFD_LENGTH), private.bytes(SUBIFD_KEY).and_then(|b| <[u8; 4]>::try_from(b).ok())) + } + + /// Synthetic `SR2Private` directories at offset 0, with what each tests: filler entries ahead of the fields, and + /// unreadable, unusual or valid first occurrences of each field ahead of a valid second one. + fn differential_cases(order: ByteOrder) -> Vec<(String, Vec)> { + let w = |v: u32| { + let mut b = Vec::new(); + order.put_u32(&mut b, v); + [b[0], b[1], b[2], b[3]] + }; + let h = |a: u16, b: u16| { + let mut out = Vec::new(); + order.put_u16(&mut out, a); + order.put_u16(&mut out, b); + [out[0], out[1], out[2], out[3]] + }; + let valid = [(SUBIFD_OFFSET, LONG, 1, w(4096)), (SUBIFD_LENGTH, LONG, 1, w(64)), (SUBIFD_KEY, UNDEFINED, 4, KEY)]; + let mut cases = Vec::new(); + for fillers in [0usize, 1, 255, 256, 257, 8189, 8190, 8191, 8192, 8193] { + let mut entries = vec![(0x7300, LONG, 1, w(7)); fillers]; + entries.extend(valid); + cases.push((format!("{fillers} fillers"), private(&entries, order))); + } + // a first occurrence ahead of the valid one; pointer values point into a 32-byte tail after the directory + let tail_at = 2 + 12 * 4 + 4; + let mut tail: Vec = (1..=32).collect(); + tail[4..8].copy_from_slice(b"\x11\x22\x33\x44"); + tail[8] = 0; + let numeric: Vec<(&str, u16, u32, [u8; 4])> = vec![ + ("unknown type 0", 0, 1, w(8)), + ("unknown type 14", 14, 1, w(8)), + ("unknown type 0xffff", 0xffff, 1, w(8)), + ("LONG[2] past the file", LONG, 2, w(u32::MAX)), + ("LONG[2] straddling the end", LONG, 2, w(tail_at as u32 + 28)), + ("LONG[u32::MAX]", LONG, u32::MAX, w(tail_at as u32)), + ("DOUBLE[u32::MAX]", 12, u32::MAX, w(tail_at as u32)), + ("LONG[0]", LONG, 0, w(0)), + ("LONG[2] in the file", LONG, 2, w(tail_at as u32)), + ("SHORT[1]", SHORT, 1, h(9, 0)), + ("SHORT[2] inline", SHORT, 2, h(9, 10)), + ("BYTE[3]", BYTE, 3, [5, 6, 7, 0]), + ("SSHORT -1", SSHORT, 1, h(0xffff, 0)), + ("SSHORT 5", SSHORT, 1, h(5, 0)), + ("SLONG -1", 9, 1, w(u32::MAX)), + ("LONG8 at a pointer", 16, 1, w(tail_at as u32)), + ("RATIONAL", 5, 1, w(tail_at as u32)), + ("ASCII", 2, 4, *b"abc\0"), + ("IFD", IFD, 1, w(4096)), + ]; + for (tag, other) in [(SUBIFD_OFFSET, [valid[1], valid[2]]), (SUBIFD_LENGTH, [valid[0], valid[2]])] { + for (what, kind, count, field) in &numeric { + let valid_one = if tag == SUBIFD_OFFSET { valid[0] } else { valid[1] }; + let mut d = private(&[(tag, *kind, *count, *field), valid_one, other[0], other[1]], order); + d.extend(&tail); + cases.push((format!("{tag:#x}: first {what}"), d)); + } + } + let keys: Vec<(&str, u16, u32, [u8; 4])> = vec![ + ("unknown type", 0, 4, KEY), + ("UNDEFINED[8] past the file", UNDEFINED, 8, w(u32::MAX)), + ("UNDEFINED[5] in the file", UNDEFINED, 5, w(tail_at as u32)), + ("UNDEFINED[3]", UNDEFINED, 3, KEY), + ("UNDEFINED[0]", UNDEFINED, 0, KEY), + ("BYTE[4]", BYTE, 4, [1, 2, 3, 4]), + ("SBYTE[4]", 6, 4, KEY), + ("ASCII[4]", 2, 4, KEY), + ("ASCII[4] with NUL", 2, 4, [0x41, 0x42, 0, 0x43]), + ("ASCII[4] invalid UTF-8", 2, 4, [0xff, 0x41, 0, 0]), + ("ASCII[4] two invalid bytes", 2, 4, [0xc3, 0x28, 0x41, 0]), + ("ASCII[9] at a pointer, NUL at 4", 2, 9, w(tail_at as u32 + 4)), + ("ASCII[8] at a pointer, no NUL", 2, 8, w(tail_at as u32 + 12)), + ("LONG[1]", LONG, 1, KEY), + ]; + for (what, kind, count, field) in &keys { + let mut d = private(&[(SUBIFD_KEY, *kind, *count, *field), valid[0], valid[1], valid[2]], order); + d.extend(&tail); + cases.push((format!("key: first {what}"), d)); + } + // the reader's value budget (8 × the data's length + 1 MiB), used up by entries aliasing the data + for aliases in [23u32, 24] { + let mut entries = vec![(0x7300, UNDEFINED, 1 << 16, w(0)); aliases as usize]; + entries.extend(valid); + let mut d = private(&entries, order); + d.resize(1 << 16, 0); + cases.push((format!("{aliases} entries using the value budget"), d)); + } + // no entries; more entries declared than the data holds + cases.push(("empty directory".into(), private(&[], order))); + let mut long = private(&valid, order); + long[..2].copy_from_slice(&if order == Little { 10u16.to_le_bytes() } else { 10u16.to_be_bytes() }); + long.truncate(2 + 12 * 3); + cases.push(("10 entries declared, 3 present".into(), long)); + cases + } + + #[test] + fn private_fields_match_the_tiff_readers_selection() { + let mut differ = Vec::new(); + for order in [Little, Big] { + for (case, d) in differential_cases(order) { + let f = private_fields(&d, 0, order); + let (ours, reader) = ((f.offset, f.length, f.key), reader_selects(&d, 0, order)); + if ours != reader { + differ.push(format!("{order:?} {case}: {ours:?}, reader {reader:?}")); + } + } + assert_eq!(private_fields(&[], 0, order), PrivateFields::default()); + assert_eq!(private_fields(&private(&[], order), u64::MAX, order), PrivateFields::default()); + } + assert!(differ.is_empty(), "{} cases differ from the TIFF reader:\n{}", differ.len(), differ.join("\n")); + // what the reader (and so this) gives in the cases the review named: the limit is 8,192 entries, a later + // copy stands in for an unreadable first one, and a used-up value budget hides the fields + let found = |d: &[u8]| private_fields(d, 0, Little) == PrivateFields { offset: Some(4096), length: Some(64), key: Some(KEY) }; + let cases = differential_cases(Little); + let case = |name: &str| cases.iter().find(|(c, _)| c == name).map(|(_, d)| d.clone()).unwrap(); + for name in [ + "0 fillers", + "256 fillers", + "8189 fillers", + "0x7200: first LONG[2] past the file", + "0x7201: first unknown type 14", + "key: first UNDEFINED[8] past the file", + "23 entries using the value budget", + ] { + assert!(found(&case(name)), "{name}"); + } + for name in ["8190 fillers", "8192 fillers", "24 entries using the value budget", "0x7200: first ASCII", "key: first UNDEFINED[3]"] { + assert!(!found(&case(name)), "{name}"); + } + } + + /// The case that broke on review: valid entries ahead of the three fields, and an unreadable first offset ahead of + /// a valid one. Main's reader finds the block; so must this one (else the black level, white balance and crop it + /// holds are lost). + #[test] + fn the_block_is_found_behind_filler_and_unreadable_entries() { + for order in [Little, Big] { + for (fillers, unreadable_first) in [(256usize, false), (300, false), (0, true), (256, true)] { + let w = |v: u32| { + let mut b = Vec::new(); + order.put_u32(&mut b, v); + [b[0], b[1], b[2], b[3]] + }; + let n = fillers + 3 + usize::from(unreadable_first); + let start = 2 + 12 * n + 4; + let block = block(start, &[(0x7310, SHORT, &[800; 4])], order); + let mut entries = vec![(0x7300, LONG, 1, w(7)); fillers]; + if unreadable_first { + entries.push((SUBIFD_OFFSET, LONG, 2, w(u32::MAX))); + } + entries.extend([ + (SUBIFD_OFFSET, LONG, 1, w(start as u32)), + (SUBIFD_LENGTH, LONG, 1, w(block.len() as u32)), + (SUBIFD_KEY, UNDEFINED, 4, KEY), + ]); + let mut file = private(&entries, order); + assert_eq!(file.len(), start); + file.extend(&block); + let mut pointer = Vec::new(); + order.put_u32(&mut pointer, 0); + let ifd0 = Ifd { + entries: vec![lightcraft_tiff::Entry { tag: t::DNG_PRIVATE_DATA, value: lightcraft_tiff::Value::Byte(pointer), offset: 0 }], + ..Default::default() + }; + let black = SubIfd::read(&file, &ifd0, order).and_then(|s| s.shorts(0x7310)); + assert_eq!(black, Some(vec![800.0; 4]), "{order:?}: {fillers} fillers, unreadable first offset {unreadable_first}"); + } + } + } + #[test] fn hostile_directories_give_none() { let start = 4096; diff --git a/crates/raw/src/vendor/srw.rs b/crates/raw/src/vendor/srw.rs index f64b451ff..7d5a70aaf 100644 --- a/crates/raw/src/vendor/srw.rs +++ b/crates/raw/src/vendor/srw.rs @@ -186,6 +186,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), }; img.validate_for(mode)?; Ok(img) diff --git a/crates/raw/tests/corpus.rs b/crates/raw/tests/corpus.rs index a96165b5e..98cbe6d2d 100644 --- a/crates/raw/tests/corpus.rs +++ b/crates/raw/tests/corpus.rs @@ -1053,6 +1053,229 @@ fn nikon_in_file_distortion_is_limited_to_validated_bodies() { eprintln!("Nikon distortion boundary: {checked} corpus files checked"); } +/// The Sony samples of the corpus manifest (`RAW_SAMPLES` in `xtask/src/main.rs`) with their pinned SHA-256. +const SONY_SAMPLES: &[(&str, &str)] = &[ + ("arw-sony-a7rm4a-compressed.arw", "690c774f1d7bc1db3fa8c2489762743d8e7586e50b6f65d0b6a61cb467972c67"), + ("arw-sony-a9m2-compressed.arw", "161c2a9da2b5f1e50be6117a0b4da0ce249660b5d7de13568d301c4034716b97"), + ("arw-sony-a7m3-compressed.arw", "250784580ea527442c09004417bb0eead484f2bf3ee8f9121a776ac65bb50d0f"), + ("arw-sony-a7m3-uncompressed.arw", "ece80551abf64949dbe826985a80f1d9b265401ee4ea6b989e1829549616fdcd"), + ("arw-sony-a7m4-14bit.arw", "639a6d4db881f1359e3ea7e1137b314e59417d24e6d4d862b7202111a2c823b2"), + ("arw-sony-a7m4-lossless-l.arw", "851b43c2116c4139104a5036f83ac3b6a148789b2142214dd7192c13972b25b6"), + ("arw-sony-a7m4-lossless-m.arw", "d453005714327addd75bcb99c1c6223173dc92f2a59542d82e6761ef0e0e7571"), + ("arw-sony-a7m4-lossless-s.arw", "cbbd0930c7d8706dff84c68a2004454266e6fd0d8354f5f76a106b5d776e0223"), + ("arw-sony-a7cr-lossless-l.arw", "3e1642f3a1ae7c9f93228c5f09f27e362608e6a4ca1b3e5d8dd9eb8fae576997"), + ("arw-sony-a7cr-lossless-m.arw", "5bcba4acc52a5b902074581b4a2f5fb3c79ec56bd0fdc1746b9e2c0f9d3e9c88"), + ("arw-sony-a7cr-compressed.arw", "f5096e8fbccf0842c8a57763cabd2836608f019eb7fd54f7b22260c2f5bc4da5"), + ("arw-sony-a7cm2-lossless-l.arw", "342d893bd71fd050c273df181e3039d3f12d88947b0662de5de5325c84cc449b"), + ("arw-sony-a7cm2-compressed-m.arw", "002c86d38a16b08a2438780728b4fda8efcc745fa6c26231dad6af67b383b11e"), + ("arw-sony-a7rm4-14bit-compressed.arw", "e6dafe42643f69ab9d1fd00414b7a1f5104df354bb589201600ac934b29b5e4a"), + ("arw-sony-rx100m3.arw", "cb81392e3a8810231ce341fb45de3e4d1b9e2f87bc47f3c314d084041ccfd138"), + ("arw-sony-rx100.arw", "579a485b5126a25cbd55cbd5dadfa7d09cf021c99cc7d4869f9e56e3f759390b"), + ("arw-sony-a7rm2-12bit-uncompressed.arw", "71e0888396ef52c7e6a990b4e72ebb4d8413a1fc1c35bada449870539cef6e39"), + ("arw-sony-a500.arw", "1407fb596a391df67c15b90026a1a76702a2e7386dbdff2f0815b903ea49cead"), + ("arw-sony-a700.arw", "3159e28892bf0771d01525cd4b6190f9c15bbb19fa2fab6d2515ced0594e8f40"), + ("arw-sony-a33.arw", "1a59856394f10d4fadb40f5ab9c6d1c89f473f4dbcdefd216fbbbbf1ad8d21f9"), + ("arw-sony-a3500-5600k.arw", "04c4fe04425c3e6fff7c8af0f811923c35e750300373357bd54342150c5505d8"), + ("arw-sony-a7s-shade.arw", "7cc338a0abc8fdad32d61006f1f8f412297b93e040c7efba572eb2bd8f8e8be2"), + ("arw-sony-a7sm2-16x9.arw", "46044fb6a9c805f2b4915cd4970316b4d04b0566bcf5287afaad3ef648d7a3db"), + ("arw-sony-a580-16x9.arw", "5b0924d39151239dce19e92e08318f4f62a5a8ac276bd7180ec8463d2cfee709"), + ("arw-sony-a900-packed12.arw", "ac7c1532df77c321e8010aa1be60c1b9f245b1ac0db6f38fe617a73aae81af50"), +]; + +/// SHA-256 of [`SONY_SAMPLES`] as `name sha256` lines sorted by name: anchors the pins independently of the +/// manifest, so editing both together (as a drift or a mistake) still fails `sony_pins_match_the_corpus_manifest`. +/// All 25 pins were checked against the downloaded files when it was set. +const SONY_PINS_SHA256: &str = "4cfc349194278146356b463c59ede500331c6ca5fc50aec4e92d7cd847b17b68"; + +/// A Sony corpus file's bytes, checked against its pinned SHA-256 before anything decodes them; `None` (with a +/// note) when the file is absent, unless `LIGHTCRAFT_CORPUS_REQUIRED=1` asks for every pinned file. A file that +/// differs from its pin fails the test. +fn pinned_sony(root: &Path, name: &str) -> Option> { + use sha2::{Digest, Sha256}; + let expected = SONY_SAMPLES.iter().find(|(n, _)| *n == name).unwrap_or_else(|| panic!("{name}: no pinned SHA-256")).1; + let bytes = match std::fs::read(root.join(name)) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + assert!(std::env::var_os("LIGHTCRAFT_CORPUS_REQUIRED").is_none_or(|v| v != "1"), "{name}: absent, and LIGHTCRAFT_CORPUS_REQUIRED=1"); + eprintln!("skip: {name} absent"); + return None; + } + Err(e) => panic!("{name}: {e}"), + }; + let actual: String = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!(actual, expected, "{name}: corpus file differs from its pinned SHA-256"); + Some(bytes) +} + +#[test] +fn sony_pins_match_the_corpus_manifest() { + use sha2::{Digest, Sha256}; + let mut lines: Vec = SONY_SAMPLES.iter().map(|(name, sha)| format!("{name} {sha}\n")).collect(); + lines.sort(); + let digest: String = Sha256::digest(lines.concat()).iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!(digest, SONY_PINS_SHA256, "the pinned Sony SHA-256 table changed: check every changed pin against its file, then update the anchor"); + let manifest = include_str!("../../../xtask/src/main.rs"); + for (name, sha) in SONY_SAMPLES { + let entry = manifest.find(&format!("\"{name}\"")).unwrap_or_else(|| panic!("{name} is not in the manifest")); + assert!(manifest[entry..].split_once("),").is_some_and(|(e, _)| e.contains(sha)), "{name}: pinned SHA-256 differs from the manifest"); + } +} + +#[test] +#[should_panic(expected = "differs from its pinned SHA-256")] +fn a_sony_fixture_that_differs_from_its_pin_fails() { + struct Gone(PathBuf); + impl Drop for Gone { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + let dir = Gone(std::env::temp_dir().join(format!("lightcraft-sony-pin-{}", std::process::id()))); + std::fs::create_dir_all(&dir.0).unwrap(); + std::fs::write(dir.0.join("arw-sony-a7cr-compressed.arw"), b"not the pinned file").unwrap(); + pinned_sony(&dir.0, "arw-sony-a7cr-compressed.arw"); +} + +/// Sony's lens-correction records (`lightcraft_raw::lens_metadata`, `crates/raw/src/vendor/arw_lens.rs`) on the public +/// samples: shading, CA and distortion decoded from the raw IFD with their settings, equal to their `SR2SubIFD` copies, +/// the same from header and full decode, and nothing applied beyond the validated ILCE-7RM4A distortion. Expected +/// tables and settings: what ExifTool, run as a black-box tool, prints for these files. Every file is checked against +/// its pinned SHA-256 before it is decoded; absent files are skipped. +#[test] +fn sony_lens_records_of_the_public_samples() { + use lightcraft_raw::Rect; + use lightcraft_raw::lens_metadata::{SonyCopy, SonyLensStatus, SonySettingMeaning as M, SonySource}; + let root = corpus_root().join("raw"); + // ILCE-7CR (Sigma 24-70mm F2.8 DG DN Art at 49.5 mm): one scene in three codings, identical tables + let a7cr_shading = [0, 64, 128, 224, 320, 480, 672, 896, 1184, 1472, 1824, 2240, 2816, 3552, 4512, 7328]; + let a7cr_ca = [ + [-256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -128], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -128, -128, -256, -256], + ]; + let a7cr_distortion = [4, 1, 1, 3, 7, 13, 22, 35, 51, 72, 98, 132, 173, 223, 279, 348]; + // ILCE-7RM4A (FE 24-105mm F4 G OSS at 24 mm) + let a7rm4a_shading = [0, 32, 112, 256, 448, 688, 976, 1296, 1712, 2336, 3120, 4080, 5104, 6112, 7024, 7904]; + let a7rm4a_ca = [ + [1408, 1280, 1152, 1024, 1024, 1024, 1024, 1024, 896, 896, 896, 768, 768, 640, 512, 384], + [512, 512, 384, 384, 384, 384, 384, 512, 512, 512, 512, 640, 640, 640, 768, 640], + ]; + let a7rm4a_distortion = [14, 0, -29, -65, -113, -171, -239, -311, -392, -471, -555, -634, -713, -782, -843, -891]; + let a7cr_crop = Rect::new(32, 20, 9504, 6336); + // (file, settings, tables, distortion status, storage, compression, linear YCbCr, Sony crop, default crop) + let cases = [ + ( + "arw-sony-a7cr-compressed.arw", + [272, 1, 0], + (a7cr_shading, a7cr_ca, a7cr_distortion), + SonyLensStatus::NotValidatedModel, + (9600, 6376), + 32767, + false, + a7cr_crop, + a7cr_crop, + ), + ( + "arw-sony-a7cr-lossless-l.arw", + [272, 1, 0], + (a7cr_shading, a7cr_ca, a7cr_distortion), + SonyLensStatus::NotValidatedModel, + (9728, 6656), + 7, + false, + a7cr_crop, + a7cr_crop, + ), + ( + "arw-sony-a7cr-lossless-m.arw", + [272, 1, 0], + (a7cr_shading, a7cr_ca, a7cr_distortion), + SonyLensStatus::NotValidatedModel, + (6656, 4608), + 7, + true, + Rect::new(20, 12, 6240, 4160), + Rect::new(20, 12, 6240, 4160), + ), + ( + "arw-sony-a7rm4a-compressed.arw", + [257, 1, 17], + (a7rm4a_shading, a7rm4a_ca, a7rm4a_distortion), + SonyLensStatus::Applied, + (9600, 6376), + 32767, + false, + Rect::new(0, 20, 9504, 6336), + Rect::new(32, 20, 9504, 6336), + ), + ]; + let mut checked = 0; + for (name, settings, (shading, ca, distortion), status, (w, h), compression, linear, sony_crop, default_crop) in cases { + let Some(bytes) = pinned_sony(&root, name) else { continue }; + let full = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(probe_info(&bytes).unwrap(), full.info(), "{name}"); + let s = full.lens_metadata.sony.as_ref().unwrap_or_else(|| panic!("{name}: no Sony lens record")); + assert!(s.sr2_read, "{name}"); + let got = [s.shading.setting.value, s.chromatic_aberration.setting.value, s.distortion.setting.value].map(|v| v.map(|v| v.value)); + assert_eq!(got, settings.map(Some), "{name}: settings"); + let meanings = [s.shading.setting.value, s.chromatic_aberration.setting.value, s.distortion.setting.value].map(|v| v.map(|v| v.meaning)); + let forced = if settings[2] == 17 { M::ForcedAuto } else { M::Off }; + assert_eq!(meanings, [Some(M::Auto), Some(M::Auto), Some(forced)], "{name}: setting meanings"); + assert_eq!( + (s.shading.table.value, s.chromatic_aberration.table.value, s.distortion.table.value), + (Some(shading), Some(ca), Some(distortion)), + "{name}" + ); + for (source, raw, sr2, agree) in [ + (s.shading.table.source, s.shading.table.raw_ifd, s.shading.table.sr2, s.shading.table.copies_agree), + ( + s.chromatic_aberration.table.source, + s.chromatic_aberration.table.raw_ifd, + s.chromatic_aberration.table.sr2, + s.chromatic_aberration.table.copies_agree, + ), + (s.distortion.table.source, s.distortion.table.raw_ifd, s.distortion.table.sr2, s.distortion.table.copies_agree), + (s.shading.setting.source, s.shading.setting.raw_ifd, s.shading.setting.sr2, s.shading.setting.copies_agree), + ( + s.chromatic_aberration.setting.source, + s.chromatic_aberration.setting.raw_ifd, + s.chromatic_aberration.setting.sr2, + s.chromatic_aberration.setting.copies_agree, + ), + (s.distortion.setting.source, s.distortion.setting.raw_ifd, s.distortion.setting.sr2, s.distortion.setting.copies_agree), + ] { + assert_eq!((source, raw, sr2, agree), (Some(SonySource::RawIfd), SonyCopy::Valid, SonyCopy::Valid, Some(true)), "{name}: provenance"); + } + assert_eq!( + (s.shading.status, s.chromatic_aberration.status, s.distortion.status), + (SonyLensStatus::NotValidatedComponent, SonyLensStatus::NotValidatedComponent, status), + "{name}" + ); + assert_eq!(full.opcodes.list3.len(), usize::from(status == SonyLensStatus::Applied), "{name}: only applied components are opcodes"); + let g = &s.geometry; + assert_eq!((g.storage_width, g.storage_height, g.compression, g.linear_ycbcr), (w, h, compression, linear), "{name}"); + assert_eq!( + (g.sony_crop, g.default_crop, g.image_crop, g.reference), + (Some(sony_crop), Some(default_crop), sony_crop, default_crop), + "{name}" + ); + assert!(g.native_3_2, "{name}"); + eprintln!("{name}: shading, CA and distortion decoded; distortion {status:?}"); + checked += 1; + } + // every Sony corpus file: only the ILCE-7RM4A's distortion is ever applied, and an applied status is an opcode + for (name, _) in SONY_SAMPLES { + let Some(bytes) = pinned_sony(&root, name) else { continue }; + let info = probe_info(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); + let Some(s) = &info.lens_metadata.sony else { continue }; + let applied = s.distortion.status == SonyLensStatus::Applied; + assert_eq!(applied, info.metadata.model.as_deref() == Some("ILCE-7RM4A"), "{name}"); + assert_eq!(info.opcodes.list3.len(), usize::from(applied), "{name}"); + assert!(![s.shading.status, s.chromatic_aberration.status].contains(&SonyLensStatus::Applied), "{name}"); + checked += 1; + } + eprintln!("Sony lens records: {checked} checks on corpus files"); +} + /// Kodak DCS520C (`crates/raw/src/vendor/kodak.rs`): the two-component lossless JPEG strip decodes through the /// file's response curve to a 1736 x 1160 GRBG mosaic with 12-bit white, per-position black levels and the /// white balance of the vendor IFD. Looks for the corpus copy, then the raw.pixls.us folder; skips without either. diff --git a/crates/raw/tests/dng.rs b/crates/raw/tests/dng.rs index f95afef22..921e8900a 100644 --- a/crates/raw/tests/dng.rs +++ b/crates/raw/tests/dng.rs @@ -61,6 +61,7 @@ fn synthetic(w: usize, h: usize, cfa: Option, cpp: usize) -> RawImage { linearized: false, opcodes: OpcodeLists::default(), metadata, + lens_metadata: Default::default(), } } diff --git a/crates/raw/tests/robust.rs b/crates/raw/tests/robust.rs index 6dc383de8..17e108f6f 100644 --- a/crates/raw/tests/robust.rs +++ b/crates/raw/tests/robust.rs @@ -26,6 +26,7 @@ fn sample(comp: DngCompression) -> Vec { linearized: false, opcodes: OpcodeLists::default(), metadata: Default::default(), + lens_metadata: Default::default(), }; lightcraft_raw::write_dng(&raw, &DngWriteOptions { compression: comp, ..Default::default() }).unwrap() } diff --git a/crates/raw/tests/sony_lens_inventory.rs b/crates/raw/tests/sony_lens_inventory.rs new file mode 100644 index 000000000..3eda88418 --- /dev/null +++ b/crates/raw/tests/sony_lens_inventory.rs @@ -0,0 +1,351 @@ +//! Opt-in local check of Sony's lens-correction records ([`lightcraft_raw::lens_metadata`]); it does nothing unless +//! `LIGHTCRAFT_SONY_LENS_DIR` names a directory of ARW files, so CI never runs it. The files there (searched +//! recursively) are only read, from their headers, and one line per file is printed: settings, where each table came +//! from, whether its raw-IFD and `SR2SubIFD` copies agree, sample ranges and status. Header and full decode are +//! compared too when `LIGHTCRAFT_SONY_LENS_FULL=1` (slow on large files). +//! +//! With `LIGHTCRAFT_SONY_LENS_INVENTORY` naming a CSV inventory of those files (comma-separated, no quoting, a header +//! row, one row per file name), every file must have a row, every row a file, and these columns must match where +//! present: `split` (the file's folder name), `raw_size` (`WxH`), `compression`, `crop_origin` / `crop_size` (Sony's +//! crop tags, `x y` / `w h`), and for each of `shading`, `ca` and `distortion`: `_entries` (TIFF count), `_bytes`, +//! `_samples` (the leading sample count), `_min` / `_max` (over the samples), `_sr2_equal` (`True` / `False`), +//! `_setting_raw` and `_setting` (a label: `Off`, `Auto`, `Auto (ILCE-1)`, `Auto fixed by lens`, `No correction +//! params available`). A file that doesn't decode must still have a row, and fails the check when it has one. +//! +//! Any decode error fails the check except an unsupported variant (`RawError::Unsupported`, e.g. the packed ARW of +//! some older bodies): those are counted, and the count must equal `LIGHTCRAFT_SONY_LENS_UNSUPPORTED` (default 0). +//! +//! `LIGHTCRAFT_SONY_LENS_DIR=… LIGHTCRAFT_SONY_LENS_INVENTORY=….csv cargo test -p lightcraft-raw --release --test +//! sony_lens_inventory -- --nocapture` + +use lightcraft_raw::lens_metadata::{SonyCopy, SonyField, SonyLensCorrections, SonyLensStatus, SonySetting, SonySettingMeaning, SonySource}; +use lightcraft_raw::{RawError, RawInfo, decode, probe_info}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +fn arw_files(dir: &Path, depth: usize, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() && depth < 8 { + arw_files(&path, depth + 1, out); + } else if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("arw")) { + out.push(path); + } + } +} + +/// One component's columns, from the decoded record. +struct Component { + setting: Option<(u16, SonySettingMeaning)>, + /// Samples when the table was decoded. + samples: Option>, + source: Option, + /// The raw-IFD and `SR2SubIFD` copies' states. + copies: (SonyCopy, SonyCopy), + agree: Option, + status: String, +} + +fn component(setting: &SonyField, table: &SonyField, samples: Option>, status: SonyLensStatus) -> Component { + Component { + setting: setting.value.map(|s| (s.value, s.meaning)), + samples, + source: table.source, + copies: (table.raw_ifd, table.sr2), + agree: table.copies_agree, + status: format!("{status:?}"), + } +} + +/// Shading, CA and distortion of a record, by CSV column prefix. +fn components(s: &SonyLensCorrections) -> [(&'static str, Component); 3] { + let (sh, ca, d) = (&s.shading, &s.chromatic_aberration, &s.distortion); + [ + ("shading", component(&sh.setting, &sh.table, sh.table.value.map(|v| v.to_vec()), sh.status)), + ("ca", component(&ca.setting, &ca.table, ca.table.value.map(|v| v.concat()), ca.status)), + ("distortion", component(&d.setting, &d.table, d.table.value.map(|v| v.to_vec()), d.status)), + ] +} + +fn label(meaning: SonySettingMeaning) -> &'static [&'static str] { + match meaning { + SonySettingMeaning::Off => &["Off"], + SonySettingMeaning::Auto => &["Auto", "Auto (ILCE-1)"], + SonySettingMeaning::ForcedAuto => &["Auto fixed by lens"], + SonySettingMeaning::Unavailable => &["No correction params available"], + SonySettingMeaning::Unknown(_) => &[], + } +} + +/// The columns this check derives from a file, by CSV column name. +fn columns(path: &Path, info: &RawInfo) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut put = |k: &str, v: String| { + out.insert(k.to_string(), v); + }; + if let Some(split) = path.parent().and_then(|p| p.file_name()) { + put("split", split.to_string_lossy().into_owned()); + } + let Some(s) = &info.lens_metadata.sony else { return out }; + let g = &s.geometry; + put("raw_size", format!("{}x{}", g.storage_width, g.storage_height)); + put("compression", g.compression.to_string()); + if let Some(c) = g.sony_crop { + put("crop_origin", format!("{} {}", c.x, c.y)); + put("crop_size", format!("{} {}", c.width, c.height)); + } + for (name, c) in components(s) { + if let Some(v) = &c.samples { + put(&format!("{name}_entries"), (v.len() + 1).to_string()); + put(&format!("{name}_bytes"), (2 * (v.len() + 1)).to_string()); + put(&format!("{name}_samples"), v.len().to_string()); + put(&format!("{name}_min"), v.iter().min().map_or(String::new(), i16::to_string)); + put(&format!("{name}_max"), v.iter().max().map_or(String::new(), i16::to_string)); + } + put(&format!("{name}_sr2_equal"), c.agree.map_or("unknown", |a| if a { "True" } else { "False" }).to_string()); + if let Some((value, _)) = c.setting { + put(&format!("{name}_setting_raw"), value.to_string()); + } + } + out +} + +fn summary(info: &RawInfo) -> String { + let Some(s) = &info.lens_metadata.sony else { return "no lens record".into() }; + let mut line = format!("sr2 {} |", if s.sr2_read { "read" } else { "absent" }); + for (name, c) in components(s) { + let setting = c.setting.map_or("-".into(), |(v, m)| format!("{v} {m:?}")); + let range = c + .samples + .as_ref() + .map_or("no table".into(), |v| format!("{} samples {}..{}", v.len(), v.iter().min().unwrap_or(&0), v.iter().max().unwrap_or(&0))); + line += + &format!(" {name}: {setting}, {range}, {:?} (copies {:?} / {:?}), agree {:?}, {} |", c.source, c.copies.0, c.copies.1, c.agree, c.status); + } + line + &format!(" crop {:?} reference {:?} 3:2 {}", s.geometry.image_crop, s.geometry.reference, s.geometry.native_3_2) +} + +/// `(header, rows by file name)` of a simple CSV file. +fn read_csv(path: &Path) -> (Vec, BTreeMap>) { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + let mut lines = text.lines().filter(|l| !l.trim().is_empty()); + let header: Vec = lines.next().expect("CSV header").split(',').map(|s| s.trim().to_string()).collect(); + let file = header.iter().position(|h| h == "file").expect("a `file` column"); + let mut rows = BTreeMap::new(); + for line in lines { + assert!(!line.contains('"'), "quoted CSV fields are not supported: {line}"); + let row: Vec = line.split(',').map(|s| s.trim().to_string()).collect(); + assert_eq!(row.len(), header.len(), "CSV row with {} fields, header has {}: {line}", row.len(), header.len()); + let name = row[file].clone(); + assert!(rows.insert(name.clone(), row).is_none(), "{name} listed twice"); + } + (header, rows) +} + +#[test] +fn sony_lens_records_reproduce_a_local_inventory() { + let Some(dir) = std::env::var_os("LIGHTCRAFT_SONY_LENS_DIR").map(PathBuf::from) else { + eprintln!("skip: LIGHTCRAFT_SONY_LENS_DIR unset"); + return; + }; + let full = std::env::var_os("LIGHTCRAFT_SONY_LENS_FULL").is_some_and(|v| v == "1"); + let inventory = std::env::var_os("LIGHTCRAFT_SONY_LENS_INVENTORY").map(PathBuf::from); + let unsupported = std::env::var("LIGHTCRAFT_SONY_LENS_UNSUPPORTED").map_or(0, |v| v.parse().expect("LIGHTCRAFT_SONY_LENS_UNSUPPORTED: a count")); + let report = check(&dir, inventory.as_deref(), full, unsupported); + assert!(report.problems.is_empty(), "{} problems", report.problems.len()); +} + +/// What [`check`] found. +#[derive(Debug, Default)] +struct Report { + files: usize, + records: usize, + without_record: usize, + unsupported: usize, + problems: Vec, +} + +/// Check every ARW under `dir` (see the module docs). Any decode error is a problem except `Unsupported`, which +/// is counted and must come to exactly `expected_unsupported`; with an inventory, every file (decoded or not) must +/// have a row, and a file with a row must decode. +fn check(dir: &Path, inventory: Option<&Path>, full: bool, expected_unsupported: usize) -> Report { + let inventory = inventory.map(read_csv); + let mut files = Vec::new(); + arw_files(dir, 0, &mut files); + files.sort(); + let mut report = Report { files: files.len(), ..Default::default() }; + if files.is_empty() { + report.problems.push(format!("no ARW files under {}", dir.display())); + } + let (mut matched, mut mismatches, mut seen) = (0usize, Vec::new(), std::collections::BTreeSet::new()); + let compared: Vec = ["split", "raw_size", "compression", "crop_origin", "crop_size"] + .into_iter() + .map(String::from) + .chain( + ["shading", "ca", "distortion"] + .into_iter() + .flat_map(|k| ["entries", "bytes", "samples", "min", "max", "sr2_equal", "setting_raw"].into_iter().map(move |c| format!("{k}_{c}"))), + ) + .collect(); + for path in &files { + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + seen.insert(name.clone()); + let info = match probe_info(&bytes) { + Ok(info) => info, + Err(e) => { + eprintln!("{name}: {e}"); + match (&inventory, &e) { + (Some((_, rows)), _) if rows.contains_key(&name) => mismatches.push(format!("{name}: does not decode: {e}")), + (Some(_), _) => mismatches.push(format!("{name}: does not decode ({e}) and is not in the inventory")), + (None, RawError::Unsupported(_)) => {} + (None, _) => mismatches.push(format!("{name}: does not decode: {e}")), + } + report.unsupported += usize::from(matches!(e, RawError::Unsupported(_))); + continue; + } + }; + if full { + match decode(&bytes) { + Ok(image) if image.info() == info => {} + Ok(_) => mismatches.push(format!("{name}: header and full decode differ")), + Err(e) => mismatches.push(format!("{name}: full decode fails: {e}")), + } + } + if info.lens_metadata.sony.is_some() { + report.records += 1; + } else { + report.without_record += 1; + } + eprintln!("{name}: {} {}", info.metadata.model.as_deref().unwrap_or("?"), summary(&info)); + let Some((header, rows)) = &inventory else { continue }; + let Some(row) = rows.get(&name) else { + mismatches.push(format!("{name}: not in the inventory")); + continue; + }; + let got = columns(path, &info); + let mut ok = true; + for (i, column) in header.iter().enumerate() { + let expected = &row[i]; + if let Some(setting) = column.strip_suffix("_setting") + && let Some(s) = &info.lens_metadata.sony + { + let meaning = components(s).into_iter().find(|(n, _)| *n == setting).and_then(|(_, c)| c.setting).map(|(_, m)| m); + if !meaning.is_some_and(|m| label(m).contains(&expected.as_str())) { + mismatches.push(format!("{name}: {column} {meaning:?}, inventory {expected}")); + ok = false; + } + continue; + } + if !compared.contains(column) { + continue; + } + let value = got.get(column).map(String::as_str).unwrap_or("(none)"); + if value != expected { + mismatches.push(format!("{name}: {column} {value}, inventory {expected}")); + ok = false; + } + } + matched += usize::from(ok); + } + if let Some((header, rows)) = &inventory { + for name in rows.keys().filter(|n| !seen.contains(*n)) { + mismatches.push(format!("{name}: in the inventory, no such file")); + } + let checked = header.iter().filter(|c| compared.contains(c) || c.ends_with("_setting")).count(); + assert!(checked > 0, "the inventory has none of the compared columns"); + eprintln!("{matched} of {} inventory rows reproduced from {} files ({checked} columns compared)", rows.len(), files.len()); + } + if report.unsupported != expected_unsupported { + mismatches.push(format!("{} files are unsupported variants, {expected_unsupported} expected", report.unsupported)); + } + eprintln!( + "{} files: {} with a lens record, {} decoded without one, {} unsupported ({expected_unsupported} expected)", + report.files, report.records, report.without_record, report.unsupported + ); + for m in &mismatches { + eprintln!("problem: {m}"); + } + report.problems.extend(mismatches); + report +} + +/// A directory under the system temporary directory, removed when dropped. +struct Scratch(PathBuf); + +impl Scratch { + fn new(name: &str) -> Scratch { + let dir = std::env::temp_dir().join(format!("lightcraft-sony-lens-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + Scratch(dir) + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// A small Sony CFA file with a distortion setting (a lens record), or, `unsupported`, the packed 12-bit ARW variant +/// the decoder refuses as unsupported. +fn synthetic_arw(unsupported: bool) -> Vec { + use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value, tags as t}; + let mut raw = IfdBuilder::new(); + raw.set(t::MAKE, Value::Ascii("SONY".into())); + raw.set(t::MODEL, Value::Ascii("ILCE-7CR".into())); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![32])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![4])); + raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA])); + raw.set(0x7036, Value::Short(vec![0])); + if unsupported { + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![12])); + raw.set(t::COMPRESSION, Value::Short(vec![32767])); + raw.set(0x7010, Value::Short(vec![8000, 10400, 12900, 14100])); + raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0; 32 * 4 + 8]] }); + } else { + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16])); + raw.set(t::COMPRESSION, Value::Short(vec![1])); + raw.set(t::WHITE_LEVEL, Value::Long(vec![16383])); + raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0; 32 * 4 * 2]] }); + } + TiffWriter::default().write(&[raw]).unwrap() +} + +#[test] +fn files_that_fail_to_decode_fail_the_check() { + // a file that is no raw at all fails the check, whatever is expected + let dir = Scratch::new("decode-error"); + std::fs::write(dir.0.join("not-a-raw.arw"), b"not a raw file").unwrap(); + let report = check(&dir.0, None, false, 0); + assert!(!report.problems.is_empty(), "an ARW that does not decode passed the check"); + assert!(!check(&dir.0, None, false, 1).problems.is_empty(), "a decode error counted as an unsupported variant"); + // an unsupported variant is counted, and must be expected + let dir = Scratch::new("unsupported"); + std::fs::write(dir.0.join("good.arw"), synthetic_arw(false)).unwrap(); + std::fs::write(dir.0.join("packed.arw"), synthetic_arw(true)).unwrap(); + let report = check(&dir.0, None, false, 0); + assert_eq!((report.files, report.records, report.unsupported), (2, 1, 1)); + assert_eq!(report.problems.len(), 1, "{:?}", report.problems); + assert!(check(&dir.0, None, true, 1).problems.is_empty()); + // with an inventory, a file that fails must still be listed: here it is not + let csv = dir.0.join("inventory.csv"); + std::fs::write(&csv, "file,distortion_setting_raw\ngood.arw,0\n").unwrap(); + // (the inventory itself sits in the directory, but only .arw files are checked) + let mut report = check(&dir.0, Some(&csv), false, 1); + report.problems.sort(); + assert_eq!(report.problems.len(), 1, "{:?}", report.problems); + assert!(report.problems[0].contains("packed.arw") && report.problems[0].contains("not in the inventory"), "{:?}", report.problems); + // listed but failing, and listed and decoding + std::fs::write(&csv, "file,distortion_setting_raw\ngood.arw,0\npacked.arw,0\n").unwrap(); + let report = check(&dir.0, Some(&csv), false, 1); + assert!(report.problems.len() == 1 && report.problems[0].contains("packed.arw: does not decode"), "{:?}", report.problems); + std::fs::remove_file(dir.0.join("packed.arw")).unwrap(); + std::fs::write(&csv, "file,distortion_setting_raw\ngood.arw,0\n").unwrap(); + assert!(check(&dir.0, Some(&csv), false, 0).problems.is_empty()); + std::fs::write(&csv, "file,distortion_setting_raw\ngood.arw,17\n").unwrap(); + assert!(!check(&dir.0, Some(&csv), false, 0).problems.is_empty(), "a wrong value passed"); +} diff --git a/crates/raw/tests/sr2_heap.rs b/crates/raw/tests/sr2_heap.rs new file mode 100644 index 000000000..2b936d18c --- /dev/null +++ b/crates/raw/tests/sr2_heap.rs @@ -0,0 +1,79 @@ +//! Peak heap of reading Sony's `SR2Private` directory (`crates/raw/src/vendor/sr2.rs`), measured with dhat's testing +//! mode. Every ARW with `DNGPrivateData` has its `SR2SubIFD` located and decrypted, so the directory that points to +//! it must not cost memory sized by its other entries: a hostile file can give it fields that each alias most of the +//! file. Only the block's offset, length and key may be read. One test per binary (dhat's profiler is global). + +use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, Tiff, TiffWriter, Value, tags as t}; + +#[global_allocator] +static ALLOC: dhat::Alloc = dhat::Alloc; + +/// A 32 × 24 Sony CFA file of `size` bytes whose `SR2Private` directory holds the block's offset, length and key plus +/// `aliases` UNDEFINED fields that each cover the first `size` bytes of the file. Plain black level and white balance +/// are present, so nothing but the lens record needs the `SR2SubIFD`. +fn hostile_arw(size: usize, aliases: u16) -> Vec { + let o = ByteOrder::Little; + let mut raw = IfdBuilder::new(); + raw.set(t::MAKE, Value::Ascii("SONY".into())); + raw.set(t::MODEL, Value::Ascii("ILCE-7CR".into())); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![32])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![24])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16])); + raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA])); + raw.set(t::COMPRESSION, Value::Short(vec![1])); + raw.set(t::WHITE_LEVEL, Value::Long(vec![16383])); + raw.set(0x7310, Value::Short(vec![512; 4])); + raw.set(0x7313, Value::Short(vec![2048, 1024, 1024, 1536])); + // a distortion setting, so the file has a lens record that says whether the SR2SubIFD was read + raw.set(0x7036, Value::Short(vec![0])); + raw.set(t::DNG_PRIVATE_DATA, Value::Byte(vec![0; 4])); + raw.set_image(ImageData::Strips { rows_per_strip: 24, strips: vec![vec![0; 32 * 24 * 2]] }); + let mut file = TiffWriter::new(o, false).write(&[raw]).unwrap(); + let pointer = Tiff::parse(&file).unwrap().ifds[0].get(t::DNG_PRIVATE_DATA).unwrap().offset as usize; + file.resize(size, 0); + let private = file.len(); + let entries = 3 + aliases; + let block_at = private + 2 + 12 * usize::from(entries) + 4; + let block_len = 64u32; + o.put_u16(&mut file, entries); + for (tag, kind, count, value) in [(0x7200u16, 4u16, 1u32, block_at as u32), (0x7201, 4, 1, block_len)] { + o.put_u16(&mut file, tag); + o.put_u16(&mut file, kind); + o.put_u32(&mut file, count); + o.put_u32(&mut file, value); + } + o.put_u16(&mut file, 0x7221); + o.put_u16(&mut file, 7); + o.put_u32(&mut file, 4); + file.extend([0x11, 0x22, 0x33, 0x44]); + for i in 0..aliases { + o.put_u16(&mut file, 0x7230 + i); + o.put_u16(&mut file, 7); + o.put_u32(&mut file, size as u32); + o.put_u32(&mut file, 0); + } + o.put_u32(&mut file, 0); + file.resize(block_at + block_len as usize, 0); + let mut at = Vec::new(); + o.put_u32(&mut at, private as u32); + file[pointer..pointer + 4].copy_from_slice(&at); + file +} + +#[test] +fn sr2_private_fields_cost_no_memory_sized_by_the_file() { + // eight fields of 16 MiB each: 128 MiB if they were decoded, within the TIFF reader's default value budget + let file = hostile_arw(16 << 20, 8); + let _profiler = dhat::Profiler::builder().testing().build(); + let header = lightcraft_raw::probe_info(&file).unwrap(); + let full = lightcraft_raw::decode(&file).unwrap(); + let peak = dhat::HeapStats::get().max_bytes; + // the block was still found and decrypted (its content is noise: no SR2SubIFD copies) + for info in [&header, &full.info()] { + let sony = info.lens_metadata.sony.as_ref().expect("a lens record"); + assert!(sony.sr2_read, "the SR2SubIFD was not read"); + } + eprintln!("peak heap {peak} bytes decoding a {} byte file", file.len()); + assert!(peak < 2 << 20, "peak heap {peak} bytes decoding a {} byte file", file.len()); +} diff --git a/crates/tiff/README.md b/crates/tiff/README.md index c691dd641..cfaaffffb 100644 --- a/crates/tiff/README.md +++ b/crates/tiff/README.md @@ -18,6 +18,10 @@ later the TIFF/DNG exporters. - `image::ImageInfo` — width/height, bits, samples, compression, photometric, planar config, predictor, sample format and the strip or tile chunk list (`chunks()`, `chunk_bytes()`); missing byte counts are inferred. +- `directory_entries`, `entry_span`, `ParseOptions::budget_for` and `decode_value` are the reader's own rules + (directory layout and entry limit, which entries it reads, its value budget, value decoding), for callers that + must select entries exactly as `parse_ifd_at` would without decoding the whole directory (`lightcraft-raw`'s + Sony `SR2Private` lookup). - `TiffWriter` writes IFD chains with strips or tiles, `SubIFDs` and pointer IFDs, in either byte order, classic or BigTIFF; `writer::rational` / `srational` approximate floats as rationals. diff --git a/crates/tiff/src/lib.rs b/crates/tiff/src/lib.rs index bafa3eb5e..1d0b74d0d 100644 --- a/crates/tiff/src/lib.rs +++ b/crates/tiff/src/lib.rs @@ -25,7 +25,7 @@ mod reader; pub mod tags; pub mod writer; -pub use reader::{ParseOptions, parse_ifd_at}; +pub use reader::{DirectoryEntries, EntrySpan, ParseOptions, decode as decode_value, directory_entries, entry_span, parse_ifd_at}; pub use writer::{IfdBuilder, ImageData, TiffWriter}; use serde::{Deserialize, Serialize}; diff --git a/crates/tiff/src/reader.rs b/crates/tiff/src/reader.rs index a644159b7..4a80a906b 100644 --- a/crates/tiff/src/reader.rs +++ b/crates/tiff/src/reader.rs @@ -26,6 +26,101 @@ impl Default for ParseOptions { } } +impl ParseOptions { + /// The value budget for `data_len` bytes of input: [`Self::value_budget`], else `8 × data_len + 1 MiB`. + pub fn budget_for(&self, data_len: usize) -> u64 { + self.value_budget.unwrap_or((data_len as u64).saturating_mul(8).saturating_add(1 << 20)) + } +} + +/// Where a directory's entries are and how many of them the reader reads, as [`directory_entries`] found them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DirectoryEntries { + /// Position of the first entry. + table: usize, + /// Entries read: the declared count, cut to the entries the data holds and to the `max_entries` limit. + read: usize, + /// The declared entry count. + declared: u64, + /// Bytes per entry: 12, or 20 in BigTIFF. + entry_size: usize, +} + +impl DirectoryEntries { + /// Positions of the entries read, in directory order. Checked arithmetic: the iteration ends rather than + /// overflow, whatever the layout. + pub fn positions(&self) -> impl Iterator + use<> { + let (table, size) = (self.table, self.entry_size); + (0..self.read).map_while(move |i| i.checked_mul(size)?.checked_add(table)) + } + + /// The number of entries read. + pub fn read(&self) -> usize { + self.read + } + + /// The entry count the directory declares. + pub fn declared(&self) -> u64 { + self.declared + } +} + +/// The directory at `offset` as the reader walks it: its entry table and how many entries it reads. The same errors +/// as [`parse_ifd_at`] for a directory it can't read: past the data, empty, or without room for one entry. +pub fn directory_entries(data: &[u8], offset: u64, order: ByteOrder, bigtiff: bool, max_entries: usize) -> Result { + let pos = usize::try_from(offset).map_err(|_| TiffError::Truncated(offset))?; + let (n, head, entry_size) = if bigtiff { + (order.read_u64(data, pos).ok_or(TiffError::Truncated(offset))?, 8usize, 20usize) + } else { + (order.read_u16(data, pos).ok_or(TiffError::Truncated(offset))? as u64, 2, 12) + }; + let table = pos + head; + let fit = (data.len().saturating_sub(table) / entry_size) as u64; + let read = n.min(fit).min(max_entries as u64) as usize; + if n == 0 { + return Err(TiffError::Invalid(format!("empty IFD at {offset}"))); + } + if read == 0 { + return Err(TiffError::Truncated(offset)); + } + Ok(DirectoryEntries { table, read, declared: n, entry_size }) +} + +/// A directory entry the reader can read: its tag, type and count, and where its values are. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EntrySpan { + pub tag: u16, + pub field_type: FieldType, + pub count: u64, + /// Position of the values in the data: the entry's own value field when they fit it, else `base` plus the offset + /// stored there. + pub start: u64, + /// `count × size` bytes. + pub len: u64, +} + +/// The entry at `pos`, when the reader reads it: a known field type, `count × size` without overflow, and values +/// inside `data`. Nothing is decoded; [`parse_ifd_at`] also drops an entry whose values exceed what is left of its +/// value budget ([`ParseOptions::budget_for`]), which this leaves to the caller. +pub fn entry_span(data: &[u8], pos: usize, order: ByteOrder, bigtiff: bool, base: u64) -> Option { + let tag = order.read_u16(data, pos)?; + let field_type = FieldType::from_u16(order.read_u16(data, pos.checked_add(2)?)?)?; + let (count, field, inline) = if bigtiff { + (order.read_u64(data, pos.checked_add(4)?)?, pos.checked_add(12)?, 8u64) + } else { + (order.read_u32(data, pos.checked_add(4)?)? as u64, pos.checked_add(8)?, 4) + }; + let len = count.checked_mul(field_type.size() as u64)?; + let start = if len <= inline { + field as u64 + } else { + let rel = if bigtiff { order.read_u64(data, field)? } else { order.read_u32(data, field)? as u64 }; + base.checked_add(rel)? + }; + let end = start.checked_add(len)?; + (end <= data.len() as u64).then_some(EntrySpan { tag, field_type, count, start, len }) +} + /// Parse the header: (byte order, bigtiff, first IFD offset). pub(crate) fn header(data: &[u8]) -> Result<(ByteOrder, bool, u64)> { header_with(data, true) @@ -97,7 +192,7 @@ struct Ctx<'a> { impl<'a> Ctx<'a> { fn new(data: &'a [u8], order: ByteOrder, big: bool, opts: &'a ParseOptions) -> Self { - let budget = opts.value_budget.unwrap_or((data.len() as u64).saturating_mul(8).saturating_add(1 << 20)); + let budget = opts.budget_for(data.len()); Self { data, order, big, opts, visited: HashSet::new(), budget } } @@ -116,34 +211,18 @@ impl<'a> Ctx<'a> { if !self.visited.insert(offset) { return Err(TiffError::Invalid(format!("IFD loop at {offset}"))); } - let pos = usize::try_from(offset).map_err(|_| TiffError::Truncated(offset))?; - let (n, head, entry_size, inline) = if self.big { - (self.order.read_u64(self.data, pos).ok_or(TiffError::Truncated(offset))?, 8usize, 20usize, 8usize) - } else { - (self.order.read_u16(self.data, pos).ok_or(TiffError::Truncated(offset))? as u64, 2, 12, 4) - }; - let table = pos + head; - let fit = (self.data.len().saturating_sub(table) / entry_size) as u64; - let n_read = n.min(fit).min(self.opts.max_entries as u64) as usize; - if n == 0 { - return Err(TiffError::Invalid(format!("empty IFD at {offset}"))); - } - if n_read == 0 { - return Err(TiffError::Truncated(offset)); - } - let mut entries = Vec::with_capacity(n_read); - for i in 0..n_read { - let p = table + i * entry_size; - if let Some(e) = self.entry(p, base, inline) { + let dir = directory_entries(self.data, offset, self.order, self.big, self.opts.max_entries)?; + let mut entries = Vec::with_capacity(dir.read); + for p in dir.positions() { + if let Some(e) = self.entry(p, base) { entries.push(e); } } entries.sort_by_key(|e| e.tag); entries.dedup_by_key(|e| e.tag); - // the next-IFD offset follows the whole table, so only a table read in full has one: a count far past the - // file (a BigTIFF's is 64-bit) must not reach the arithmetic (issue #1516) - let next = if n == n_read as u64 { - match n_read.checked_mul(entry_size).and_then(|len| table.checked_add(len)).and_then(|p| self.read_off(p)) { + // Only a fully read table has a next-IFD offset. Keep hostile BigTIFF counts out of the arithmetic (#1516). + let next = if dir.declared == dir.read as u64 { + match dir.read.checked_mul(dir.entry_size).and_then(|len| dir.table.checked_add(len)).and_then(|p| self.read_off(p)) { Some(0) | None => 0, Some(v) => base.checked_add(v).unwrap_or(0), } @@ -181,33 +260,19 @@ impl<'a> Ctx<'a> { ifd.interop = interop; } - fn entry(&mut self, p: usize, base: u64, inline: usize) -> Option { - let o = self.order; - let tag = o.read_u16(self.data, p)?; - let typ = FieldType::from_u16(o.read_u16(self.data, p + 2)?)?; - let (count, field) = if self.big { (o.read_u64(self.data, p + 4)?, p + 12) } else { (o.read_u32(self.data, p + 4)? as u64, p + 8) }; - let total = count.checked_mul(typ.size() as u64)?; - let start = if total <= inline as u64 { - field as u64 - } else { - let rel = self.read_off(field)?; - base.checked_add(rel)? - }; - let end = start.checked_add(total)?; - if end > self.data.len() as u64 { + fn entry(&mut self, p: usize, base: u64) -> Option { + let span = entry_span(self.data, p, self.order, self.big, base)?; + if span.len > self.budget { return None; } - if total > self.budget { - return None; - } - self.budget -= total; - let bytes = &self.data[start as usize..end as usize]; - Some(Entry { tag, value: decode(o, typ, bytes), offset: start }) + self.budget -= span.len; + let bytes = self.data.get(usize::try_from(span.start).ok()?..usize::try_from(span.start + span.len).ok()?)?; + Some(Entry { tag: span.tag, value: decode(self.order, span.field_type, bytes), offset: span.start }) } } -/// Decode `bytes` (exactly `count × size` long) as values of type `typ`. -pub(crate) fn decode(o: ByteOrder, typ: FieldType, b: &[u8]) -> Value { +/// Decode `bytes` (exactly `count × size` long, or the first values of an entry) as values of type `typ`. +pub fn decode(o: ByteOrder, typ: FieldType, b: &[u8]) -> Value { fn arr(c: &[u8]) -> [u8; N] { let mut a = [0u8; N]; a.copy_from_slice(c); @@ -237,3 +302,24 @@ pub(crate) fn decode(o: ByteOrder, typ: FieldType, b: &[u8]) -> Value { FieldType::SLong8 => Value::SLong8(b.as_chunks::<8>().0.iter().map(|c| o.u64(arr(c)) as i64).collect()), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn directory_positions_never_overflow() { + // a hand-built layout the reader never produces: the iterator stops instead of overflowing + let hostile = DirectoryEntries { table: usize::MAX, read: 2, declared: 2, entry_size: 12 }; + assert_eq!(hostile.positions().collect::>(), vec![usize::MAX]); + assert_eq!(hostile.positions().nth(1), None); + let wide = DirectoryEntries { table: usize::MAX - 30, read: 4, declared: 4, entry_size: 12 }; + assert_eq!(wide.positions().collect::>(), vec![usize::MAX - 30, usize::MAX - 18, usize::MAX - 6]); + // a directory read from data lists its entries + let mut data = Vec::new(); + ByteOrder::Little.put_u16(&mut data, 2); + data.extend([0; 24]); + let dir = directory_entries(&data, 0, ByteOrder::Little, false, 8192).unwrap(); + assert_eq!(dir.positions().collect::>(), vec![2, 14]); + } +} diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md index 7bb187811..e5e0feb9d 100644 --- a/docs/parity-checklist.md +++ b/docs/parity-checklist.md @@ -211,7 +211,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts). | LR-EDIT-DETAIL-SUPERRES | Super resolution | P2 | ⬜ | | | | LR-EDIT-DETAIL-AISHARPEN | AI sharpen | OOS | 🚫 | | | | LR-EDIT-OPTICS-CA | Remove chromatic aberration | P1 | ✅ | `crates/ui-egui/src/panels/edit.rs` (checkbox), `ctl:optics.caRed`, `ctl:optics.caBlue` | | -| LR-EDIT-OPTICS-PROFILE | Lens profile corrections | P1 | 🟡 | `ctl:optics.profileDistortion`, `ctl:optics.profileVignetting`, `crates/pipeline/src/optics.rs`, `crates/raw/src/vendor/rw2.rs`, `crates/raw/src/vendor/raf_lens.rs`, `crates/engine/src/crs.rs` | uses corrections embedded in the file: DNG `WarpRectilinear` / `WarpRectilinear2` (opcode 14, DNG 1.6: degree-14 radial polynomial, valid-radius clamp, reciprocal option, applied in list order with the spec's skip rule for a `WarpRectilinear` after an optional one, so Pixel DNGs' two-stage warp is corrected; up to 3 chained stages, sampled on the CPU because the polynomials need `f64`; issue #635) / `FixVignetteRadial`, and the camera's own distortion correction in Panasonic / Leica RW2 / RWL files (tag `0x0119`, read as the equivalent `WarpRectilinear`; on 79 photos from 13 bodies and 11 lenses the corrected render lines up with the camera JPEG within 0.75 px at 1440 px at the corners, from up to 13 % of the half diagonal uncorrected; issue #256); an imported preset's `LensProfileEnable=0` leaves them on, as Lightroom applies built-in corrections regardless (inferred); Sony ILCE-7RM4A Bayer ARW raw-IFD distortion tables (`0x7037`, 16-sample layout, native 3:2 framing) use the same warp path, enabled only for the independently validated ILCE-7RM4A with FE 24–105 and FE 200–600 lenses; other Sony models remain uncorrected (see `docs/sony-lens-corrections.md`); Nikon NEF distortion from NikonNEFInfo (`0xc7d5`: polynomial, frame-filling zoom and the optical-centre offset left by in-body stabilisation; `crates/raw/src/vendor/nef_lens.rs`, `docs/nikon-lens-corrections.md`) for the Z 6II, Z 8 and Z f, full-frame image area only: F-mount lenses on the FTZ match the camera JPEG within 0.2 px median, Z lenses within 0.5–1.5 px except the 24-70 f/4's wide end, where the camera applies a further correction that is not in the readable data (7 px median at 24 mm, from 16 px / 106 px p95 uncorrected); Fujifilm RAF raw-IFD tables (`0xf00b` distortion, `0xf010` vignetting; `crates/raw/src/vendor/raf_lens.rs`, `docs/fuji-lens-corrections.md`) become a `WarpRectilinear` with the camera's frame-filling zoom and a `FixVignetteRadial` gain for 59 body models with sample files checked against the camera JPEG (the table indexed by the source radius with last knot at the corner; on 67 body/lens combinations the corrected render's radial shape error against the camera JPEG falls from a median of 0.72 % to 0.05 % of the half diagonal, largest 0.42 %, and the vignetting gain of the JPEG tracks the table at a median correlation of 0.987); files whose table scale is not the recorded image's half diagonal (X-S20, X-T30 III, X-M5, X-T5 crops) and six unmatched bodies stay uncorrected; Fujifilm's lateral CA table (`0xf00f`) is real but is not applied, because our demosaic already registers the planes and the stored shift over-corrects; vignetting and lateral CA data of Olympus and Sony raws not read yet; Reload from Disk reads the file's lens data for photos catalogued without it and turns the correction on for those still at import settings (issue #314); lens-profile database: a lensfun-format XML database in `LIGHTCRAFT_LENSFUN_DIR` (none is bundled) supplies distortion (poly3 / poly5 / PTLens), TCA and vignetting for lenses it lists (`crates/lensdb`, `crates/engine/src/lens_profiles.rs`, `docs/lens-profiles.md`); no settings switch or bundled database yet | +| LR-EDIT-OPTICS-PROFILE | Lens profile corrections | P1 | 🟡 | `ctl:optics.profileDistortion`, `ctl:optics.profileVignetting`, `crates/pipeline/src/optics.rs`, `crates/raw/src/vendor/rw2.rs`, `crates/raw/src/vendor/raf_lens.rs`, `crates/engine/src/crs.rs` | uses corrections embedded in the file: DNG `WarpRectilinear` / `WarpRectilinear2` (opcode 14, DNG 1.6: degree-14 radial polynomial, valid-radius clamp, reciprocal option, applied in list order with the spec's skip rule for a `WarpRectilinear` after an optional one, so Pixel DNGs' two-stage warp is corrected; up to 3 chained stages, sampled on the CPU because the polynomials need `f64`; issue #635) / `FixVignetteRadial`, and the camera's own distortion correction in Panasonic / Leica RW2 / RWL files (tag `0x0119`, read as the equivalent `WarpRectilinear`; on 79 photos from 13 bodies and 11 lenses the corrected render lines up with the camera JPEG within 0.75 px at 1440 px at the corners, from up to 13 % of the half diagonal uncorrected; issue #256); an imported preset's `LensProfileEnable=0` leaves them on, as Lightroom applies built-in corrections regardless (inferred); Sony ILCE-7RM4A Bayer ARW raw-IFD distortion tables (`0x7037`, 16-sample layout, native 3:2 framing) use the same warp path, enabled only for the independently validated ILCE-7RM4A with FE 24–105 and FE 200–600 lenses; other Sony models remain uncorrected (see `docs/sony-lens-corrections.md`); Nikon NEF distortion from NikonNEFInfo (`0xc7d5`: polynomial, frame-filling zoom and the optical-centre offset left by in-body stabilisation; `crates/raw/src/vendor/nef_lens.rs`, `docs/nikon-lens-corrections.md`) for the Z 6II, Z 8 and Z f, full-frame image area only: F-mount lenses on the FTZ match the camera JPEG within 0.2 px median, Z lenses within 0.5–1.5 px except the 24-70 f/4's wide end, where the camera applies a further correction that is not in the readable data (7 px median at 24 mm, from 16 px / 106 px p95 uncorrected); Fujifilm RAF raw-IFD tables (`0xf00b` distortion, `0xf010` vignetting; `crates/raw/src/vendor/raf_lens.rs`, `docs/fuji-lens-corrections.md`) become a `WarpRectilinear` with the camera's frame-filling zoom and a `FixVignetteRadial` gain for 59 body models with sample files checked against the camera JPEG (the table indexed by the source radius with last knot at the corner; on 67 body/lens combinations the corrected render's radial shape error against the camera JPEG falls from a median of 0.72 % to 0.05 % of the half diagonal, largest 0.42 %, and the vignetting gain of the JPEG tracks the table at a median correlation of 0.987); files whose table scale is not the recorded image's half diagonal (X-S20, X-T30 III, X-M5, X-T5 crops) and six unmatched bodies stay uncorrected; Fujifilm's lateral CA table (`0xf00f`) is real but is not applied, because our demosaic already registers the planes and the stored shift over-corrects; vignetting and lateral CA data of Olympus raws not read yet; Sony ARW shading, lateral CA and distortion tables (raw IFD `0x7031`–`0x7037`, with the encrypted `SR2SubIFD` copies as a checked fallback) are decoded with their camera settings, provenance and framing evidence for every body that records them (`crates/raw/src/lens_metadata.rs`), but only the ILCE-7RM4A distortion above is applied: on the ILCE-7CR the unchanged distortion interpretation misses the camera JPEG at 24 mm (median 1.6 px, p95 6.2 px at 2376 px), and Sony shading and CA semantics are not validated on any body (`docs/sony-lens-corrections.md`); Reload from Disk reads the file's lens data for photos catalogued without it and turns the correction on for those still at import settings (issue #314); lens-profile database: a lensfun-format XML database in `LIGHTCRAFT_LENSFUN_DIR` (none is bundled) supplies distortion (poly3 / poly5 / PTLens), TCA and vignetting for lenses it lists (`crates/lensdb`, `crates/engine/src/lens_profiles.rs`, `docs/lens-profiles.md`); no settings switch or bundled database yet | ) | LR-EDIT-OPTICS-DEFRINGE | Defringe | P1 | ✅ | `ctl:optics.defringe*` | no fringe eyedropper | | LR-EDIT-OPTICS-MANUAL | Manual distortion / vignetting | P1 | ✅ | `ctl:optics.distortion`, `ctl:optics.vignetting`, `ctl:optics.vignettingMidpoint` | | @@ -790,6 +790,7 @@ Latin stays upright. Advanced Japanese composition remains open. Regression cove | Date | Change | Summary | |---|---|---| +| 2026-10-12 | minor | LR-EDIT-OPTICS-PROFILE: Sony ARW shading, CA and distortion tables decoded with settings and provenance (raw IFD, `SR2SubIFD` fallback); applied corrections unchanged (ILCE-7RM4A distortion only); status unchanged (🟡) | | 2026-10-13 | patch | LR-VIEW-COMPARE, LRC-LIB-COMPARE: ⌘/Ctrl- and Shift-clicks on the filmstrip in Compare change the selection as in the grid; the two photos shown stay selected and the active photo is unchanged (#187); statuses unchanged | | 2026-10-13 | minor | KEYC-SELECTONLYACTIVE: Edit → Select Only Active Photo (⌘⇧D) lands as a new row; status 🟡 (headless-tested, native macOS menu path unverified) | | 2026-10-13 | patch | LRC-LIB-CATALOG-IMPORT: a catalog whose tables are all still in its `-wal` file (main file schema format 0) opens instead of failing as an unsupported schema format; status unchanged | diff --git a/docs/sony-lens-corrections.md b/docs/sony-lens-corrections.md index 375f773ab..53c5e9755 100644 --- a/docs/sony-lens-corrections.md +++ b/docs/sony-lens-corrections.md @@ -1,25 +1,137 @@ -# Sony embedded distortion corrections +# Sony embedded lens corrections -LightCraft reads the signed 16-sample distortion table in the raw image IFD (`0x7037`) of -Sony **ILCE-7RM4A (A7R IVA)** Bayer ARWs. Other camera models are deliberately left uncorrected, -even when they contain a table with the same layout. -It converts that table into `OpcodeList3` / `WarpRectilinear`, so the existing profile correction control, -CPU/GPU optics path, EXIF orientation handling, and DNG export use the same correction. +Sony ARWs record three lens corrections per shot in the raw image IFD: shading (vignetting), lateral chromatic +aberration (CA) and distortion, each as a camera setting and a radial table, with copies in the encrypted +`SR2SubIFD`. LightCraft **decodes all three** wherever a file carries them and **applies one**: the distortion table +of the **ILCE-7RM4A (A7R IVA)** in Bayer ARWs with native 3:2 framing, independently validated with the +FE 24–105mm F4 G OSS and FE 200–600mm F5.6–6.3 G OSS. Every other model, and shading and CA on every model, are +decoded only: no warp, vignetting or CA correction is added for them. The decoded record stays with the image +(`RawInfo::lens_metadata` / `RawImage::lens_metadata`, types in `crates/raw/src/lens_metadata.rs`, decoder in +`crates/raw/src/vendor/arw_lens.rs`) for diagnostics and for the validation work described below. -This adds **distortion only**. Sony vignetting and lateral chromatic-aberration tables are not decoded. -The enabled and independently validated model is ILCE-7RM4A with the FE 24–105mm F4 G OSS and -FE 200–600mm F5.6–6.3 G OSS. Linear YCbCr ARWs and non-3:2 camera crops are excluded pending validation. An aspect crop may -retain the full-frame radial normalization; the decoder does not guess that relationship. Older files that carry -only encrypted correction metadata, different table lengths, and rejected tables remain uncorrected. -The camera's distortion Off setting does not erase its table. Newly imported photos use LightCraft's existing -embedded-profile default (enabled); the profile correction control can disable or adjust it. +The applied distortion is converted into `OpcodeList3` / `WarpRectilinear`, so the existing profile correction +control, CPU/GPU optics path, EXIF orientation handling, and DNG export use the same correction. Linear YCbCr ARWs +and non-3:2 camera crops are excluded pending validation. An aspect crop may retain the full-frame radial +normalization; the decoder does not guess that relationship. Tables of other lengths and rejected tables remain +uncorrected, and a table found only in the encrypted `SR2SubIFD` is never applied. The camera's distortion Off +setting does not erase its table. Newly imported photos use LightCraft's existing embedded-profile default +(enabled); the profile correction control can disable or adjust it. + +## What is decoded + +| Component | Raw-IFD setting / table | `SR2SubIFD` setting / table | Accepted layout | +|---|---|---|---| +| Shading | `0x7031` / `0x7032` | `0x797c` / `0x797d` | SHORT\[1\]; SSHORT\[17\]: 16, then 16 samples | +| Lateral CA | `0x7034` / `0x7035` | `0x797f` / `0x7980` | SHORT\[1\]; SSHORT\[33\]: 32, then two 16-sample curves | +| Distortion | `0x7036` / `0x7037` | `0x7981` / `0x7982` | SHORT\[1\]; SSHORT\[17\]: 16, then 16 samples | + +- **Bounded, exact reading.** A raw-IFD field is the one the TIFF reader parsed, the copy the image itself uses: + its first occurrence, within the reader's limits (8,192 entries per directory, a total value budget). The directory + is also scanned, in either byte order and in BigTIFF, for diagnostics only: a tag that occurs more than once is + flagged (`raw_ifd_repeated`, `sr2_repeated`), and an entry the reader left out is reported with its reason (values + outside the file, an unknown type, or past the reader's limits) rather than treated as absent. `SR2SubIFD` fields + are read straight from their first entry. A setting may also be a nonnegative SSHORT (the documented editor type + differs from the type the cameras store). Anything else is refused with its reason (`SonyFieldError`: field type, + count, leading sample count, values outside the file or block, past the reader's limits, a negative setting). + Nothing is allocated from a count in the file. +- **Raw IFD first, `SR2SubIFD` as a checked fallback.** The raw IFD's copy wins. The `SR2SubIFD`'s copy is used only + when the raw IFD lacks the tag; a malformed raw-IFD copy is reported, never silently replaced. When both copies are + well-formed their agreement is recorded (`copies_agree`), and on disagreement the raw IFD's copy is used. The block + is decrypted (by the existing `vendor/sr2.rs`) for every ARW that has one, so the check always runs; a block that + can't be found or decrypted gives no copies. Locating it selects the `SR2Private` directory's offset, length and + key exactly as the TIFF reader would (the reader's own directory layout, 8,192-entry limit, readable-entry rules + and value budget, then the first readable occurrence of each), but decodes only each field's first value, so the + directory's other entries cost no memory however large they claim to be; the block itself is limited to 1 MiB. The + plain tag numbers are not looked up inside the `SR2SubIFD`. +- **Settings keep their number and a documented meaning:** shading 256 Off, 257 and 272 Auto, 511 unavailable; CA + 0 Off, 1 Auto, 255 unavailable; distortion 0 Off, 1 Auto, 17 Auto fixed by the lens, 255 unavailable. Any other value + is `Unknown(value)`, never Off (the A7CR's Low shading setting has no documented number). Values are not read as + bit masks. Off does not mean absent: the table is still recorded. +- **Only the 16-sample (CA: 2 × 16) layout is interpreted.** Many bodies store 11 (CA: 22) samples padded into the + same 17 (33) values: older compacts and bodies such as the DSC-RX100 III and ILCE-3500, and the ILCE-7M4's APS-C + captures (its full-frame captures carry 16). Those tables are refused as `CountPrefix(11)` and stay + uninterpreted. An all-zero table is kept: it is a valid no-op, not a sign of an unsupported lens. +- **Framing evidence travels with the tables:** the stored frame (padding included), Sony's crop tags and the DNG + default crop as recorded, the image crop, the reference rectangle the distortion conversion is centred on and + normalized to (the default crop when present, else the image crop), compression, whether the data is linear + YCbCr (lossless M/S), and whether both crops are native 3:2. +- **Per-component status.** `Applied` (converted into an opcode), `NoTable`, `NotValidatedComponent` (shading and CA + on every body), `NotValidatedModel`, `NotValidatedMode` (linear YCbCr, non-3:2 framing, or a distortion table found + only in the `SR2SubIFD`), `Neutral` (all-zero table in the validated scope) and `Rejected` (the validated + conversion refused the table). A readable table never implies a usable correction; only `Applied` components + become opcodes. + +The applied path is unchanged: the validated conversion still receives the raw IFD's `0x7037` table exactly as the +TIFF reader parsed it, with the same model, framing and fit checks, and produces the same warp. A repeated tag keeps +the reader's choice and is only flagged, and a table the reader never reads (past its entry limit) is reported but +never applied, so already-catalogued photos, their previews and their stored lens data stay consistent with a fresh +decode. + +## What each component still needs + +The following summarizes an investigation on the ILCE-7CR (A7CR): an owner's collection of 131 raws with four +lenses (Voigtländer APO-LANTHAR 28mm F2 and 50mm F2, both manual focus; Sony FE 24–50mm F2.8 G; Tamron 35–150mm +F2–2.8), plus the three public raw.pixls.us A7CR samples. No private photographs, names or derived images are +part of this repository. + +- **Inventory.** All 131 raws carry all three tables in both the raw IFD and the `SR2SubIFD`, and the copies agree + in every file; all four lenses provide tables, the manual-focus ones included. Shading is 272 (Auto) and CA 1 + (Auto) everywhere; distortion is 17 (Auto, fixed by the lens) on the nine FE 24–50 files and 0 (Off) on the rest. + Every image crop is 9504 × 6336 at (32, 20), in uncompressed, compressed and lossless-L (padded to 9728 × 6656) + storage. The decoder reproduces this inventory exactly (see *Local inventory check*). +- **Distortion.** The table layout and crop tags match the A7R IVA's, but its interpretation applied unchanged does + not reproduce the A7CR camera JPEG at 24 mm: at 2376 px wide, median / 95th-percentile feature residuals were + 1.61 / 6.17 px (12.98 / 51.19 px uncorrected, 6,539 matches), with a signed radial pattern (about +1.7 px at + radii 0.25–0.5, −7.4 px at 0.8–1) that no translation removes. At 50 mm it agrees (0.23 / 1.05 px, from + 2.21 / 12.25 px). Scaling the raw table by about 10/9, or using the MakerNote's distortion array instead, brings + 24 mm to about 0.37 / 1.10 and 0.33 / 0.97 px. Those are exploratory comparisons on two files against the camera + JPEG, not a model: the MakerNote values are roughly 1.11 times the raw table without one exact ratio, and the + A7R IVA was validated against a different reference renderer (Imaging Edge exports). Needed: A7CR Imaging Edge + On/Off exports at 24 and 50 mm and with another lens, compared with the raw table at its fixed scale; a decision on + the reference (camera JPEG or Imaging Edge); then ILCE-7CR added to the validated models for the full-frame Bayer + 3:2 mode only. Neither a 10/9 multiplier nor the MakerNote table is used. +- **Shading.** The layout decodes, but no public specification gives the sample-to-gain function, its radius + convention on the A7CR, or whether the raw samples already contain part of the correction. Brightness ratios on six + natural scenes show no consistent missing peripheral brightening. Needed: evenly lit flat fields with shading + Off / Auto / Low at several apertures and focal lengths, raw CFA ratios between them, and Imaging Edge On/Off + exports, to fix the gain function and processing stage and rule out double correction. +- **Lateral CA.** A public analysis reads the table as a red, then a blue curve relative to green, at a 2⁻²¹ + relative radial scale. On three distortion-Off files that candidate reduced red and blue feature residuals + modestly (for example 0.35 → 0.29 px median for red with the 28 mm), which supports investigating it but certifies + neither scale, interpolation nor processing stage. Two product changes come before it: embedded CA must be + independent of the distortion amount (today `pipeline::optics` scales the whole embedded per-plane warp with the + profile distortion control), and the red and blue planes must be resampled in camera RGB, before the camera colour + model mixes them (`engine::files::load_bytes_now` applies the colour model before the optics stage). Then + validation on full-resolution achromatic edges. +- **Other modes.** APS-C, aspect crops and lossless M/S (linear YCbCr) captures were not measured and stay + excluded. + +## Local inventory check + +`crates/raw/tests/sony_lens_inventory.rs` is opt-in and does nothing in CI. With `LIGHTCRAFT_SONY_LENS_DIR` naming a +directory of ARWs it decodes each from its headers and prints its record (settings, sources, copies' agreement, +sample ranges, statuses); `LIGHTCRAFT_SONY_LENS_FULL=1` also compares header and full decode. With +`LIGHTCRAFT_SONY_LENS_INVENTORY` naming a CSV inventory it checks every file's table counts, sizes, sample counts, +minima and maxima, raw-IFD versus `SR2SubIFD` agreement, settings (numbers and labels), stored size, compression and +crop tags. A file with a row must decode, and every file must have a row, whether it decodes or not. Any decode +error fails the check except an unsupported variant, which is counted and must equal `LIGHTCRAFT_SONY_LENS_UNSUPPORTED` +(default 0). On the 131 A7CR raws above, all 131 inventory rows reproduce and every file decodes. On 133 public Sony +ARWs (127 raw.pixls.us samples from 100 bodies, four corpus copies and two files attached to a public issue), 110 carry +a record, 14 decode without one, and 9 are unsupported variants (eight packed ARWs of older bodies and one ARW +without a CFA image); header and full decode agree on the 124 that decode, and no record has disagreeing copies. ## Clean-room evidence -The [ExifTool EXIF tag-name documentation](https://exiftool.org/TagNames/EXIF.html) identifies -`0x7037` as 17 signed 16-bit integers. The first value is the count, 16, followed by radial samples. -No external photo decoder implementation or Adobe lens profile was consulted. -Sony Imaging Edge Edit 4.1 was used only as a black-box reference on local scratch copies. +The [ExifTool EXIF tag-name documentation](https://exiftool.org/TagNames/EXIF.html) names the raw-IFD tags +`0x7031` to `0x7037` (VignettingCorrection / VignettingCorrParams, ChromaticAberrationCorrection / +ChromaticAberrationCorrParams, DistortionCorrection / DistortionCorrParams) with their setting values, and gives the +tables as 17 or 33 signed 16-bit integers whose first value is the count. The +[Sony SR2SubIFD documentation](https://exiftool.org/TagNames/Sony.html#SR2SubIFD) names the encrypted copies +`0x797d`, `0x7980` and `0x7982`; the setting tags next to them (`0x797c`, `0x797f`, `0x7981`) are unnamed there and +were identified by inspecting files, where they equal the raw-IFD settings. The corpus test's expected tables are the +values ExifTool, run as a black-box tool, prints for the public samples. No external photo decoder implementation, +lens database or Adobe lens profile was consulted. Sony Imaging Edge Edit 4.1 was used only as a black-box reference +on local scratch copies. The observed inverse mapping, with radius `r` normalized to the default crop's half diagonal, is: @@ -45,7 +157,8 @@ so the warp stays on the optical centre either way. On the A7R IVA that framing pixels left of Sony's exports (eight pixels in a 2376-pixel-wide export). The main raw-IFD table is important: the similarly named MakerNote table has different values and -must not be substituted into this formula. Main raw-IFD and SR2SubIFD values agreed in the test files. +must not be substituted into this formula. Main raw-IFD and SR2SubIFD values agreed in the test files, and the +`SR2SubIFD` copy is never converted. ## Validation @@ -90,7 +203,18 @@ The distortion geometry remains aligned while every body's image framing follows Tests cover signedness/length rejection, zero and malformed tables, independent 600 mm reference geometry, barrel framing, offset crop normalization, both TIFF byte orders, header/full consistency, -and preservation through DNG export. No private photographs or reference TIFFs are committed. +and preservation through DNG export. The decoder's synthetic tests (`crates/raw/src/vendor/arw_lens.rs`) add both +byte orders and BigTIFF, exact SHORT / SSHORT types, negative samples, missing, short, long, huge and zero counts, +wrong leading counts, values outside the file, a table past the TIFF reader's entry limit (not applied), repeated +tags (the reader's copy kept, as before), unknown and negative settings, raw-IFD versus +`SR2SubIFD` precedence and agreement, the correct `SR2SubIFD` tag numbers, a wrong decryption key, hostile entries +inside the decrypted block, linear YCbCr, framing evidence (Sony's crop at x = 0 against the default crop, agreeing +crops, padded storage, aspect crops, crop tags outside the frame), all eight EXIF orientations leaving the record +unchanged, and neutral, out-of-range and folding tables. On a small classic-TIFF fixture in both byte orders, every +truncation and four mutations of each byte outside the pixel data (set to 0x00 and 0xff, top and low bit flipped) +decode or fail without a panic. `tests/sr2_heap.rs` measures the peak heap of decoding a 16 MiB file whose +`SR2Private` directory holds eight fields aliasing the whole file: 37 kB, against 134 MB when those fields were +decoded. No private photographs or reference TIFFs are committed. ## Camera coverage and regression boundary @@ -99,27 +223,35 @@ model names, and every other model remain unsupported by this distortion decoder numbers and table lengths do not establish the scale, interpolation or crop normalization on another body. Support can expand after independent geometry validation; do not add model-name prefix matching. -The current `cargo xtask corpus --download` manifest contains 23 Sony CC0 samples across 17 bodies, -including samples 4822 (A7R IVA / 24–105 mm) and 3989 (A9 II / 200–600 mm). All 23 files have -pinned SHA-256 hashes and are checked with `sony_embedded_distortion_is_limited_to_validated_models`: +The current `cargo xtask corpus --download` manifest contains 25 Sony CC0 samples across 18 bodies, +including samples 4822 (A7R IVA / 24–105 mm) and 3989 (A9 II / 200–600 mm), each with a pinned SHA-256 that the +download checks. `sony_embedded_distortion_is_limited_to_validated_models` checks 23 of them (the two ILCE-7CM2 +codings have their own test), and `sony_lens_records_of_the_public_samples` verifies all 25 against their pins +(`SONY_SAMPLES` in `crates/raw/tests/corpus.rs`, itself anchored by a pinned digest and compared with the manifest) +before decoding. **No CI job fetches the corpus**: these fixture tests run only where it has been downloaded, and +otherwise skip each absent file. `LIGHTCRAFT_CORPUS_REQUIRED=1` makes `sony_lens_records_of_the_public_samples` +fail on any absent pinned file instead. -| Camera | Corpus samples | Raw-IFD table observed | Expected correction | +| Camera | Corpus samples | Decoded record: tables, settings (shading / CA / distortion) | Applied | |---|---|---|---| -| ILCE-7RM4A | Compressed (4822) | 16 samples | One distortion warp | -| ILCE-9M2 | Compressed (3989) | 16 samples | None | -| ILCE-7M3 | Compressed, uncompressed | 16 samples | None | -| ILCE-7M4 | 14-bit, lossless L/M/S | 11 samples in the 14-bit file; 16 in L/M/S | None | -| ILCE-7RM2 | 12-bit uncompressed | 16 samples | None | -| DSC-RX100 | One | No raw-IFD table | None | -| DSC-RX100M3 | One | 11 samples | None | -| ILCE-7CR | Compressed, lossless L/M | Not used; model excluded | None | -| ILCE-7RM4 | 14-bit compressed | Not used; model excluded | None | -| DSLR-A500, DSLR-A700, SLT-A33 | One each | Not used; model excluded | None | -| ILCE-3500, ILCE-7S | 5600 K, Shade respectively | Not used; model excluded | None | -| ILCE-7SM2, DSLR-A580 | 16:9 framing, one each | Not used; model excluded | None | -| DSLR-A900 | Packed 12-bit | Not used; model excluded | None | +| ILCE-7RM4A | Compressed (4822) | All three tables; 257 / 1 / 17 | One distortion warp | +| ILCE-9M2 | Compressed (3989) | All three; 257 / 1 / 0 | None | +| ILCE-7M3 | Compressed, uncompressed | All three; 257 / 1 / 0 | None | +| ILCE-7M4 | 14-bit, lossless L/M/S | L/M/S: all three, 272 / 1 / 17; the 14-bit file (APS-C framing, 4608 × 3072): settings only, its 11-sample tables are not interpreted | None | +| ILCE-7RM2 | 12-bit uncompressed | All three; 256 / 0 / 1 | None | +| DSC-RX100 | One | Shading, from the `SR2SubIFD` (the raw IFD has no tables); CA and distortion in the 11-sample layout | None | +| DSC-RX100M3 | One | Shading; CA and distortion in the 11-sample layout | None | +| ILCE-7CR | Compressed, lossless L/M | All three; 272 / 1 / 0 | None | +| ILCE-7RM4 | 14-bit compressed | All three; 257 / 1 / 17 | None | +| ILCE-3500 | 5600 K | Settings only (unavailable) | None | +| ILCE-7S | Shade | An all-zero CA table; settings unavailable | None | +| ILCE-7SM2 | 16:9 framing | All-zero shading and CA tables; settings unavailable | None | +| DSLR-A500, DSLR-A700, SLT-A33, DSLR-A580, DSLR-A900 | One each | No record | None | -These are **model-boundary regression checks, not new geometric validation of those models**. In particular, +These are **model-boundary regression checks, not new geometric validation of those models**. +`sony_lens_records_of_the_public_samples` also checks the exact tables, settings, provenance and framing evidence of +the three A7CR codings and the A7R IVA sample, and that across the Sony corpus only the A7R IVA's distortion is ever +applied. In particular, refusing the A7 III tables resolves the unverified corner warp identified in review. The lossless M/S files are linear YCbCr and remain excluded independently of the model check. Header and full decode must agree for every sample. Synthetic tests also verify both TIFF byte orders, exact model matching, From 8feda7a99721478cffe752fe60e6b0728d625a05 Mon Sep 17 00:00:00 2001 From: Alex Chambers Date: Mon, 12 Oct 2026 04:42:32 +0200 Subject: [PATCH 2/2] LR-IMP-FORMATS: the TIFF reader's next-IFD pointer no longer overflows on huge BigTIFF entry counts Pre-existing on main. The reader computed the position of a directory's next-IFD pointer before it checked that every declared entry had been read: a saturating multiplication of the declared count, then an unchecked addition. A 44-byte BigTIFF declaring u64::MAX entries therefore panicked with overflow checks on. The position is now computed only for a directory read whole, with checked arithmetic. 32-bit note: the same guard compared the counts after narrowing the declared one to usize, so on wasm32 a BigTIFF declaring 4,294,967,297 entries but holding one passed for a whole directory and its next pointer was followed. The counts now compare as 64-bit numbers (reads_every_entry). On wasm32 the old comparison followed all six malformed test chains and the new one follows none; 64-bit results are unchanged for every directory whose count is read whole. Regression tests (crates/tiff/src/reader.rs): a_bigtiff_directory_declaring_u64_max_entries_parses_without_overflow, which panicked before; a_bigtiff_count_past_u32_max_never_follows_the_next_pointer (the whole chain, on every target); and only_a_directory_read_whole_has_a_next_pointer (the comparison itself, on every target). Co-Authored-By: Claude Opus 5.5 --- crates/raw/src/dngwrite.rs | 1 + crates/raw/tests/robust.rs | 1 + crates/tiff/README.md | 4 +- crates/tiff/src/reader.rs | 89 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index 379267c87..370b7c2b4 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -434,6 +434,7 @@ pub fn write_rendered_dng( linearized: false, opcodes: crate::OpcodeLists::default(), metadata: metadata.clone(), + lens_metadata: Default::default(), }; let order = ByteOrder::Little; write_dng(&raw, &DngWriteOptions { compression: DngCompression::Deflate { tile: 256, half: true }, order, xmp }) diff --git a/crates/raw/tests/robust.rs b/crates/raw/tests/robust.rs index 17e108f6f..70103bfd7 100644 --- a/crates/raw/tests/robust.rs +++ b/crates/raw/tests/robust.rs @@ -162,6 +162,7 @@ fn non_finite_black_levels_and_samples_develop() { linearized: false, opcodes: OpcodeLists::default(), metadata: Default::default(), + lens_metadata: Default::default(), }; for img in [raw.clone(), lightcraft_raw::decode(&lightcraft_raw::write_dng(&raw, &DngWriteOptions::default()).unwrap()).unwrap()] { let rgb = img.develop(Method::Ahd).unwrap(); diff --git a/crates/tiff/README.md b/crates/tiff/README.md index cfaaffffb..e6a771288 100644 --- a/crates/tiff/README.md +++ b/crates/tiff/README.md @@ -29,7 +29,9 @@ later the TIFF/DNG exporters. The reader never panics on malformed input (proptests feed random and mutated files): offsets are bounds-checked; out-of-range or unknown-type entries are skipped; IFD loops are detected; IFD count, -nesting depth and entries per IFD are limited; the total bytes of decoded values are capped at +nesting depth and entries per IFD are limited; a next-IFD pointer is followed only when every declared entry was +read, with the counts compared as 64-bit numbers (a BigTIFF declaring 4,294,967,297 entries but holding one can't +pass for a whole directory on a 32-bit target); the total bytes of decoded values are capped at `8 × input + 1 MiB` so a file cannot make us allocate far more than its own size. ## Sources diff --git a/crates/tiff/src/reader.rs b/crates/tiff/src/reader.rs index 4a80a906b..5f38dfad2 100644 --- a/crates/tiff/src/reader.rs +++ b/crates/tiff/src/reader.rs @@ -121,6 +121,13 @@ pub fn entry_span(data: &[u8], pos: usize, order: ByteOrder, bigtiff: bool, base (end <= data.len() as u64).then_some(EntrySpan { tag, field_type, count, start, len }) } +/// Whether a directory declaring `declared` entries was read whole (`read` entries), so that its next-IFD pointer +/// follows them. The counts compare as 64-bit numbers: narrowing the declared count to `usize` instead would, on a +/// 32-bit target, let a BigTIFF declaring 4,294,967,297 entries but holding one pass for a whole directory. +fn reads_every_entry(declared: u64, read: usize) -> bool { + u64::try_from(read).is_ok_and(|read| read == declared) +} + /// Parse the header: (byte order, bigtiff, first IFD offset). pub(crate) fn header(data: &[u8]) -> Result<(ByteOrder, bool, u64)> { header_with(data, true) @@ -220,9 +227,10 @@ impl<'a> Ctx<'a> { } entries.sort_by_key(|e| e.tag); entries.dedup_by_key(|e| e.tag); - // Only a fully read table has a next-IFD offset. Keep hostile BigTIFF counts out of the arithmetic (#1516). - let next = if dir.declared == dir.read as u64 { - match dir.read.checked_mul(dir.entry_size).and_then(|len| dir.table.checked_add(len)).and_then(|p| self.read_off(p)) { + // the next-IFD pointer follows the last entry, read only when every declared entry was (checked: a BigTIFF + // count can be anything up to u64::MAX) + let next = if reads_every_entry(dir.declared, dir.read) { + match dir.read.checked_mul(dir.entry_size).and_then(|n| n.checked_add(dir.table)).and_then(|p| self.read_off(p)) { Some(0) | None => 0, Some(v) => base.checked_add(v).unwrap_or(0), } @@ -307,6 +315,65 @@ pub fn decode(o: ByteOrder, typ: FieldType, b: &[u8]) -> Value { mod tests { use super::*; + /// A BigTIFF whose first IFD, at 80, declares `declared` entries but holds one, followed by a next-IFD pointer + /// `next`; a valid one-entry IFD sits at 16. + fn bigtiff_chain(order: ByteOrder, declared: u64, next: u64) -> Vec { + let ifd = |d: &mut Vec, count: u64, tag: u16, next: u64| { + order.put_u64(d, count); + order.put_u16(d, tag); + order.put_u16(d, 3); + order.put_u64(d, 1); + order.put_u16(d, 8); + d.extend([0; 6]); + order.put_u64(d, next); + }; + let mut d = if order == ByteOrder::Little { b"II".to_vec() } else { b"MM".to_vec() }; + order.put_u16(&mut d, 43); + order.put_u16(&mut d, 8); + order.put_u16(&mut d, 0); + order.put_u64(&mut d, 80); + ifd(&mut d, 1, 0x0101, 0); + d.resize(80, 0); + ifd(&mut d, declared, 0x0100, next); + d + } + + #[test] + fn only_a_directory_read_whole_has_a_next_pointer() { + // the comparison itself, on every target: a declared count past u32::MAX never matches a small read count + for (declared, read, whole) in [ + (0, 0, true), + (1, 1, true), + (8192, 8192, true), + (8193, 8192, false), + (1, 0, false), + ((1 << 32) + 1, 1, false), + ((2 << 32) + 1, 1, false), + ((1 << 63) + 1, 1, false), + (u64::MAX, 1, false), + ] { + assert_eq!(reads_every_entry(declared, read), whole, "{declared} declared, {read} read"); + } + } + + #[test] + fn a_bigtiff_count_past_u32_max_never_follows_the_next_pointer() { + // the whole chain: on 32-bit targets (wasm32) the old cast truncated these counts to 1 and followed the pointer + for order in [ByteOrder::Little, ByteOrder::Big] { + for declared in [(1u64 << 32) + 1, (2u64 << 32) + 1, (1u64 << 63) + 1] { + let d = bigtiff_chain(order, declared, 16); + let tiff = Tiff::parse(&d).unwrap(); + assert_eq!(tiff.ifds.len(), 1, "{order:?} {declared}"); + assert_eq!(tiff.ifds[0].u64(0x0100), Some(8)); + assert_eq!(parse_ifd_at(&d, 80, order, 0, true, &ParseOptions::default()).unwrap().1, 0, "{order:?} {declared}"); + } + // control: the same file declaring its one entry follows the pointer to the IFD at 16 + let whole = Tiff::parse(&bigtiff_chain(order, 1, 16)).unwrap(); + assert_eq!(whole.ifds.len(), 2, "{order:?}"); + assert_eq!(whole.ifds[1].u64(0x0101), Some(8)); + } + } + #[test] fn directory_positions_never_overflow() { // a hand-built layout the reader never produces: the iterator stops instead of overflowing @@ -322,4 +389,20 @@ mod tests { let dir = directory_entries(&data, 0, ByteOrder::Little, false, 8192).unwrap(); assert_eq!(dir.positions().collect::>(), vec![2, 14]); } + + #[test] + fn a_bigtiff_directory_declaring_u64_max_entries_parses_without_overflow() { + // 44 bytes: BigTIFF header (first IFD at 16), an entry count of u64::MAX, one 20-byte entry + let mut data = b"II\x2b\x00\x08\x00\x00\x00".to_vec(); + data.extend(16u64.to_le_bytes()); + data.extend(u64::MAX.to_le_bytes()); + data.extend(0x0100u16.to_le_bytes()); + data.extend(3u16.to_le_bytes()); + data.extend(1u64.to_le_bytes()); + data.extend([8, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(data.len(), 44); + let tiff = Tiff::parse(&data).unwrap(); + assert_eq!(tiff.ifds.len(), 1); + assert_eq!(tiff.ifds[0].u64(0x0100), Some(8)); + } }