mirror of
https://github.com/storytold/lightcraft.git
synced 2026-10-19 02:10:25 -05:00
Merge PR #1176
# Conflicts: # crates/raw/src/vendor/arw.rs # docs/parity-checklist.md # docs/sony-lens-corrections.md
This commit is contained in:
commit
f613aeba59
40 files changed
+2644
-161
No files matched your search
Generated
+1
@@ -3419,6 +3419,7 @@ dependencies = [
|
||||
name = "lightcraft-raw"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"dhat",
|
||||
"jpeg-encoder",
|
||||
"jxl-oxide",
|
||||
"lightcraft-color",
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ bytemuck = { version = "1", features = ["derive"] }
|
||||
crc32fast = "1"
|
||||
siphasher = "1"
|
||||
libc = "0.2" # dev-only: process CPU time in benches
|
||||
dhat = "0.3" # opt-in heap profiling (lightcraft-cli --features dhat-heap); never in default builds
|
||||
dhat = "0.3" # opt-in heap profiling (lightcraft-cli --features dhat-heap) and heap tests; never in product builds
|
||||
|
||||
# formats
|
||||
quick-xml = "0.38"
|
||||
|
||||
@@ -1705,6 +1705,7 @@ mod tests {
|
||||
linearized: true,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: lightcraft_meta::Metadata::default(),
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
let proxy = sensor_proxy(&raw, 2, 384).unwrap();
|
||||
assert_eq!((proxy.width, proxy.height), (32, 32));
|
||||
@@ -2048,6 +2049,7 @@ mod tests {
|
||||
linearized: false,
|
||||
opcodes: Default::default(),
|
||||
metadata: lightcraft_raw::Metadata { make: Some(make.to_owned()), model: Some(model.to_owned()), ..Default::default() },
|
||||
lens_metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3231,6 +3233,7 @@ mod tests {
|
||||
linearized: true,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: lightcraft_meta::Metadata { make: Some("SONY".into()), model: Some("ILCE-7M4".into()), ..Default::default() },
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
assert!(crate::camera_profiles::get("ILCE-7M4").is_some());
|
||||
// the camera's rendering: a bright curve on the scene's luminance, as a grey (or tinted) 8-bit sRGB JPEG
|
||||
|
||||
@@ -586,6 +586,7 @@ pub(crate) fn synthetic_dng_of(w: usize, h: usize, cfa: &str, data: Vec<u16>, xm
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
write_dng(&raw, &DngWriteOptions { xmp: xmp.map(str::to_string), ..Default::default() }).unwrap()
|
||||
}
|
||||
|
||||
@@ -113,6 +113,7 @@ pub fn write_linear_dng(
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists { list3: lens.to_vec(), ..Default::default() },
|
||||
metadata: meta,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
let compression = DngCompression::Deflate { tile: 256, half: samples == DngSamples::Half };
|
||||
lightcraft_raw::write_dng(&raw, &DngWriteOptions { compression, ..Default::default() }).map_err(|e| MergeError::Output(e.to_string()))
|
||||
|
||||
@@ -33,3 +33,5 @@ proptest = { workspace = true }
|
||||
zune-jpegxl = { workspace = true }
|
||||
# Pinned CC0 corpus identities; not a product dependency.
|
||||
sha2 = { version = "0.10", default-features = false, features = ["std"] }
|
||||
# Peak-heap regression tests (tests/sr2_heap.rs); not a product dependency.
|
||||
dhat = { workspace = true }
|
||||
@@ -227,6 +227,7 @@ mod tests {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
let path = dir.path("valid.dng");
|
||||
std::fs::write(&path, write_dng(&raw, &DngWriteOptions::default()).expect("encode generated fixture")).expect("write fixture");
|
||||
|
||||
@@ -165,6 +165,7 @@ mod tests {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Metadata::default(),
|
||||
lens_metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -289,6 +289,7 @@ mod tests {
|
||||
linearized: false,
|
||||
opcodes: Default::default(),
|
||||
metadata: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -340,6 +340,7 @@ pub(crate) fn decode_as(bytes: &[u8], mode: Mode, format: RawFormat) -> Result<R
|
||||
linearized,
|
||||
opcodes,
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
@@ -540,6 +540,7 @@ pub fn write_rendered_dng(
|
||||
linearized: false,
|
||||
opcodes: crate::OpcodeLists::default(),
|
||||
metadata: metadata.clone(),
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
let order = ByteOrder::Little;
|
||||
write_dng(&raw, &DngWriteOptions { compression: DngCompression::Deflate { tile: 256, half: true }, order, xmp })
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
//! Lens-correction data a raw file records, kept with where it came from and separately from what LightCraft
|
||||
//! applies.
|
||||
//!
|
||||
//! A readable table is not a correction. Each component carries a [`SonyLensStatus`]: only
|
||||
//! [`SonyLensStatus::Applied`] components were converted into the image's `OpcodeList3`, and that is decided by
|
||||
//! independent validation per model, coding and component (see `docs/sony-lens-corrections.md`), never by the mere
|
||||
//! presence of a table. Everything else here is metadata for diagnostics and later validation work.
|
||||
|
||||
use crate::Rect;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Vendor lens-correction metadata of a raw file (see the module docs). Empty for formats that record none, or
|
||||
/// whose tables are read straight into opcodes.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct LensMetadata {
|
||||
/// Sony ARW: the raw image IFD's shading, lateral chromatic aberration and distortion records, with their
|
||||
/// encrypted `SR2SubIFD` copies. `None` when neither directory holds any of them.
|
||||
#[serde(default)]
|
||||
pub sony: Option<SonyLensCorrections>,
|
||||
}
|
||||
|
||||
/// Sony's in-file lens-correction record: three independent components, each a camera setting and a radial table.
|
||||
///
|
||||
/// Tag identities and storage types from the ExifTool EXIF / Sony tag-name documentation: raw-IFD `0x7031` / `0x7032`
|
||||
/// (shading, "VignettingCorrection" / "VignettingCorrParams"), `0x7034` / `0x7035` (lateral chromatic aberration),
|
||||
/// `0x7036` / `0x7037` (distortion), and the `SR2SubIFD` tables `0x797d`, `0x7980`, `0x7982`. The `SR2SubIFD` setting
|
||||
/// tags `0x797c`, `0x797f`, `0x7981` are unnamed there; they sit next to the tables and equal the raw-IFD settings
|
||||
/// in every file inspected.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SonyLensCorrections {
|
||||
/// Shading (vignetting): 16 radial samples whose gain function, radius convention and processing stage are not
|
||||
/// established. Opaque signed data.
|
||||
pub shading: SonyComponent<[i16; 16]>,
|
||||
/// Lateral chromatic aberration: two 16-sample radial curves. A public analysis reads them as red, then blue,
|
||||
/// relative to green; that is not validated here.
|
||||
pub chromatic_aberration: SonyComponent<[[i16; 16]; 2]>,
|
||||
/// Distortion: 16 radial samples (see `docs/sony-lens-corrections.md` for the validated interpretation).
|
||||
pub distortion: SonyComponent<[i16; 16]>,
|
||||
/// Whether the encrypted `SR2SubIFD` was found and decrypted. `false`: no copies to fall back on or compare.
|
||||
pub sr2_read: bool,
|
||||
/// The framing and coding the tables have to be interpreted against.
|
||||
pub geometry: SonyLensGeometry,
|
||||
}
|
||||
|
||||
/// One correction component: the camera's setting, its table, and whether LightCraft applies it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SonyComponent<T> {
|
||||
pub setting: SonyField<SonySetting>,
|
||||
/// The table's samples after the leading sample count, which must equal their number.
|
||||
pub table: SonyField<T>,
|
||||
pub status: SonyLensStatus,
|
||||
}
|
||||
|
||||
/// One field read from the raw IFD and from the `SR2SubIFD`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SonyField<T> {
|
||||
/// The value in use: the raw IFD's when it holds a well-formed one, else the `SR2SubIFD`'s when the raw IFD
|
||||
/// lacks the tag. `None` when no copy is usable, and when the raw IFD's copy is malformed: a corrupt field is
|
||||
/// reported, not silently replaced by the other copy. The raw IFD's copy is the one the TIFF reader that decodes
|
||||
/// the image parsed (its first occurrence, within the reader's limits).
|
||||
pub value: Option<T>,
|
||||
pub source: Option<SonySource>,
|
||||
pub raw_ifd: SonyCopy,
|
||||
pub sr2: SonyCopy,
|
||||
/// Whether the two copies are equal, when both are well-formed. The raw IFD's wins when they differ.
|
||||
pub copies_agree: Option<bool>,
|
||||
/// Diagnostics: the tag occurs more than once in the raw IFD / the `SR2SubIFD`. Only the first occurrence is used,
|
||||
/// as the TIFF reader does (the raw IFD's: the first one whose values it could read).
|
||||
#[serde(default)]
|
||||
pub raw_ifd_repeated: bool,
|
||||
#[serde(default)]
|
||||
pub sr2_repeated: bool,
|
||||
}
|
||||
|
||||
/// Where a value was read.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum SonySource {
|
||||
/// The plain raw image IFD.
|
||||
RawIfd,
|
||||
/// The encrypted `SR2SubIFD`.
|
||||
Sr2SubIfd,
|
||||
}
|
||||
|
||||
/// How one copy of a field was found.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum SonyCopy {
|
||||
/// Not in that directory (or the `SR2SubIFD` could not be read).
|
||||
Absent,
|
||||
/// Present with exactly the supported type and layout.
|
||||
Valid,
|
||||
/// Present, but refused.
|
||||
Rejected(SonyFieldError),
|
||||
}
|
||||
|
||||
/// Why a present field was refused.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum SonyFieldError {
|
||||
/// Another TIFF field type than SSHORT (8) for a table, or SHORT (3) / SSHORT for a setting.
|
||||
FieldType(u16),
|
||||
/// Another number of values than the supported layout's: 17 (shading, distortion), 33 (CA) or 1 (setting).
|
||||
Count(u64),
|
||||
/// The leading sample count is not the layout's 16 (32 for CA). Older bodies store 11 (22) samples padded to the
|
||||
/// same length, a layout that is not interpreted; a count of 0 goes with an "unavailable" setting.
|
||||
CountPrefix(i16),
|
||||
/// The values lie outside the file (raw IFD) or the decrypted block (`SR2SubIFD`).
|
||||
OutsideData,
|
||||
/// In the raw IFD but past what the TIFF reader that decodes the image reads (8,192 entries per directory, a
|
||||
/// total value budget), so the image doesn't use it either.
|
||||
PastReaderLimits,
|
||||
/// A setting stored as a negative SSHORT.
|
||||
NegativeSetting(i16),
|
||||
}
|
||||
|
||||
/// A camera correction setting: the stored number and its documented meaning.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct SonySetting {
|
||||
pub value: u16,
|
||||
pub meaning: SonySettingMeaning,
|
||||
}
|
||||
|
||||
/// The documented meaning of a setting value (ExifTool EXIF tag-name documentation). Values are compared whole;
|
||||
/// they are not read as bit masks.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum SonySettingMeaning {
|
||||
/// Shading 256, CA 0, distortion 0. The table is still recorded: Off does not mean absent.
|
||||
Off,
|
||||
/// Shading 257 or 272, CA 1, distortion 1.
|
||||
Auto,
|
||||
/// Distortion 17: Auto, fixed by the lens.
|
||||
ForcedAuto,
|
||||
/// "No correction params available": shading 511, CA 255, distortion 255.
|
||||
Unavailable,
|
||||
/// Any other value (e.g. a shading "Low" setting, whose number is not documented). Never treated as Off.
|
||||
Unknown(u16),
|
||||
}
|
||||
|
||||
/// Whether a component is applied, and if not, why.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum SonyLensStatus {
|
||||
/// Converted into the image's `OpcodeList3` (today: ILCE-7RM4A and ILCE-7RM6 Bayer distortion in native 3:2 framing).
|
||||
Applied,
|
||||
/// No usable table.
|
||||
NoTable,
|
||||
/// Decoded only: no independent validation covers this component's rendering semantics on any body (shading,
|
||||
/// lateral CA).
|
||||
NotValidatedComponent,
|
||||
/// Decoded only: the component is validated on other camera models, not on this one.
|
||||
NotValidatedModel,
|
||||
/// Decoded only: the model is validated, but not this coding or framing (linear YCbCr, a non-3:2 crop) or a
|
||||
/// table found only in the `SR2SubIFD`.
|
||||
NotValidatedMode,
|
||||
/// Validated scope, but the table is all zeros: a valid no-op, nothing to apply.
|
||||
Neutral,
|
||||
/// Validated scope, but the conversion refused this table (values outside the validated range, a degenerate
|
||||
/// crop, or a fitted map that misses the table or folds).
|
||||
Rejected,
|
||||
}
|
||||
|
||||
/// The framing and coding evidence for interpreting the tables, all in raw-frame pixels.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SonyLensGeometry {
|
||||
/// The stored raw frame, including any padding (lossless L files are padded).
|
||||
pub storage_width: usize,
|
||||
pub storage_height: usize,
|
||||
/// Sony's crop tags (`0x74c7` / `0x74c8`) as recorded.
|
||||
pub sony_crop: Option<Rect>,
|
||||
/// The raw IFD's `DefaultCropOrigin` / `DefaultCropSize` as recorded.
|
||||
pub default_crop: Option<Rect>,
|
||||
/// The image's crop (the decoder's precedence: Sony's tags, then the default crop, then the recorded size).
|
||||
pub image_crop: Rect,
|
||||
/// The rectangle the validated distortion conversion is centred on and normalised to: the default crop when
|
||||
/// the file has one, else the image crop. Clipped to the frame.
|
||||
pub reference: Rect,
|
||||
/// TIFF compression of the raw image (1 uncompressed, 7 lossless, 32767 ARW2, 32766 ARW6).
|
||||
pub compression: u16,
|
||||
/// Downsized lossless M/S data (linear YCbCr, not sensor CFA).
|
||||
pub linear_ycbcr: bool,
|
||||
/// Both the image crop and the reference are 3:2 within 0.5 %, the only framing validated.
|
||||
pub native_3_2: bool,
|
||||
}
|
||||
|
||||
impl<T> SonyField<T> {
|
||||
/// Whether either directory holds the tag (well-formed or not).
|
||||
pub fn is_present(&self) -> bool {
|
||||
self.raw_ifd != SonyCopy::Absent || self.sr2 != SonyCopy::Absent
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,7 @@ pub mod highlight;
|
||||
#[cfg(feature = "jxl")]
|
||||
mod jxl;
|
||||
pub mod kelvin_presets;
|
||||
pub mod lens_metadata;
|
||||
pub mod ljpeg;
|
||||
pub(crate) mod llvc;
|
||||
pub mod opcodes;
|
||||
@@ -51,6 +52,7 @@ mod wb_estimate;
|
||||
|
||||
pub use demosaic::{Method, demosaic};
|
||||
pub use dngwrite::{DngCompression, DngWriteOptions, write_dng, write_rendered_dng};
|
||||
pub use lens_metadata::LensMetadata;
|
||||
pub use lightcraft_color::Mat3;
|
||||
pub use lightcraft_geom::Orientation;
|
||||
pub use lightcraft_meta::Metadata;
|
||||
@@ -430,6 +432,9 @@ pub struct RawInfo {
|
||||
pub wb_multipliers: Option<[f32; 3]>,
|
||||
pub opcodes: OpcodeLists,
|
||||
pub metadata: Metadata,
|
||||
/// Lens-correction tables the file records, decoded but not necessarily applied (see [`crate::lens_metadata`]).
|
||||
#[serde(default)]
|
||||
pub lens_metadata: LensMetadata,
|
||||
}
|
||||
|
||||
impl RawInfo {
|
||||
@@ -663,6 +668,10 @@ pub struct RawImage {
|
||||
pub linearized: bool,
|
||||
pub opcodes: OpcodeLists,
|
||||
pub metadata: Metadata,
|
||||
/// Lens-correction tables the file records, decoded but not necessarily applied (see [`crate::lens_metadata`]); only
|
||||
/// validated components are in `opcodes`.
|
||||
#[serde(default)]
|
||||
pub lens_metadata: LensMetadata,
|
||||
}
|
||||
|
||||
/// Black-subtracted, white-normalised data covering the active area.
|
||||
@@ -694,6 +703,7 @@ impl RawImage {
|
||||
wb_multipliers: self.wb_multipliers,
|
||||
opcodes: self.opcodes.clone(),
|
||||
metadata: self.metadata.clone(),
|
||||
lens_metadata: self.lens_metadata.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -713,6 +723,7 @@ impl RawImage {
|
||||
wb_multipliers: self.wb_multipliers,
|
||||
opcodes: self.opcodes,
|
||||
metadata: self.metadata,
|
||||
lens_metadata: self.lens_metadata,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -880,6 +891,7 @@ mod tests {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Metadata::default(),
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
let direct = raw.develop(Method::Bilinear).unwrap();
|
||||
let by_hand = raw.finish_demosaiced(demosaic(&raw.normalized().unwrap(), Method::Bilinear));
|
||||
@@ -920,6 +932,7 @@ mod tests {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Metadata::default(),
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
assert!(raw.validate().is_err(), "`x + width` wrapped: the area is outside the image");
|
||||
assert!(raw.normalized().is_err());
|
||||
|
||||
@@ -137,6 +137,7 @@ mod tests {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Metadata::default(),
|
||||
lens_metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Vendored
+46
-22
@@ -45,8 +45,9 @@
|
||||
//! through the generic TIFF path.
|
||||
|
||||
use super::arw6;
|
||||
use crate::lens_metadata::SonyLensGeometry;
|
||||
use crate::tiffraw::{Packing, check_image, read_image_in};
|
||||
use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg};
|
||||
use crate::{BlackLevel, Cfa, ColorData, LensMetadata, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg};
|
||||
use lightcraft_geom::Orientation;
|
||||
use lightcraft_tiff::image::{Chunk, ImageInfo, Layout, chunk_bytes};
|
||||
use lightcraft_tiff::tags::{self as t, photometric};
|
||||
@@ -553,6 +554,14 @@ fn dng_default_crop(raw: &Ifd, w: usize, h: usize) -> Option<Rect> {
|
||||
None
|
||||
}
|
||||
|
||||
/// A rectangle exactly as an origin tag and a size tag record it (two values each), when both exist and fit `usize`.
|
||||
fn recorded_rect(raw: &Ifd, origin: u16, size: u16) -> Option<Rect> {
|
||||
let (Some(origin), Some(size)) = (raw.u64s(origin), raw.u64s(size)) else { return None };
|
||||
let (&[x, y], &[w, h]) = (origin.as_slice(), size.as_slice()) else { return None };
|
||||
let n = |v: u64| usize::try_from(v).ok();
|
||||
Some(Rect::new(n(x)?, n(y)?, n(w)?, n(h)?))
|
||||
}
|
||||
|
||||
/// The image area for files without Sony's crop tags (`0x74c7/0x74c8`, written since about 2017): the DNG-style
|
||||
/// default crop when the raw IFD has one, else the image size the camera records (see [`recorded_size_crop`]).
|
||||
///
|
||||
@@ -749,6 +758,39 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
let mut metadata = lightcraft_meta::from_tiff(&tiff);
|
||||
metadata.width = Some(crop.width as u32);
|
||||
metadata.height = Some(crop.height as u32);
|
||||
// The lens-correction record. The distortion table is centred on the DNG-style default crop when the file has
|
||||
// one: on the ILCE-7RM4A it starts at x = 32 where Sony's crop tags (the image crop, unchanged) start at 0, and
|
||||
// the warp was validated against Sony's exports with that centre. The opcode's centre is in active-area
|
||||
// coordinates, so it stays on the optical centre whichever crop frames the image.
|
||||
let reference = dng_default_crop(raw, w, h).unwrap_or(crop);
|
||||
let geometry = SonyLensGeometry {
|
||||
storage_width: w,
|
||||
storage_height: h,
|
||||
sony_crop: recorded_rect(raw, CROP_TOP_LEFT, CROP_SIZE),
|
||||
default_crop: recorded_rect(raw, t::DEFAULT_CROP_ORIGIN, t::DEFAULT_CROP_SIZE),
|
||||
image_crop: crop,
|
||||
reference,
|
||||
compression: info.compression,
|
||||
linear_ycbcr: linear_rgb,
|
||||
native_3_2: super::arw_lens::is_3_2(crop) && super::arw_lens::is_3_2(reference),
|
||||
};
|
||||
// decrypted for the copies' agreement check too, not only when a plain field is missing: a few dozen kB, found by
|
||||
// reading only three `SR2Private` entries (see `sr2::SubIfd::read`)
|
||||
let (mut sony_lens, warp) = super::arw_lens::corrections(bytes, tiff.order, (raw, tiff.bigtiff), sr2(), &model, geometry);
|
||||
// The ILCE-7RM6's tables follow other conventions, measured against its camera JPEGs ([`super::arw_lens_rm6`]);
|
||||
// its allowlist and `arw_lens`'s are disjoint, so at most one of them produces an opcode. Both the image framing
|
||||
// and the table geometry must stay within the validated 3:2 scope.
|
||||
let rm6: Vec<crate::Opcode> = if linear_rgb || !super::arw_lens::is_3_2(crop) {
|
||||
Vec::new()
|
||||
} else {
|
||||
super::arw_lens_rm6::lens_opcodes(raw, &model, Rect::new(0, 0, w, h), reference)
|
||||
};
|
||||
if !rm6.is_empty()
|
||||
&& let Some(record) = sony_lens.as_mut()
|
||||
{
|
||||
// distortion only: its lateral CA planes are off (see `arw_lens_rm6::lens_opcodes`)
|
||||
record.distortion.status = crate::lens_metadata::SonyLensStatus::Applied;
|
||||
}
|
||||
let img = RawImage {
|
||||
format: RawFormat::Arw,
|
||||
width: w,
|
||||
@@ -768,28 +810,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
// makes a neutral surface magenta. WB edits remain relative to this as-shot RGB.
|
||||
wb_multipliers: if linear_rgb { Some([1.0; 3]) } else { wb },
|
||||
linearized: false,
|
||||
// The distortion table is centred on the DNG-style default crop when the file has one: on the ILCE-7RM4A
|
||||
// it starts at x = 32 where Sony's crop tags (the image crop, unchanged) start at 0, and the warp was
|
||||
// validated against Sony's exports with that centre. The opcode's centre is in active-area coordinates, so
|
||||
// it stays on the optical centre whichever crop frames the image. The ILCE-7RM6's tables follow other
|
||||
// conventions, measured against its camera JPEGs ([`super::arw_lens_rm6`]); the two allowlists are
|
||||
// disjoint, so at most one of them produces an opcode.
|
||||
opcodes: OpcodeLists {
|
||||
// DefaultCrop can retain native geometry while Sony's tags select an aspect crop.
|
||||
// Both the image framing and the table geometry must stay within the validated 3:2 scope.
|
||||
list3: if linear_rgb || crop.height == 0 || (crop.width as f64 / crop.height as f64 - 1.5).abs() > 0.005 {
|
||||
Vec::new()
|
||||
} else {
|
||||
let geometry = dng_default_crop(raw, w, h).unwrap_or(crop);
|
||||
let active = Rect::new(0, 0, w, h);
|
||||
super::arw_lens::distortion(&model, raw, active, geometry)
|
||||
.into_iter()
|
||||
.chain(super::arw_lens_rm6::lens_opcodes(raw, &model, active, geometry))
|
||||
.collect()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
// only validated components become opcodes (see `arw_lens`, and `arw_lens_rm6` for the ILCE-7RM6)
|
||||
opcodes: OpcodeLists { list3: warp.into_iter().chain(rm6).collect(), ..Default::default() },
|
||||
metadata,
|
||||
lens_metadata: LensMetadata { sony: sony_lens },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1055
-46
File diff suppressed because it is too large.
Load diff
Vendored
+1
@@ -251,6 +251,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -393,6 +393,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode) -> Result
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -255,6 +255,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
image.validate_for(mode)?;
|
||||
Ok(image)
|
||||
|
||||
Vendored
+1
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -117,6 +117,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -121,6 +121,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -327,6 +327,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
..Default::default()
|
||||
},
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -336,6 +336,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -337,6 +337,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -214,6 +214,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists { list3: super::raf_lens::corrections(&camera_name(bytes), &ifd, active), ..Default::default() },
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+1
@@ -616,6 +616,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists { list3: distortion(ifd0, active).into_iter().collect(), ..Default::default() },
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
Vendored
+264
-6
@@ -45,7 +45,7 @@
|
||||
//! of an accident 2⁻⁴⁰⁶⁴). For the DSC-R1's key `63 be 01 3e` the 11932-byte block becomes a well-formed 101-entry
|
||||
//! directory (black `0x7300`, white balance `0x7303`, crop `0x74c3`).
|
||||
|
||||
use lightcraft_tiff::{ByteOrder, Ifd, ParseOptions, tags as t};
|
||||
use lightcraft_tiff::{ByteOrder, EntrySpan, Ifd, ParseOptions, decode_value, directory_entries, entry_span, tags as t};
|
||||
|
||||
/// `SR2Private` tags.
|
||||
const SUBIFD_OFFSET: u16 = 0x7200;
|
||||
@@ -136,11 +136,8 @@ impl SubIfd {
|
||||
/// [`MAX_LEN`].
|
||||
pub(crate) fn read(bytes: &[u8], ifd0: &Ifd, order: ByteOrder) -> Option<Self> {
|
||||
let at = order.read_u32(ifd0.bytes(t::DNG_PRIVATE_DATA)?, 0)?;
|
||||
let opts = ParseOptions { max_ifds: 1, max_depth: 0, follow_children: false, ..Default::default() };
|
||||
let (private, _) = lightcraft_tiff::parse_ifd_at(bytes, u64::from(at), order, 0, false, &opts).ok()?;
|
||||
let key: [u8; 4] = private.bytes(SUBIFD_KEY)?.try_into().ok()?;
|
||||
let start = usize::try_from(private.u64(SUBIFD_OFFSET)?).ok()?;
|
||||
let len = usize::try_from(private.u64(SUBIFD_LENGTH)?).ok()?;
|
||||
let fields = private_fields(bytes, u64::from(at), order);
|
||||
let (start, len, key) = (fields.offset?, fields.length?, fields.key?);
|
||||
if len > MAX_LEN {
|
||||
return None;
|
||||
}
|
||||
@@ -165,6 +162,13 @@ impl SubIfd {
|
||||
self.order
|
||||
}
|
||||
|
||||
/// The decrypted block (the directory from its entry count on) and the file offset it was stored at, which its
|
||||
/// value offsets count from: for readers that check each entry's exact type and count themselves
|
||||
/// (`arw_lens`).
|
||||
pub(crate) fn block(&self) -> (&[u8], usize) {
|
||||
(&self.plain, self.start)
|
||||
}
|
||||
|
||||
/// The field type and value bytes of a SHORT or SSHORT entry; `None` when the tag is absent, has another type
|
||||
/// or its values lie outside the block.
|
||||
pub(crate) fn short_bytes(&self, tag: u16) -> Option<(u16, &[u8])> {
|
||||
@@ -193,9 +197,69 @@ impl SubIfd {
|
||||
}
|
||||
}
|
||||
|
||||
/// The `SR2Private` fields that locate the block.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
struct PrivateFields {
|
||||
offset: Option<usize>,
|
||||
length: Option<usize>,
|
||||
key: Option<[u8; 4]>,
|
||||
}
|
||||
|
||||
/// The block's file offset, length and key from the `SR2Private` directory at `at`, exactly as the TIFF reader would
|
||||
/// give them (`parse_ifd_at` with its default limits, then its `u64` and `bytes` lookups), without decoding the
|
||||
/// directory. The selection uses the reader's own rules: its directory layout and entry limit
|
||||
/// ([`directory_entries`]), which entries it can read ([`entry_span`]) and its value budget; each field is its first
|
||||
/// readable occurrence. Of that entry only the first value is decoded (the key's first five bytes, which settle
|
||||
/// whether it is four bytes), so a hostile directory can't make this allocate memory sized by its other entries,
|
||||
/// which could each alias the whole file, before the block's own [`MAX_LEN`] is checked.
|
||||
fn private_fields(bytes: &[u8], at: u64, order: ByteOrder) -> PrivateFields {
|
||||
let opts = ParseOptions::default();
|
||||
let Ok(dir) = directory_entries(bytes, at, order, false, opts.max_entries) else { return PrivateFields::default() };
|
||||
let mut budget = opts.budget_for(bytes.len());
|
||||
let (mut offset, mut length, mut key) = (None, None, None);
|
||||
for p in dir.positions() {
|
||||
let Some(span) = entry_span(bytes, p, order, false, 0) else { continue };
|
||||
if span.len > budget {
|
||||
continue;
|
||||
}
|
||||
budget -= span.len;
|
||||
match span.tag {
|
||||
SUBIFD_OFFSET if offset.is_none() => offset = Some(first_number(bytes, &span, order)),
|
||||
SUBIFD_LENGTH if length.is_none() => length = Some(first_number(bytes, &span, order)),
|
||||
SUBIFD_KEY if key.is_none() => key = Some(four_bytes(bytes, &span, order)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
PrivateFields { offset: offset.flatten(), length: length.flatten(), key: key.flatten() }
|
||||
}
|
||||
|
||||
/// The entry's first value as the reader's `u64` lookup reads it (an unsigned or non-negative integer), as a `usize`.
|
||||
fn first_number(bytes: &[u8], span: &EntrySpan, order: ByteOrder) -> Option<usize> {
|
||||
if span.count == 0 {
|
||||
return None;
|
||||
}
|
||||
let start = usize::try_from(span.start).ok()?;
|
||||
let first = bytes.get(start..start.checked_add(span.field_type.size())?)?;
|
||||
usize::try_from(decode_value(order, span.field_type, first).get_u64(0)?).ok()
|
||||
}
|
||||
|
||||
/// The entry's bytes as the reader's `bytes` lookup gives them, when they are exactly four. Decoding the first five
|
||||
/// value bytes settles it: BYTE and UNDEFINED give all their bytes, ASCII its text before the first NUL, which
|
||||
/// decoding never makes shorter.
|
||||
fn four_bytes(bytes: &[u8], span: &EntrySpan, order: ByteOrder) -> Option<[u8; 4]> {
|
||||
let start = usize::try_from(span.start).ok()?;
|
||||
let head = bytes.get(start..start.checked_add(usize::try_from(span.len).ok()?.min(5))?)?;
|
||||
<[u8; 4]>::try_from(decode_value(order, span.field_type, head).as_bytes()?).ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
/// TIFF field types.
|
||||
const BYTE: u16 = 1;
|
||||
const LONG: u16 = 4;
|
||||
const UNDEFINED: u16 = 7;
|
||||
const IFD: u16 = 13;
|
||||
use lightcraft_tiff::ByteOrder::{Big, Little};
|
||||
|
||||
/// An `SR2SubIFD` block for file offset `start`, encrypted: SHORT/SSHORT entries `(tag, type, values)`, values
|
||||
@@ -345,6 +409,200 @@ pub(crate) mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// An `SR2Private` directory of `entries` `(tag, type, count, value field)` at file offset 0.
|
||||
fn private(entries: &[(u16, u16, u32, [u8; 4])], order: ByteOrder) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
order.put_u16(&mut out, entries.len() as u16);
|
||||
for &(tag, kind, count, value) in entries {
|
||||
order.put_u16(&mut out, tag);
|
||||
order.put_u16(&mut out, kind);
|
||||
order.put_u32(&mut out, count);
|
||||
out.extend(value);
|
||||
}
|
||||
order.put_u32(&mut out, 0);
|
||||
out
|
||||
}
|
||||
|
||||
/// `(offset, length, key)` as one of the readers selects them.
|
||||
type Selected = (Option<usize>, Option<usize>, Option<[u8; 4]>);
|
||||
|
||||
/// What `SubIfd::read` on main selects: the TIFF reader's parse of the directory with main's options, then the
|
||||
/// directory's `u64` and `bytes` lookups.
|
||||
fn reader_selects(bytes: &[u8], at: usize, order: ByteOrder) -> Selected {
|
||||
let opts = lightcraft_tiff::ParseOptions { max_ifds: 1, max_depth: 0, follow_children: false, ..Default::default() };
|
||||
let Ok((private, _)) = lightcraft_tiff::parse_ifd_at(bytes, at as u64, order, 0, false, &opts) else { return (None, None, None) };
|
||||
let n = |tag: u16| private.u64(tag).and_then(|v| usize::try_from(v).ok());
|
||||
(n(SUBIFD_OFFSET), n(SUBIFD_LENGTH), private.bytes(SUBIFD_KEY).and_then(|b| <[u8; 4]>::try_from(b).ok()))
|
||||
}
|
||||
|
||||
/// Synthetic `SR2Private` directories at offset 0, with what each tests: filler entries ahead of the fields, and
|
||||
/// unreadable, unusual or valid first occurrences of each field ahead of a valid second one.
|
||||
fn differential_cases(order: ByteOrder) -> Vec<(String, Vec<u8>)> {
|
||||
let w = |v: u32| {
|
||||
let mut b = Vec::new();
|
||||
order.put_u32(&mut b, v);
|
||||
[b[0], b[1], b[2], b[3]]
|
||||
};
|
||||
let h = |a: u16, b: u16| {
|
||||
let mut out = Vec::new();
|
||||
order.put_u16(&mut out, a);
|
||||
order.put_u16(&mut out, b);
|
||||
[out[0], out[1], out[2], out[3]]
|
||||
};
|
||||
let valid = [(SUBIFD_OFFSET, LONG, 1, w(4096)), (SUBIFD_LENGTH, LONG, 1, w(64)), (SUBIFD_KEY, UNDEFINED, 4, KEY)];
|
||||
let mut cases = Vec::new();
|
||||
for fillers in [0usize, 1, 255, 256, 257, 8189, 8190, 8191, 8192, 8193] {
|
||||
let mut entries = vec![(0x7300, LONG, 1, w(7)); fillers];
|
||||
entries.extend(valid);
|
||||
cases.push((format!("{fillers} fillers"), private(&entries, order)));
|
||||
}
|
||||
// a first occurrence ahead of the valid one; pointer values point into a 32-byte tail after the directory
|
||||
let tail_at = 2 + 12 * 4 + 4;
|
||||
let mut tail: Vec<u8> = (1..=32).collect();
|
||||
tail[4..8].copy_from_slice(b"\x11\x22\x33\x44");
|
||||
tail[8] = 0;
|
||||
let numeric: Vec<(&str, u16, u32, [u8; 4])> = vec![
|
||||
("unknown type 0", 0, 1, w(8)),
|
||||
("unknown type 14", 14, 1, w(8)),
|
||||
("unknown type 0xffff", 0xffff, 1, w(8)),
|
||||
("LONG[2] past the file", LONG, 2, w(u32::MAX)),
|
||||
("LONG[2] straddling the end", LONG, 2, w(tail_at as u32 + 28)),
|
||||
("LONG[u32::MAX]", LONG, u32::MAX, w(tail_at as u32)),
|
||||
("DOUBLE[u32::MAX]", 12, u32::MAX, w(tail_at as u32)),
|
||||
("LONG[0]", LONG, 0, w(0)),
|
||||
("LONG[2] in the file", LONG, 2, w(tail_at as u32)),
|
||||
("SHORT[1]", SHORT, 1, h(9, 0)),
|
||||
("SHORT[2] inline", SHORT, 2, h(9, 10)),
|
||||
("BYTE[3]", BYTE, 3, [5, 6, 7, 0]),
|
||||
("SSHORT -1", SSHORT, 1, h(0xffff, 0)),
|
||||
("SSHORT 5", SSHORT, 1, h(5, 0)),
|
||||
("SLONG -1", 9, 1, w(u32::MAX)),
|
||||
("LONG8 at a pointer", 16, 1, w(tail_at as u32)),
|
||||
("RATIONAL", 5, 1, w(tail_at as u32)),
|
||||
("ASCII", 2, 4, *b"abc\0"),
|
||||
("IFD", IFD, 1, w(4096)),
|
||||
];
|
||||
for (tag, other) in [(SUBIFD_OFFSET, [valid[1], valid[2]]), (SUBIFD_LENGTH, [valid[0], valid[2]])] {
|
||||
for (what, kind, count, field) in &numeric {
|
||||
let valid_one = if tag == SUBIFD_OFFSET { valid[0] } else { valid[1] };
|
||||
let mut d = private(&[(tag, *kind, *count, *field), valid_one, other[0], other[1]], order);
|
||||
d.extend(&tail);
|
||||
cases.push((format!("{tag:#x}: first {what}"), d));
|
||||
}
|
||||
}
|
||||
let keys: Vec<(&str, u16, u32, [u8; 4])> = vec![
|
||||
("unknown type", 0, 4, KEY),
|
||||
("UNDEFINED[8] past the file", UNDEFINED, 8, w(u32::MAX)),
|
||||
("UNDEFINED[5] in the file", UNDEFINED, 5, w(tail_at as u32)),
|
||||
("UNDEFINED[3]", UNDEFINED, 3, KEY),
|
||||
("UNDEFINED[0]", UNDEFINED, 0, KEY),
|
||||
("BYTE[4]", BYTE, 4, [1, 2, 3, 4]),
|
||||
("SBYTE[4]", 6, 4, KEY),
|
||||
("ASCII[4]", 2, 4, KEY),
|
||||
("ASCII[4] with NUL", 2, 4, [0x41, 0x42, 0, 0x43]),
|
||||
("ASCII[4] invalid UTF-8", 2, 4, [0xff, 0x41, 0, 0]),
|
||||
("ASCII[4] two invalid bytes", 2, 4, [0xc3, 0x28, 0x41, 0]),
|
||||
("ASCII[9] at a pointer, NUL at 4", 2, 9, w(tail_at as u32 + 4)),
|
||||
("ASCII[8] at a pointer, no NUL", 2, 8, w(tail_at as u32 + 12)),
|
||||
("LONG[1]", LONG, 1, KEY),
|
||||
];
|
||||
for (what, kind, count, field) in &keys {
|
||||
let mut d = private(&[(SUBIFD_KEY, *kind, *count, *field), valid[0], valid[1], valid[2]], order);
|
||||
d.extend(&tail);
|
||||
cases.push((format!("key: first {what}"), d));
|
||||
}
|
||||
// the reader's value budget (8 × the data's length + 1 MiB), used up by entries aliasing the data
|
||||
for aliases in [23u32, 24] {
|
||||
let mut entries = vec![(0x7300, UNDEFINED, 1 << 16, w(0)); aliases as usize];
|
||||
entries.extend(valid);
|
||||
let mut d = private(&entries, order);
|
||||
d.resize(1 << 16, 0);
|
||||
cases.push((format!("{aliases} entries using the value budget"), d));
|
||||
}
|
||||
// no entries; more entries declared than the data holds
|
||||
cases.push(("empty directory".into(), private(&[], order)));
|
||||
let mut long = private(&valid, order);
|
||||
long[..2].copy_from_slice(&if order == Little { 10u16.to_le_bytes() } else { 10u16.to_be_bytes() });
|
||||
long.truncate(2 + 12 * 3);
|
||||
cases.push(("10 entries declared, 3 present".into(), long));
|
||||
cases
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_fields_match_the_tiff_readers_selection() {
|
||||
let mut differ = Vec::new();
|
||||
for order in [Little, Big] {
|
||||
for (case, d) in differential_cases(order) {
|
||||
let f = private_fields(&d, 0, order);
|
||||
let (ours, reader) = ((f.offset, f.length, f.key), reader_selects(&d, 0, order));
|
||||
if ours != reader {
|
||||
differ.push(format!("{order:?} {case}: {ours:?}, reader {reader:?}"));
|
||||
}
|
||||
}
|
||||
assert_eq!(private_fields(&[], 0, order), PrivateFields::default());
|
||||
assert_eq!(private_fields(&private(&[], order), u64::MAX, order), PrivateFields::default());
|
||||
}
|
||||
assert!(differ.is_empty(), "{} cases differ from the TIFF reader:\n{}", differ.len(), differ.join("\n"));
|
||||
// what the reader (and so this) gives in the cases the review named: the limit is 8,192 entries, a later
|
||||
// copy stands in for an unreadable first one, and a used-up value budget hides the fields
|
||||
let found = |d: &[u8]| private_fields(d, 0, Little) == PrivateFields { offset: Some(4096), length: Some(64), key: Some(KEY) };
|
||||
let cases = differential_cases(Little);
|
||||
let case = |name: &str| cases.iter().find(|(c, _)| c == name).map(|(_, d)| d.clone()).unwrap();
|
||||
for name in [
|
||||
"0 fillers",
|
||||
"256 fillers",
|
||||
"8189 fillers",
|
||||
"0x7200: first LONG[2] past the file",
|
||||
"0x7201: first unknown type 14",
|
||||
"key: first UNDEFINED[8] past the file",
|
||||
"23 entries using the value budget",
|
||||
] {
|
||||
assert!(found(&case(name)), "{name}");
|
||||
}
|
||||
for name in ["8190 fillers", "8192 fillers", "24 entries using the value budget", "0x7200: first ASCII", "key: first UNDEFINED[3]"] {
|
||||
assert!(!found(&case(name)), "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The case that broke on review: valid entries ahead of the three fields, and an unreadable first offset ahead of
|
||||
/// a valid one. Main's reader finds the block; so must this one (else the black level, white balance and crop it
|
||||
/// holds are lost).
|
||||
#[test]
|
||||
fn the_block_is_found_behind_filler_and_unreadable_entries() {
|
||||
for order in [Little, Big] {
|
||||
for (fillers, unreadable_first) in [(256usize, false), (300, false), (0, true), (256, true)] {
|
||||
let w = |v: u32| {
|
||||
let mut b = Vec::new();
|
||||
order.put_u32(&mut b, v);
|
||||
[b[0], b[1], b[2], b[3]]
|
||||
};
|
||||
let n = fillers + 3 + usize::from(unreadable_first);
|
||||
let start = 2 + 12 * n + 4;
|
||||
let block = block(start, &[(0x7310, SHORT, &[800; 4])], order);
|
||||
let mut entries = vec![(0x7300, LONG, 1, w(7)); fillers];
|
||||
if unreadable_first {
|
||||
entries.push((SUBIFD_OFFSET, LONG, 2, w(u32::MAX)));
|
||||
}
|
||||
entries.extend([
|
||||
(SUBIFD_OFFSET, LONG, 1, w(start as u32)),
|
||||
(SUBIFD_LENGTH, LONG, 1, w(block.len() as u32)),
|
||||
(SUBIFD_KEY, UNDEFINED, 4, KEY),
|
||||
]);
|
||||
let mut file = private(&entries, order);
|
||||
assert_eq!(file.len(), start);
|
||||
file.extend(&block);
|
||||
let mut pointer = Vec::new();
|
||||
order.put_u32(&mut pointer, 0);
|
||||
let ifd0 = Ifd {
|
||||
entries: vec![lightcraft_tiff::Entry { tag: t::DNG_PRIVATE_DATA, value: lightcraft_tiff::Value::Byte(pointer), offset: 0 }],
|
||||
..Default::default()
|
||||
};
|
||||
let black = SubIfd::read(&file, &ifd0, order).and_then(|s| s.shorts(0x7310));
|
||||
assert_eq!(black, Some(vec![800.0; 4]), "{order:?}: {fillers} fillers, unreadable first offset {unreadable_first}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_directories_give_none() {
|
||||
let start = 4096;
|
||||
|
||||
Vendored
+1
@@ -186,6 +186,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
@@ -1138,6 +1138,229 @@ fn nikon_in_file_distortion_is_limited_to_validated_bodies() {
|
||||
eprintln!("Nikon distortion boundary: {checked} corpus files checked");
|
||||
}
|
||||
|
||||
/// The Sony samples of the corpus manifest (`RAW_SAMPLES` in `xtask/src/main.rs`) with their pinned SHA-256.
|
||||
const SONY_SAMPLES: &[(&str, &str)] = &[
|
||||
("arw-sony-a7rm4a-compressed.arw", "690c774f1d7bc1db3fa8c2489762743d8e7586e50b6f65d0b6a61cb467972c67"),
|
||||
("arw-sony-a9m2-compressed.arw", "161c2a9da2b5f1e50be6117a0b4da0ce249660b5d7de13568d301c4034716b97"),
|
||||
("arw-sony-a7m3-compressed.arw", "250784580ea527442c09004417bb0eead484f2bf3ee8f9121a776ac65bb50d0f"),
|
||||
("arw-sony-a7m3-uncompressed.arw", "ece80551abf64949dbe826985a80f1d9b265401ee4ea6b989e1829549616fdcd"),
|
||||
("arw-sony-a7m4-14bit.arw", "639a6d4db881f1359e3ea7e1137b314e59417d24e6d4d862b7202111a2c823b2"),
|
||||
("arw-sony-a7m4-lossless-l.arw", "851b43c2116c4139104a5036f83ac3b6a148789b2142214dd7192c13972b25b6"),
|
||||
("arw-sony-a7m4-lossless-m.arw", "d453005714327addd75bcb99c1c6223173dc92f2a59542d82e6761ef0e0e7571"),
|
||||
("arw-sony-a7m4-lossless-s.arw", "cbbd0930c7d8706dff84c68a2004454266e6fd0d8354f5f76a106b5d776e0223"),
|
||||
("arw-sony-a7cr-lossless-l.arw", "3e1642f3a1ae7c9f93228c5f09f27e362608e6a4ca1b3e5d8dd9eb8fae576997"),
|
||||
("arw-sony-a7cr-lossless-m.arw", "5bcba4acc52a5b902074581b4a2f5fb3c79ec56bd0fdc1746b9e2c0f9d3e9c88"),
|
||||
("arw-sony-a7cr-compressed.arw", "f5096e8fbccf0842c8a57763cabd2836608f019eb7fd54f7b22260c2f5bc4da5"),
|
||||
("arw-sony-a7cm2-lossless-l.arw", "342d893bd71fd050c273df181e3039d3f12d88947b0662de5de5325c84cc449b"),
|
||||
("arw-sony-a7cm2-compressed-m.arw", "002c86d38a16b08a2438780728b4fda8efcc745fa6c26231dad6af67b383b11e"),
|
||||
("arw-sony-a7rm4-14bit-compressed.arw", "e6dafe42643f69ab9d1fd00414b7a1f5104df354bb589201600ac934b29b5e4a"),
|
||||
("arw-sony-rx100m3.arw", "cb81392e3a8810231ce341fb45de3e4d1b9e2f87bc47f3c314d084041ccfd138"),
|
||||
("arw-sony-rx100.arw", "579a485b5126a25cbd55cbd5dadfa7d09cf021c99cc7d4869f9e56e3f759390b"),
|
||||
("arw-sony-a7rm2-12bit-uncompressed.arw", "71e0888396ef52c7e6a990b4e72ebb4d8413a1fc1c35bada449870539cef6e39"),
|
||||
("arw-sony-a500.arw", "1407fb596a391df67c15b90026a1a76702a2e7386dbdff2f0815b903ea49cead"),
|
||||
("arw-sony-a700.arw", "3159e28892bf0771d01525cd4b6190f9c15bbb19fa2fab6d2515ced0594e8f40"),
|
||||
("arw-sony-a33.arw", "1a59856394f10d4fadb40f5ab9c6d1c89f473f4dbcdefd216fbbbbf1ad8d21f9"),
|
||||
("arw-sony-a3500-5600k.arw", "04c4fe04425c3e6fff7c8af0f811923c35e750300373357bd54342150c5505d8"),
|
||||
("arw-sony-a7s-shade.arw", "7cc338a0abc8fdad32d61006f1f8f412297b93e040c7efba572eb2bd8f8e8be2"),
|
||||
("arw-sony-a7sm2-16x9.arw", "46044fb6a9c805f2b4915cd4970316b4d04b0566bcf5287afaad3ef648d7a3db"),
|
||||
("arw-sony-a580-16x9.arw", "5b0924d39151239dce19e92e08318f4f62a5a8ac276bd7180ec8463d2cfee709"),
|
||||
("arw-sony-a900-packed12.arw", "ac7c1532df77c321e8010aa1be60c1b9f245b1ac0db6f38fe617a73aae81af50"),
|
||||
];
|
||||
|
||||
/// SHA-256 of [`SONY_SAMPLES`] as `name sha256` lines sorted by name: anchors the pins independently of the
|
||||
/// manifest, so editing both together (as a drift or a mistake) still fails `sony_pins_match_the_corpus_manifest`.
|
||||
/// All 25 pins were checked against the downloaded files when it was set.
|
||||
const SONY_PINS_SHA256: &str = "4cfc349194278146356b463c59ede500331c6ca5fc50aec4e92d7cd847b17b68";
|
||||
|
||||
/// A Sony corpus file's bytes, checked against its pinned SHA-256 before anything decodes them; `None` (with a
|
||||
/// note) when the file is absent, unless `LIGHTCRAFT_CORPUS_REQUIRED=1` asks for every pinned file. A file that
|
||||
/// differs from its pin fails the test.
|
||||
fn pinned_sony(root: &Path, name: &str) -> Option<Vec<u8>> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let expected = SONY_SAMPLES.iter().find(|(n, _)| *n == name).unwrap_or_else(|| panic!("{name}: no pinned SHA-256")).1;
|
||||
let bytes = match std::fs::read(root.join(name)) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
assert!(std::env::var_os("LIGHTCRAFT_CORPUS_REQUIRED").is_none_or(|v| v != "1"), "{name}: absent, and LIGHTCRAFT_CORPUS_REQUIRED=1");
|
||||
eprintln!("skip: {name} absent");
|
||||
return None;
|
||||
}
|
||||
Err(e) => panic!("{name}: {e}"),
|
||||
};
|
||||
let actual: String = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect();
|
||||
assert_eq!(actual, expected, "{name}: corpus file differs from its pinned SHA-256");
|
||||
Some(bytes)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sony_pins_match_the_corpus_manifest() {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut lines: Vec<String> = SONY_SAMPLES.iter().map(|(name, sha)| format!("{name} {sha}\n")).collect();
|
||||
lines.sort();
|
||||
let digest: String = Sha256::digest(lines.concat()).iter().map(|b| format!("{b:02x}")).collect();
|
||||
assert_eq!(digest, SONY_PINS_SHA256, "the pinned Sony SHA-256 table changed: check every changed pin against its file, then update the anchor");
|
||||
let manifest = include_str!("../../../xtask/src/main.rs");
|
||||
for (name, sha) in SONY_SAMPLES {
|
||||
let entry = manifest.find(&format!("\"{name}\"")).unwrap_or_else(|| panic!("{name} is not in the manifest"));
|
||||
assert!(manifest[entry..].split_once("),").is_some_and(|(e, _)| e.contains(sha)), "{name}: pinned SHA-256 differs from the manifest");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "differs from its pinned SHA-256")]
|
||||
fn a_sony_fixture_that_differs_from_its_pin_fails() {
|
||||
struct Gone(PathBuf);
|
||||
impl Drop for Gone {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
let dir = Gone(std::env::temp_dir().join(format!("lightcraft-sony-pin-{}", std::process::id())));
|
||||
std::fs::create_dir_all(&dir.0).unwrap();
|
||||
std::fs::write(dir.0.join("arw-sony-a7cr-compressed.arw"), b"not the pinned file").unwrap();
|
||||
pinned_sony(&dir.0, "arw-sony-a7cr-compressed.arw");
|
||||
}
|
||||
|
||||
/// Sony's lens-correction records (`lightcraft_raw::lens_metadata`, `crates/raw/src/vendor/arw_lens.rs`) on the public
|
||||
/// samples: shading, CA and distortion decoded from the raw IFD with their settings, equal to their `SR2SubIFD` copies,
|
||||
/// the same from header and full decode, and nothing applied beyond the validated ILCE-7RM4A distortion. Expected
|
||||
/// tables and settings: what ExifTool, run as a black-box tool, prints for these files. Every file is checked against
|
||||
/// its pinned SHA-256 before it is decoded; absent files are skipped.
|
||||
#[test]
|
||||
fn sony_lens_records_of_the_public_samples() {
|
||||
use lightcraft_raw::Rect;
|
||||
use lightcraft_raw::lens_metadata::{SonyCopy, SonyLensStatus, SonySettingMeaning as M, SonySource};
|
||||
let root = corpus_root().join("raw");
|
||||
// ILCE-7CR (Sigma 24-70mm F2.8 DG DN Art at 49.5 mm): one scene in three codings, identical tables
|
||||
let a7cr_shading = [0, 64, 128, 224, 320, 480, 672, 896, 1184, 1472, 1824, 2240, 2816, 3552, 4512, 7328];
|
||||
let a7cr_ca = [
|
||||
[-256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, -128],
|
||||
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -128, -128, -256, -256],
|
||||
];
|
||||
let a7cr_distortion = [4, 1, 1, 3, 7, 13, 22, 35, 51, 72, 98, 132, 173, 223, 279, 348];
|
||||
// ILCE-7RM4A (FE 24-105mm F4 G OSS at 24 mm)
|
||||
let a7rm4a_shading = [0, 32, 112, 256, 448, 688, 976, 1296, 1712, 2336, 3120, 4080, 5104, 6112, 7024, 7904];
|
||||
let a7rm4a_ca = [
|
||||
[1408, 1280, 1152, 1024, 1024, 1024, 1024, 1024, 896, 896, 896, 768, 768, 640, 512, 384],
|
||||
[512, 512, 384, 384, 384, 384, 384, 512, 512, 512, 512, 640, 640, 640, 768, 640],
|
||||
];
|
||||
let a7rm4a_distortion = [14, 0, -29, -65, -113, -171, -239, -311, -392, -471, -555, -634, -713, -782, -843, -891];
|
||||
let a7cr_crop = Rect::new(32, 20, 9504, 6336);
|
||||
// (file, settings, tables, distortion status, storage, compression, linear YCbCr, Sony crop, default crop)
|
||||
let cases = [
|
||||
(
|
||||
"arw-sony-a7cr-compressed.arw",
|
||||
[272, 1, 0],
|
||||
(a7cr_shading, a7cr_ca, a7cr_distortion),
|
||||
SonyLensStatus::NotValidatedModel,
|
||||
(9600, 6376),
|
||||
32767,
|
||||
false,
|
||||
a7cr_crop,
|
||||
a7cr_crop,
|
||||
),
|
||||
(
|
||||
"arw-sony-a7cr-lossless-l.arw",
|
||||
[272, 1, 0],
|
||||
(a7cr_shading, a7cr_ca, a7cr_distortion),
|
||||
SonyLensStatus::NotValidatedModel,
|
||||
(9728, 6656),
|
||||
7,
|
||||
false,
|
||||
a7cr_crop,
|
||||
a7cr_crop,
|
||||
),
|
||||
(
|
||||
"arw-sony-a7cr-lossless-m.arw",
|
||||
[272, 1, 0],
|
||||
(a7cr_shading, a7cr_ca, a7cr_distortion),
|
||||
SonyLensStatus::NotValidatedModel,
|
||||
(6656, 4608),
|
||||
7,
|
||||
true,
|
||||
Rect::new(20, 12, 6240, 4160),
|
||||
Rect::new(20, 12, 6240, 4160),
|
||||
),
|
||||
(
|
||||
"arw-sony-a7rm4a-compressed.arw",
|
||||
[257, 1, 17],
|
||||
(a7rm4a_shading, a7rm4a_ca, a7rm4a_distortion),
|
||||
SonyLensStatus::Applied,
|
||||
(9600, 6376),
|
||||
32767,
|
||||
false,
|
||||
Rect::new(0, 20, 9504, 6336),
|
||||
Rect::new(32, 20, 9504, 6336),
|
||||
),
|
||||
];
|
||||
let mut checked = 0;
|
||||
for (name, settings, (shading, ca, distortion), status, (w, h), compression, linear, sony_crop, default_crop) in cases {
|
||||
let Some(bytes) = pinned_sony(&root, name) else { continue };
|
||||
let full = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}"));
|
||||
assert_eq!(probe_info(&bytes).unwrap(), full.info(), "{name}");
|
||||
let s = full.lens_metadata.sony.as_ref().unwrap_or_else(|| panic!("{name}: no Sony lens record"));
|
||||
assert!(s.sr2_read, "{name}");
|
||||
let got = [s.shading.setting.value, s.chromatic_aberration.setting.value, s.distortion.setting.value].map(|v| v.map(|v| v.value));
|
||||
assert_eq!(got, settings.map(Some), "{name}: settings");
|
||||
let meanings = [s.shading.setting.value, s.chromatic_aberration.setting.value, s.distortion.setting.value].map(|v| v.map(|v| v.meaning));
|
||||
let forced = if settings[2] == 17 { M::ForcedAuto } else { M::Off };
|
||||
assert_eq!(meanings, [Some(M::Auto), Some(M::Auto), Some(forced)], "{name}: setting meanings");
|
||||
assert_eq!(
|
||||
(s.shading.table.value, s.chromatic_aberration.table.value, s.distortion.table.value),
|
||||
(Some(shading), Some(ca), Some(distortion)),
|
||||
"{name}"
|
||||
);
|
||||
for (source, raw, sr2, agree) in [
|
||||
(s.shading.table.source, s.shading.table.raw_ifd, s.shading.table.sr2, s.shading.table.copies_agree),
|
||||
(
|
||||
s.chromatic_aberration.table.source,
|
||||
s.chromatic_aberration.table.raw_ifd,
|
||||
s.chromatic_aberration.table.sr2,
|
||||
s.chromatic_aberration.table.copies_agree,
|
||||
),
|
||||
(s.distortion.table.source, s.distortion.table.raw_ifd, s.distortion.table.sr2, s.distortion.table.copies_agree),
|
||||
(s.shading.setting.source, s.shading.setting.raw_ifd, s.shading.setting.sr2, s.shading.setting.copies_agree),
|
||||
(
|
||||
s.chromatic_aberration.setting.source,
|
||||
s.chromatic_aberration.setting.raw_ifd,
|
||||
s.chromatic_aberration.setting.sr2,
|
||||
s.chromatic_aberration.setting.copies_agree,
|
||||
),
|
||||
(s.distortion.setting.source, s.distortion.setting.raw_ifd, s.distortion.setting.sr2, s.distortion.setting.copies_agree),
|
||||
] {
|
||||
assert_eq!((source, raw, sr2, agree), (Some(SonySource::RawIfd), SonyCopy::Valid, SonyCopy::Valid, Some(true)), "{name}: provenance");
|
||||
}
|
||||
assert_eq!(
|
||||
(s.shading.status, s.chromatic_aberration.status, s.distortion.status),
|
||||
(SonyLensStatus::NotValidatedComponent, SonyLensStatus::NotValidatedComponent, status),
|
||||
"{name}"
|
||||
);
|
||||
assert_eq!(full.opcodes.list3.len(), usize::from(status == SonyLensStatus::Applied), "{name}: only applied components are opcodes");
|
||||
let g = &s.geometry;
|
||||
assert_eq!((g.storage_width, g.storage_height, g.compression, g.linear_ycbcr), (w, h, compression, linear), "{name}");
|
||||
assert_eq!(
|
||||
(g.sony_crop, g.default_crop, g.image_crop, g.reference),
|
||||
(Some(sony_crop), Some(default_crop), sony_crop, default_crop),
|
||||
"{name}"
|
||||
);
|
||||
assert!(g.native_3_2, "{name}");
|
||||
eprintln!("{name}: shading, CA and distortion decoded; distortion {status:?}");
|
||||
checked += 1;
|
||||
}
|
||||
// every Sony corpus file: only the ILCE-7RM4A's distortion is ever applied, and an applied status is an opcode
|
||||
for (name, _) in SONY_SAMPLES {
|
||||
let Some(bytes) = pinned_sony(&root, name) else { continue };
|
||||
let info = probe_info(&bytes).unwrap_or_else(|e| panic!("{name}: {e}"));
|
||||
let Some(s) = &info.lens_metadata.sony else { continue };
|
||||
let applied = s.distortion.status == SonyLensStatus::Applied;
|
||||
assert_eq!(applied, info.metadata.model.as_deref() == Some("ILCE-7RM4A"), "{name}");
|
||||
assert_eq!(info.opcodes.list3.len(), usize::from(applied), "{name}");
|
||||
assert!(![s.shading.status, s.chromatic_aberration.status].contains(&SonyLensStatus::Applied), "{name}");
|
||||
checked += 1;
|
||||
}
|
||||
eprintln!("Sony lens records: {checked} checks on corpus files");
|
||||
}
|
||||
|
||||
/// Kodak DCS520C (`crates/raw/src/vendor/kodak.rs`): the two-component lossless JPEG strip decodes through the
|
||||
/// file's response curve to a 1736 x 1160 GRBG mosaic with 12-bit white, per-position black levels and the
|
||||
/// white balance of the vendor IFD. Looks for the corpus copy, then the raw.pixls.us folder; skips without either.
|
||||
|
||||
@@ -61,6 +61,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
lens_metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ fn mosaic(w: usize, h: usize) -> RawImage {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
lightcraft_raw::write_dng(&raw, &DngWriteOptions { compression: comp, ..Default::default() }).unwrap()
|
||||
}
|
||||
@@ -163,6 +164,7 @@ fn non_finite_black_levels_and_samples_develop() {
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
for img in [raw.clone(), lightcraft_raw::decode(&lightcraft_raw::write_dng(&raw, &DngWriteOptions::default()).unwrap()).unwrap()] {
|
||||
let rgb = img.develop(Method::Ahd).unwrap();
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
//! Opt-in local check of Sony's lens-correction records ([`lightcraft_raw::lens_metadata`]); it does nothing unless
|
||||
//! `LIGHTCRAFT_SONY_LENS_DIR` names a directory of ARW files, so CI never runs it. The files there (searched
|
||||
//! recursively) are only read, from their headers, and one line per file is printed: settings, where each table came
|
||||
//! from, whether its raw-IFD and `SR2SubIFD` copies agree, sample ranges and status. Header and full decode are
|
||||
//! compared too when `LIGHTCRAFT_SONY_LENS_FULL=1` (slow on large files).
|
||||
//!
|
||||
//! With `LIGHTCRAFT_SONY_LENS_INVENTORY` naming a CSV inventory of those files (comma-separated, no quoting, a header
|
||||
//! row, one row per file name), every file must have a row, every row a file, and these columns must match where
|
||||
//! present: `split` (the file's folder name), `raw_size` (`WxH`), `compression`, `crop_origin` / `crop_size` (Sony's
|
||||
//! crop tags, `x y` / `w h`), and for each of `shading`, `ca` and `distortion`: `_entries` (TIFF count), `_bytes`,
|
||||
//! `_samples` (the leading sample count), `_min` / `_max` (over the samples), `_sr2_equal` (`True` / `False`),
|
||||
//! `_setting_raw` and `_setting` (a label: `Off`, `Auto`, `Auto (ILCE-1)`, `Auto fixed by lens`, `No correction
|
||||
//! params available`). A file that doesn't decode must still have a row, and fails the check when it has one.
|
||||
//!
|
||||
//! Any decode error fails the check except an unsupported variant (`RawError::Unsupported`, e.g. the packed ARW of
|
||||
//! some older bodies): those are counted, and the count must equal `LIGHTCRAFT_SONY_LENS_UNSUPPORTED` (default 0).
|
||||
//!
|
||||
//! `LIGHTCRAFT_SONY_LENS_DIR=… LIGHTCRAFT_SONY_LENS_INVENTORY=….csv cargo test -p lightcraft-raw --release --test
|
||||
//! sony_lens_inventory -- --nocapture`
|
||||
|
||||
use lightcraft_raw::lens_metadata::{SonyCopy, SonyField, SonyLensCorrections, SonyLensStatus, SonySetting, SonySettingMeaning, SonySource};
|
||||
use lightcraft_raw::{RawError, RawInfo, decode, probe_info};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
fn arw_files(dir: &Path, depth: usize, out: &mut Vec<PathBuf>) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else { return };
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() && depth < 8 {
|
||||
arw_files(&path, depth + 1, out);
|
||||
} else if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("arw")) {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One component's columns, from the decoded record.
|
||||
struct Component {
|
||||
setting: Option<(u16, SonySettingMeaning)>,
|
||||
/// Samples when the table was decoded.
|
||||
samples: Option<Vec<i16>>,
|
||||
source: Option<SonySource>,
|
||||
/// The raw-IFD and `SR2SubIFD` copies' states.
|
||||
copies: (SonyCopy, SonyCopy),
|
||||
agree: Option<bool>,
|
||||
status: String,
|
||||
}
|
||||
|
||||
fn component<T>(setting: &SonyField<SonySetting>, table: &SonyField<T>, samples: Option<Vec<i16>>, status: SonyLensStatus) -> Component {
|
||||
Component {
|
||||
setting: setting.value.map(|s| (s.value, s.meaning)),
|
||||
samples,
|
||||
source: table.source,
|
||||
copies: (table.raw_ifd, table.sr2),
|
||||
agree: table.copies_agree,
|
||||
status: format!("{status:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Shading, CA and distortion of a record, by CSV column prefix.
|
||||
fn components(s: &SonyLensCorrections) -> [(&'static str, Component); 3] {
|
||||
let (sh, ca, d) = (&s.shading, &s.chromatic_aberration, &s.distortion);
|
||||
[
|
||||
("shading", component(&sh.setting, &sh.table, sh.table.value.map(|v| v.to_vec()), sh.status)),
|
||||
("ca", component(&ca.setting, &ca.table, ca.table.value.map(|v| v.concat()), ca.status)),
|
||||
("distortion", component(&d.setting, &d.table, d.table.value.map(|v| v.to_vec()), d.status)),
|
||||
]
|
||||
}
|
||||
|
||||
fn label(meaning: SonySettingMeaning) -> &'static [&'static str] {
|
||||
match meaning {
|
||||
SonySettingMeaning::Off => &["Off"],
|
||||
SonySettingMeaning::Auto => &["Auto", "Auto (ILCE-1)"],
|
||||
SonySettingMeaning::ForcedAuto => &["Auto fixed by lens"],
|
||||
SonySettingMeaning::Unavailable => &["No correction params available"],
|
||||
SonySettingMeaning::Unknown(_) => &[],
|
||||
}
|
||||
}
|
||||
|
||||
/// The columns this check derives from a file, by CSV column name.
|
||||
fn columns(path: &Path, info: &RawInfo) -> BTreeMap<String, String> {
|
||||
let mut out = BTreeMap::new();
|
||||
let mut put = |k: &str, v: String| {
|
||||
out.insert(k.to_string(), v);
|
||||
};
|
||||
if let Some(split) = path.parent().and_then(|p| p.file_name()) {
|
||||
put("split", split.to_string_lossy().into_owned());
|
||||
}
|
||||
let Some(s) = &info.lens_metadata.sony else { return out };
|
||||
let g = &s.geometry;
|
||||
put("raw_size", format!("{}x{}", g.storage_width, g.storage_height));
|
||||
put("compression", g.compression.to_string());
|
||||
if let Some(c) = g.sony_crop {
|
||||
put("crop_origin", format!("{} {}", c.x, c.y));
|
||||
put("crop_size", format!("{} {}", c.width, c.height));
|
||||
}
|
||||
for (name, c) in components(s) {
|
||||
if let Some(v) = &c.samples {
|
||||
put(&format!("{name}_entries"), (v.len() + 1).to_string());
|
||||
put(&format!("{name}_bytes"), (2 * (v.len() + 1)).to_string());
|
||||
put(&format!("{name}_samples"), v.len().to_string());
|
||||
put(&format!("{name}_min"), v.iter().min().map_or(String::new(), i16::to_string));
|
||||
put(&format!("{name}_max"), v.iter().max().map_or(String::new(), i16::to_string));
|
||||
}
|
||||
put(&format!("{name}_sr2_equal"), c.agree.map_or("unknown", |a| if a { "True" } else { "False" }).to_string());
|
||||
if let Some((value, _)) = c.setting {
|
||||
put(&format!("{name}_setting_raw"), value.to_string());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn summary(info: &RawInfo) -> String {
|
||||
let Some(s) = &info.lens_metadata.sony else { return "no lens record".into() };
|
||||
let mut line = format!("sr2 {} |", if s.sr2_read { "read" } else { "absent" });
|
||||
for (name, c) in components(s) {
|
||||
let setting = c.setting.map_or("-".into(), |(v, m)| format!("{v} {m:?}"));
|
||||
let range = c
|
||||
.samples
|
||||
.as_ref()
|
||||
.map_or("no table".into(), |v| format!("{} samples {}..{}", v.len(), v.iter().min().unwrap_or(&0), v.iter().max().unwrap_or(&0)));
|
||||
line +=
|
||||
&format!(" {name}: {setting}, {range}, {:?} (copies {:?} / {:?}), agree {:?}, {} |", c.source, c.copies.0, c.copies.1, c.agree, c.status);
|
||||
}
|
||||
line + &format!(" crop {:?} reference {:?} 3:2 {}", s.geometry.image_crop, s.geometry.reference, s.geometry.native_3_2)
|
||||
}
|
||||
|
||||
/// `(header, rows by file name)` of a simple CSV file.
|
||||
fn read_csv(path: &Path) -> (Vec<String>, BTreeMap<String, Vec<String>>) {
|
||||
let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
|
||||
let mut lines = text.lines().filter(|l| !l.trim().is_empty());
|
||||
let header: Vec<String> = lines.next().expect("CSV header").split(',').map(|s| s.trim().to_string()).collect();
|
||||
let file = header.iter().position(|h| h == "file").expect("a `file` column");
|
||||
let mut rows = BTreeMap::new();
|
||||
for line in lines {
|
||||
assert!(!line.contains('"'), "quoted CSV fields are not supported: {line}");
|
||||
let row: Vec<String> = line.split(',').map(|s| s.trim().to_string()).collect();
|
||||
assert_eq!(row.len(), header.len(), "CSV row with {} fields, header has {}: {line}", row.len(), header.len());
|
||||
let name = row[file].clone();
|
||||
assert!(rows.insert(name.clone(), row).is_none(), "{name} listed twice");
|
||||
}
|
||||
(header, rows)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sony_lens_records_reproduce_a_local_inventory() {
|
||||
let Some(dir) = std::env::var_os("LIGHTCRAFT_SONY_LENS_DIR").map(PathBuf::from) else {
|
||||
eprintln!("skip: LIGHTCRAFT_SONY_LENS_DIR unset");
|
||||
return;
|
||||
};
|
||||
let full = std::env::var_os("LIGHTCRAFT_SONY_LENS_FULL").is_some_and(|v| v == "1");
|
||||
let inventory = std::env::var_os("LIGHTCRAFT_SONY_LENS_INVENTORY").map(PathBuf::from);
|
||||
let unsupported = std::env::var("LIGHTCRAFT_SONY_LENS_UNSUPPORTED").map_or(0, |v| v.parse().expect("LIGHTCRAFT_SONY_LENS_UNSUPPORTED: a count"));
|
||||
let report = check(&dir, inventory.as_deref(), full, unsupported);
|
||||
assert!(report.problems.is_empty(), "{} problems", report.problems.len());
|
||||
}
|
||||
|
||||
/// What [`check`] found.
|
||||
#[derive(Debug, Default)]
|
||||
struct Report {
|
||||
files: usize,
|
||||
records: usize,
|
||||
without_record: usize,
|
||||
unsupported: usize,
|
||||
problems: Vec<String>,
|
||||
}
|
||||
|
||||
/// Check every ARW under `dir` (see the module docs). Any decode error is a problem except `Unsupported`, which
|
||||
/// is counted and must come to exactly `expected_unsupported`; with an inventory, every file (decoded or not) must
|
||||
/// have a row, and a file with a row must decode.
|
||||
fn check(dir: &Path, inventory: Option<&Path>, full: bool, expected_unsupported: usize) -> Report {
|
||||
let inventory = inventory.map(read_csv);
|
||||
let mut files = Vec::new();
|
||||
arw_files(dir, 0, &mut files);
|
||||
files.sort();
|
||||
let mut report = Report { files: files.len(), ..Default::default() };
|
||||
if files.is_empty() {
|
||||
report.problems.push(format!("no ARW files under {}", dir.display()));
|
||||
}
|
||||
let (mut matched, mut mismatches, mut seen) = (0usize, Vec::new(), std::collections::BTreeSet::new());
|
||||
let compared: Vec<String> = ["split", "raw_size", "compression", "crop_origin", "crop_size"]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.chain(
|
||||
["shading", "ca", "distortion"]
|
||||
.into_iter()
|
||||
.flat_map(|k| ["entries", "bytes", "samples", "min", "max", "sr2_equal", "setting_raw"].into_iter().map(move |c| format!("{k}_{c}"))),
|
||||
)
|
||||
.collect();
|
||||
for path in &files {
|
||||
let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
|
||||
let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
|
||||
seen.insert(name.clone());
|
||||
let info = match probe_info(&bytes) {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
eprintln!("{name}: {e}");
|
||||
match (&inventory, &e) {
|
||||
(Some((_, rows)), _) if rows.contains_key(&name) => mismatches.push(format!("{name}: does not decode: {e}")),
|
||||
(Some(_), _) => mismatches.push(format!("{name}: does not decode ({e}) and is not in the inventory")),
|
||||
(None, RawError::Unsupported(_)) => {}
|
||||
(None, _) => mismatches.push(format!("{name}: does not decode: {e}")),
|
||||
}
|
||||
report.unsupported += usize::from(matches!(e, RawError::Unsupported(_)));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if full {
|
||||
match decode(&bytes) {
|
||||
Ok(image) if image.info() == info => {}
|
||||
Ok(_) => mismatches.push(format!("{name}: header and full decode differ")),
|
||||
Err(e) => mismatches.push(format!("{name}: full decode fails: {e}")),
|
||||
}
|
||||
}
|
||||
if info.lens_metadata.sony.is_some() {
|
||||
report.records += 1;
|
||||
} else {
|
||||
report.without_record += 1;
|
||||
}
|
||||
eprintln!("{name}: {} {}", info.metadata.model.as_deref().unwrap_or("?"), summary(&info));
|
||||
let Some((header, rows)) = &inventory else { continue };
|
||||
let Some(row) = rows.get(&name) else {
|
||||
mismatches.push(format!("{name}: not in the inventory"));
|
||||
continue;
|
||||
};
|
||||
let got = columns(path, &info);
|
||||
let mut ok = true;
|
||||
for (i, column) in header.iter().enumerate() {
|
||||
let expected = &row[i];
|
||||
if let Some(setting) = column.strip_suffix("_setting")
|
||||
&& let Some(s) = &info.lens_metadata.sony
|
||||
{
|
||||
let meaning = components(s).into_iter().find(|(n, _)| *n == setting).and_then(|(_, c)| c.setting).map(|(_, m)| m);
|
||||
if !meaning.is_some_and(|m| label(m).contains(&expected.as_str())) {
|
||||
mismatches.push(format!("{name}: {column} {meaning:?}, inventory {expected}"));
|
||||
ok = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if !compared.contains(column) {
|
||||
continue;
|
||||
}
|
||||
let value = got.get(column).map(String::as_str).unwrap_or("(none)");
|
||||
if value != expected {
|
||||
mismatches.push(format!("{name}: {column} {value}, inventory {expected}"));
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
matched += usize::from(ok);
|
||||
}
|
||||
if let Some((header, rows)) = &inventory {
|
||||
for name in rows.keys().filter(|n| !seen.contains(*n)) {
|
||||
mismatches.push(format!("{name}: in the inventory, no such file"));
|
||||
}
|
||||
let checked = header.iter().filter(|c| compared.contains(c) || c.ends_with("_setting")).count();
|
||||
assert!(checked > 0, "the inventory has none of the compared columns");
|
||||
eprintln!("{matched} of {} inventory rows reproduced from {} files ({checked} columns compared)", rows.len(), files.len());
|
||||
}
|
||||
if report.unsupported != expected_unsupported {
|
||||
mismatches.push(format!("{} files are unsupported variants, {expected_unsupported} expected", report.unsupported));
|
||||
}
|
||||
eprintln!(
|
||||
"{} files: {} with a lens record, {} decoded without one, {} unsupported ({expected_unsupported} expected)",
|
||||
report.files, report.records, report.without_record, report.unsupported
|
||||
);
|
||||
for m in &mismatches {
|
||||
eprintln!("problem: {m}");
|
||||
}
|
||||
report.problems.extend(mismatches);
|
||||
report
|
||||
}
|
||||
|
||||
/// A directory under the system temporary directory, removed when dropped.
|
||||
struct Scratch(PathBuf);
|
||||
|
||||
impl Scratch {
|
||||
fn new(name: &str) -> Scratch {
|
||||
let dir = std::env::temp_dir().join(format!("lightcraft-sony-lens-{name}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
Scratch(dir)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Scratch {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// A small Sony CFA file with a distortion setting (a lens record), or, `unsupported`, the packed 12-bit ARW variant
|
||||
/// the decoder refuses as unsupported.
|
||||
fn synthetic_arw(unsupported: bool) -> Vec<u8> {
|
||||
use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value, tags as t};
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::MAKE, Value::Ascii("SONY".into()));
|
||||
raw.set(t::MODEL, Value::Ascii("ILCE-7CR".into()));
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![32]));
|
||||
raw.set(t::IMAGE_LENGTH, Value::Long(vec![4]));
|
||||
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
|
||||
raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA]));
|
||||
raw.set(0x7036, Value::Short(vec![0]));
|
||||
if unsupported {
|
||||
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![12]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![32767]));
|
||||
raw.set(0x7010, Value::Short(vec![8000, 10400, 12900, 14100]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0; 32 * 4 + 8]] });
|
||||
} else {
|
||||
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![1]));
|
||||
raw.set(t::WHITE_LEVEL, Value::Long(vec![16383]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0; 32 * 4 * 2]] });
|
||||
}
|
||||
TiffWriter::default().write(&[raw]).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_that_fail_to_decode_fail_the_check() {
|
||||
// a file that is no raw at all fails the check, whatever is expected
|
||||
let dir = Scratch::new("decode-error");
|
||||
std::fs::write(dir.0.join("not-a-raw.arw"), b"not a raw file").unwrap();
|
||||
let report = check(&dir.0, None, false, 0);
|
||||
assert!(!report.problems.is_empty(), "an ARW that does not decode passed the check");
|
||||
assert!(!check(&dir.0, None, false, 1).problems.is_empty(), "a decode error counted as an unsupported variant");
|
||||
// an unsupported variant is counted, and must be expected
|
||||
let dir = Scratch::new("unsupported");
|
||||
std::fs::write(dir.0.join("good.arw"), synthetic_arw(false)).unwrap();
|
||||
std::fs::write(dir.0.join("packed.arw"), synthetic_arw(true)).unwrap();
|
||||
let report = check(&dir.0, None, false, 0);
|
||||
assert_eq!((report.files, report.records, report.unsupported), (2, 1, 1));
|
||||
assert_eq!(report.problems.len(), 1, "{:?}", report.problems);
|
||||
assert!(check(&dir.0, None, true, 1).problems.is_empty());
|
||||
// with an inventory, a file that fails must still be listed: here it is not
|
||||
let csv = dir.0.join("inventory.csv");
|
||||
std::fs::write(&csv, "file,distortion_setting_raw\ngood.arw,0\n").unwrap();
|
||||
// (the inventory itself sits in the directory, but only .arw files are checked)
|
||||
let mut report = check(&dir.0, Some(&csv), false, 1);
|
||||
report.problems.sort();
|
||||
assert_eq!(report.problems.len(), 1, "{:?}", report.problems);
|
||||
assert!(report.problems[0].contains("packed.arw") && report.problems[0].contains("not in the inventory"), "{:?}", report.problems);
|
||||
// listed but failing, and listed and decoding
|
||||
std::fs::write(&csv, "file,distortion_setting_raw\ngood.arw,0\npacked.arw,0\n").unwrap();
|
||||
let report = check(&dir.0, Some(&csv), false, 1);
|
||||
assert!(report.problems.len() == 1 && report.problems[0].contains("packed.arw: does not decode"), "{:?}", report.problems);
|
||||
std::fs::remove_file(dir.0.join("packed.arw")).unwrap();
|
||||
std::fs::write(&csv, "file,distortion_setting_raw\ngood.arw,0\n").unwrap();
|
||||
assert!(check(&dir.0, Some(&csv), false, 0).problems.is_empty());
|
||||
std::fs::write(&csv, "file,distortion_setting_raw\ngood.arw,17\n").unwrap();
|
||||
assert!(!check(&dir.0, Some(&csv), false, 0).problems.is_empty(), "a wrong value passed");
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
//! Peak heap of reading Sony's `SR2Private` directory (`crates/raw/src/vendor/sr2.rs`), measured with dhat's testing
|
||||
//! mode. Every ARW with `DNGPrivateData` has its `SR2SubIFD` located and decrypted, so the directory that points to
|
||||
//! it must not cost memory sized by its other entries: a hostile file can give it fields that each alias most of the
|
||||
//! file. Only the block's offset, length and key may be read. One test per binary (dhat's profiler is global).
|
||||
|
||||
use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, Tiff, TiffWriter, Value, tags as t};
|
||||
|
||||
#[global_allocator]
|
||||
static ALLOC: dhat::Alloc = dhat::Alloc;
|
||||
|
||||
/// A 32 × 24 Sony CFA file of `size` bytes whose `SR2Private` directory holds the block's offset, length and key plus
|
||||
/// `aliases` UNDEFINED fields that each cover the first `size` bytes of the file. Plain black level and white balance
|
||||
/// are present, so nothing but the lens record needs the `SR2SubIFD`.
|
||||
fn hostile_arw(size: usize, aliases: u16) -> Vec<u8> {
|
||||
let o = ByteOrder::Little;
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::MAKE, Value::Ascii("SONY".into()));
|
||||
raw.set(t::MODEL, Value::Ascii("ILCE-7CR".into()));
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![32]));
|
||||
raw.set(t::IMAGE_LENGTH, Value::Long(vec![24]));
|
||||
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
|
||||
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
|
||||
raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![1]));
|
||||
raw.set(t::WHITE_LEVEL, Value::Long(vec![16383]));
|
||||
raw.set(0x7310, Value::Short(vec![512; 4]));
|
||||
raw.set(0x7313, Value::Short(vec![2048, 1024, 1024, 1536]));
|
||||
// a distortion setting, so the file has a lens record that says whether the SR2SubIFD was read
|
||||
raw.set(0x7036, Value::Short(vec![0]));
|
||||
raw.set(t::DNG_PRIVATE_DATA, Value::Byte(vec![0; 4]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 24, strips: vec![vec![0; 32 * 24 * 2]] });
|
||||
let mut file = TiffWriter::new(o, false).write(&[raw]).unwrap();
|
||||
let pointer = Tiff::parse(&file).unwrap().ifds[0].get(t::DNG_PRIVATE_DATA).unwrap().offset as usize;
|
||||
file.resize(size, 0);
|
||||
let private = file.len();
|
||||
let entries = 3 + aliases;
|
||||
let block_at = private + 2 + 12 * usize::from(entries) + 4;
|
||||
let block_len = 64u32;
|
||||
o.put_u16(&mut file, entries);
|
||||
for (tag, kind, count, value) in [(0x7200u16, 4u16, 1u32, block_at as u32), (0x7201, 4, 1, block_len)] {
|
||||
o.put_u16(&mut file, tag);
|
||||
o.put_u16(&mut file, kind);
|
||||
o.put_u32(&mut file, count);
|
||||
o.put_u32(&mut file, value);
|
||||
}
|
||||
o.put_u16(&mut file, 0x7221);
|
||||
o.put_u16(&mut file, 7);
|
||||
o.put_u32(&mut file, 4);
|
||||
file.extend([0x11, 0x22, 0x33, 0x44]);
|
||||
for i in 0..aliases {
|
||||
o.put_u16(&mut file, 0x7230 + i);
|
||||
o.put_u16(&mut file, 7);
|
||||
o.put_u32(&mut file, size as u32);
|
||||
o.put_u32(&mut file, 0);
|
||||
}
|
||||
o.put_u32(&mut file, 0);
|
||||
file.resize(block_at + block_len as usize, 0);
|
||||
let mut at = Vec::new();
|
||||
o.put_u32(&mut at, private as u32);
|
||||
file[pointer..pointer + 4].copy_from_slice(&at);
|
||||
file
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sr2_private_fields_cost_no_memory_sized_by_the_file() {
|
||||
// eight fields of 16 MiB each: 128 MiB if they were decoded, within the TIFF reader's default value budget
|
||||
let file = hostile_arw(16 << 20, 8);
|
||||
let _profiler = dhat::Profiler::builder().testing().build();
|
||||
let header = lightcraft_raw::probe_info(&file).unwrap();
|
||||
let full = lightcraft_raw::decode(&file).unwrap();
|
||||
let peak = dhat::HeapStats::get().max_bytes;
|
||||
// the block was still found and decrypted (its content is noise: no SR2SubIFD copies)
|
||||
for info in [&header, &full.info()] {
|
||||
let sony = info.lens_metadata.sony.as_ref().expect("a lens record");
|
||||
assert!(sony.sr2_read, "the SR2SubIFD was not read");
|
||||
}
|
||||
eprintln!("peak heap {peak} bytes decoding a {} byte file", file.len());
|
||||
assert!(peak < 2 << 20, "peak heap {peak} bytes decoding a {} byte file", file.len());
|
||||
}
|
||||
@@ -18,6 +18,10 @@ later the TIFF/DNG exporters.
|
||||
- `image::ImageInfo` — width/height, bits, samples, compression, photometric, planar config, predictor,
|
||||
sample format and the strip or tile chunk list (`chunks()`, `chunk_bytes()`); missing byte counts are
|
||||
inferred.
|
||||
- `directory_entries`, `entry_span`, `ParseOptions::budget_for` and `decode_value` are the reader's own rules
|
||||
(directory layout and entry limit, which entries it reads, its value budget, value decoding), for callers that
|
||||
must select entries exactly as `parse_ifd_at` would without decoding the whole directory (`lightcraft-raw`'s
|
||||
Sony `SR2Private` lookup).
|
||||
- `TiffWriter` writes IFD chains with strips or tiles, `SubIFDs` and pointer IFDs, in either byte order,
|
||||
classic or BigTIFF; `writer::rational` / `srational` approximate floats as rationals.
|
||||
|
||||
@@ -25,7 +29,9 @@ later the TIFF/DNG exporters.
|
||||
|
||||
The reader never panics on malformed input (proptests feed random and mutated files): offsets are
|
||||
bounds-checked; out-of-range or unknown-type entries are skipped; IFD loops are detected; IFD count,
|
||||
nesting depth and entries per IFD are limited; the total bytes of decoded values are capped at
|
||||
nesting depth and entries per IFD are limited; a next-IFD pointer is followed only when every declared entry was
|
||||
read, with the counts compared as 64-bit numbers (a BigTIFF declaring 4,294,967,297 entries but holding one can't
|
||||
pass for a whole directory on a 32-bit target); the total bytes of decoded values are capped at
|
||||
`8 × input + 1 MiB` so a file cannot make us allocate far more than its own size.
|
||||
|
||||
## Sources
|
||||
|
||||
@@ -25,7 +25,7 @@ mod reader;
|
||||
pub mod tags;
|
||||
pub mod writer;
|
||||
|
||||
pub use reader::{ParseOptions, parse_ifd_at};
|
||||
pub use reader::{DirectoryEntries, EntrySpan, ParseOptions, decode as decode_value, directory_entries, entry_span, parse_ifd_at};
|
||||
pub use writer::{IfdBuilder, ImageData, TiffWriter};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
+215
-46
@@ -26,6 +26,108 @@ impl Default for ParseOptions {
|
||||
}
|
||||
}
|
||||
|
||||
impl ParseOptions {
|
||||
/// The value budget for `data_len` bytes of input: [`Self::value_budget`], else `8 × data_len + 1 MiB`.
|
||||
pub fn budget_for(&self, data_len: usize) -> u64 {
|
||||
self.value_budget.unwrap_or((data_len as u64).saturating_mul(8).saturating_add(1 << 20))
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a directory's entries are and how many of them the reader reads, as [`directory_entries`] found them.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct DirectoryEntries {
|
||||
/// Position of the first entry.
|
||||
table: usize,
|
||||
/// Entries read: the declared count, cut to the entries the data holds and to the `max_entries` limit.
|
||||
read: usize,
|
||||
/// The declared entry count.
|
||||
declared: u64,
|
||||
/// Bytes per entry: 12, or 20 in BigTIFF.
|
||||
entry_size: usize,
|
||||
}
|
||||
|
||||
impl DirectoryEntries {
|
||||
/// Positions of the entries read, in directory order. Checked arithmetic: the iteration ends rather than
|
||||
/// overflow, whatever the layout.
|
||||
pub fn positions(&self) -> impl Iterator<Item = usize> + use<> {
|
||||
let (table, size) = (self.table, self.entry_size);
|
||||
(0..self.read).map_while(move |i| i.checked_mul(size)?.checked_add(table))
|
||||
}
|
||||
|
||||
/// The number of entries read.
|
||||
pub fn read(&self) -> usize {
|
||||
self.read
|
||||
}
|
||||
|
||||
/// The entry count the directory declares.
|
||||
pub fn declared(&self) -> u64 {
|
||||
self.declared
|
||||
}
|
||||
}
|
||||
|
||||
/// The directory at `offset` as the reader walks it: its entry table and how many entries it reads. The same errors
|
||||
/// as [`parse_ifd_at`] for a directory it can't read: past the data, empty, or without room for one entry.
|
||||
pub fn directory_entries(data: &[u8], offset: u64, order: ByteOrder, bigtiff: bool, max_entries: usize) -> Result<DirectoryEntries> {
|
||||
let pos = usize::try_from(offset).map_err(|_| TiffError::Truncated(offset))?;
|
||||
let (n, head, entry_size) = if bigtiff {
|
||||
(order.read_u64(data, pos).ok_or(TiffError::Truncated(offset))?, 8usize, 20usize)
|
||||
} else {
|
||||
(order.read_u16(data, pos).ok_or(TiffError::Truncated(offset))? as u64, 2, 12)
|
||||
};
|
||||
let table = pos + head;
|
||||
let fit = (data.len().saturating_sub(table) / entry_size) as u64;
|
||||
let read = n.min(fit).min(max_entries as u64) as usize;
|
||||
if n == 0 {
|
||||
return Err(TiffError::Invalid(format!("empty IFD at {offset}")));
|
||||
}
|
||||
if read == 0 {
|
||||
return Err(TiffError::Truncated(offset));
|
||||
}
|
||||
Ok(DirectoryEntries { table, read, declared: n, entry_size })
|
||||
}
|
||||
|
||||
/// A directory entry the reader can read: its tag, type and count, and where its values are.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct EntrySpan {
|
||||
pub tag: u16,
|
||||
pub field_type: FieldType,
|
||||
pub count: u64,
|
||||
/// Position of the values in the data: the entry's own value field when they fit it, else `base` plus the offset
|
||||
/// stored there.
|
||||
pub start: u64,
|
||||
/// `count × size` bytes.
|
||||
pub len: u64,
|
||||
}
|
||||
|
||||
/// The entry at `pos`, when the reader reads it: a known field type, `count × size` without overflow, and values
|
||||
/// inside `data`. Nothing is decoded; [`parse_ifd_at`] also drops an entry whose values exceed what is left of its
|
||||
/// value budget ([`ParseOptions::budget_for`]), which this leaves to the caller.
|
||||
pub fn entry_span(data: &[u8], pos: usize, order: ByteOrder, bigtiff: bool, base: u64) -> Option<EntrySpan> {
|
||||
let tag = order.read_u16(data, pos)?;
|
||||
let field_type = FieldType::from_u16(order.read_u16(data, pos.checked_add(2)?)?)?;
|
||||
let (count, field, inline) = if bigtiff {
|
||||
(order.read_u64(data, pos.checked_add(4)?)?, pos.checked_add(12)?, 8u64)
|
||||
} else {
|
||||
(order.read_u32(data, pos.checked_add(4)?)? as u64, pos.checked_add(8)?, 4)
|
||||
};
|
||||
let len = count.checked_mul(field_type.size() as u64)?;
|
||||
let start = if len <= inline {
|
||||
field as u64
|
||||
} else {
|
||||
let rel = if bigtiff { order.read_u64(data, field)? } else { order.read_u32(data, field)? as u64 };
|
||||
base.checked_add(rel)?
|
||||
};
|
||||
let end = start.checked_add(len)?;
|
||||
(end <= data.len() as u64).then_some(EntrySpan { tag, field_type, count, start, len })
|
||||
}
|
||||
|
||||
/// Whether a directory declaring `declared` entries was read whole (`read` entries), so that its next-IFD pointer
|
||||
/// follows them. The counts compare as 64-bit numbers: narrowing the declared count to `usize` instead would, on a
|
||||
/// 32-bit target, let a BigTIFF declaring 4,294,967,297 entries but holding one pass for a whole directory.
|
||||
fn reads_every_entry(declared: u64, read: usize) -> bool {
|
||||
u64::try_from(read).is_ok_and(|read| read == declared)
|
||||
}
|
||||
|
||||
/// Parse the header: (byte order, bigtiff, first IFD offset).
|
||||
pub(crate) fn header(data: &[u8]) -> Result<(ByteOrder, bool, u64)> {
|
||||
header_with(data, true)
|
||||
@@ -97,7 +199,7 @@ struct Ctx<'a> {
|
||||
|
||||
impl<'a> Ctx<'a> {
|
||||
fn new(data: &'a [u8], order: ByteOrder, big: bool, opts: &'a ParseOptions) -> Self {
|
||||
let budget = opts.value_budget.unwrap_or((data.len() as u64).saturating_mul(8).saturating_add(1 << 20));
|
||||
let budget = opts.budget_for(data.len());
|
||||
Self { data, order, big, opts, visited: HashSet::new(), budget }
|
||||
}
|
||||
|
||||
@@ -116,34 +218,19 @@ impl<'a> Ctx<'a> {
|
||||
if !self.visited.insert(offset) {
|
||||
return Err(TiffError::Invalid(format!("IFD loop at {offset}")));
|
||||
}
|
||||
let pos = usize::try_from(offset).map_err(|_| TiffError::Truncated(offset))?;
|
||||
let (n, head, entry_size, inline) = if self.big {
|
||||
(self.order.read_u64(self.data, pos).ok_or(TiffError::Truncated(offset))?, 8usize, 20usize, 8usize)
|
||||
} else {
|
||||
(self.order.read_u16(self.data, pos).ok_or(TiffError::Truncated(offset))? as u64, 2, 12, 4)
|
||||
};
|
||||
let table = pos + head;
|
||||
let fit = (self.data.len().saturating_sub(table) / entry_size) as u64;
|
||||
let n_read = n.min(fit).min(self.opts.max_entries as u64) as usize;
|
||||
if n == 0 {
|
||||
return Err(TiffError::Invalid(format!("empty IFD at {offset}")));
|
||||
}
|
||||
if n_read == 0 {
|
||||
return Err(TiffError::Truncated(offset));
|
||||
}
|
||||
let mut entries = Vec::with_capacity(n_read);
|
||||
for i in 0..n_read {
|
||||
let p = table + i * entry_size;
|
||||
if let Some(e) = self.entry(p, base, inline) {
|
||||
let dir = directory_entries(self.data, offset, self.order, self.big, self.opts.max_entries)?;
|
||||
let mut entries = Vec::with_capacity(dir.read);
|
||||
for p in dir.positions() {
|
||||
if let Some(e) = self.entry(p, base) {
|
||||
entries.push(e);
|
||||
}
|
||||
}
|
||||
entries.sort_by_key(|e| e.tag);
|
||||
entries.dedup_by_key(|e| e.tag);
|
||||
// the next-IFD offset follows the whole table, so only a table read in full has one: a count far past the
|
||||
// file (a BigTIFF's is 64-bit) must not reach the arithmetic (issue #1516)
|
||||
let next = if n == n_read as u64 {
|
||||
match n_read.checked_mul(entry_size).and_then(|len| table.checked_add(len)).and_then(|p| self.read_off(p)) {
|
||||
// the next-IFD pointer follows the last entry, read only when every declared entry was (checked: a BigTIFF
|
||||
// count can be anything up to u64::MAX)
|
||||
let next = if reads_every_entry(dir.declared, dir.read) {
|
||||
match dir.read.checked_mul(dir.entry_size).and_then(|n| n.checked_add(dir.table)).and_then(|p| self.read_off(p)) {
|
||||
Some(0) | None => 0,
|
||||
Some(v) => base.checked_add(v).unwrap_or(0),
|
||||
}
|
||||
@@ -181,33 +268,19 @@ impl<'a> Ctx<'a> {
|
||||
ifd.interop = interop;
|
||||
}
|
||||
|
||||
fn entry(&mut self, p: usize, base: u64, inline: usize) -> Option<Entry> {
|
||||
let o = self.order;
|
||||
let tag = o.read_u16(self.data, p)?;
|
||||
let typ = FieldType::from_u16(o.read_u16(self.data, p + 2)?)?;
|
||||
let (count, field) = if self.big { (o.read_u64(self.data, p + 4)?, p + 12) } else { (o.read_u32(self.data, p + 4)? as u64, p + 8) };
|
||||
let total = count.checked_mul(typ.size() as u64)?;
|
||||
let start = if total <= inline as u64 {
|
||||
field as u64
|
||||
} else {
|
||||
let rel = self.read_off(field)?;
|
||||
base.checked_add(rel)?
|
||||
};
|
||||
let end = start.checked_add(total)?;
|
||||
if end > self.data.len() as u64 {
|
||||
fn entry(&mut self, p: usize, base: u64) -> Option<Entry> {
|
||||
let span = entry_span(self.data, p, self.order, self.big, base)?;
|
||||
if span.len > self.budget {
|
||||
return None;
|
||||
}
|
||||
if total > self.budget {
|
||||
return None;
|
||||
}
|
||||
self.budget -= total;
|
||||
let bytes = &self.data[start as usize..end as usize];
|
||||
Some(Entry { tag, value: decode(o, typ, bytes), offset: start })
|
||||
self.budget -= span.len;
|
||||
let bytes = self.data.get(usize::try_from(span.start).ok()?..usize::try_from(span.start + span.len).ok()?)?;
|
||||
Some(Entry { tag: span.tag, value: decode(self.order, span.field_type, bytes), offset: span.start })
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode `bytes` (exactly `count × size` long) as values of type `typ`.
|
||||
pub(crate) fn decode(o: ByteOrder, typ: FieldType, b: &[u8]) -> Value {
|
||||
/// Decode `bytes` (exactly `count × size` long, or the first values of an entry) as values of type `typ`.
|
||||
pub fn decode(o: ByteOrder, typ: FieldType, b: &[u8]) -> Value {
|
||||
fn arr<const N: usize>(c: &[u8]) -> [u8; N] {
|
||||
let mut a = [0u8; N];
|
||||
a.copy_from_slice(c);
|
||||
@@ -237,3 +310,99 @@ pub(crate) fn decode(o: ByteOrder, typ: FieldType, b: &[u8]) -> Value {
|
||||
FieldType::SLong8 => Value::SLong8(b.as_chunks::<8>().0.iter().map(|c| o.u64(arr(c)) as i64).collect()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A BigTIFF whose first IFD, at 80, declares `declared` entries but holds one, followed by a next-IFD pointer
|
||||
/// `next`; a valid one-entry IFD sits at 16.
|
||||
fn bigtiff_chain(order: ByteOrder, declared: u64, next: u64) -> Vec<u8> {
|
||||
let ifd = |d: &mut Vec<u8>, count: u64, tag: u16, next: u64| {
|
||||
order.put_u64(d, count);
|
||||
order.put_u16(d, tag);
|
||||
order.put_u16(d, 3);
|
||||
order.put_u64(d, 1);
|
||||
order.put_u16(d, 8);
|
||||
d.extend([0; 6]);
|
||||
order.put_u64(d, next);
|
||||
};
|
||||
let mut d = if order == ByteOrder::Little { b"II".to_vec() } else { b"MM".to_vec() };
|
||||
order.put_u16(&mut d, 43);
|
||||
order.put_u16(&mut d, 8);
|
||||
order.put_u16(&mut d, 0);
|
||||
order.put_u64(&mut d, 80);
|
||||
ifd(&mut d, 1, 0x0101, 0);
|
||||
d.resize(80, 0);
|
||||
ifd(&mut d, declared, 0x0100, next);
|
||||
d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_directory_read_whole_has_a_next_pointer() {
|
||||
// the comparison itself, on every target: a declared count past u32::MAX never matches a small read count
|
||||
for (declared, read, whole) in [
|
||||
(0, 0, true),
|
||||
(1, 1, true),
|
||||
(8192, 8192, true),
|
||||
(8193, 8192, false),
|
||||
(1, 0, false),
|
||||
((1 << 32) + 1, 1, false),
|
||||
((2 << 32) + 1, 1, false),
|
||||
((1 << 63) + 1, 1, false),
|
||||
(u64::MAX, 1, false),
|
||||
] {
|
||||
assert_eq!(reads_every_entry(declared, read), whole, "{declared} declared, {read} read");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bigtiff_count_past_u32_max_never_follows_the_next_pointer() {
|
||||
// the whole chain: on 32-bit targets (wasm32) the old cast truncated these counts to 1 and followed the pointer
|
||||
for order in [ByteOrder::Little, ByteOrder::Big] {
|
||||
for declared in [(1u64 << 32) + 1, (2u64 << 32) + 1, (1u64 << 63) + 1] {
|
||||
let d = bigtiff_chain(order, declared, 16);
|
||||
let tiff = Tiff::parse(&d).unwrap();
|
||||
assert_eq!(tiff.ifds.len(), 1, "{order:?} {declared}");
|
||||
assert_eq!(tiff.ifds[0].u64(0x0100), Some(8));
|
||||
assert_eq!(parse_ifd_at(&d, 80, order, 0, true, &ParseOptions::default()).unwrap().1, 0, "{order:?} {declared}");
|
||||
}
|
||||
// control: the same file declaring its one entry follows the pointer to the IFD at 16
|
||||
let whole = Tiff::parse(&bigtiff_chain(order, 1, 16)).unwrap();
|
||||
assert_eq!(whole.ifds.len(), 2, "{order:?}");
|
||||
assert_eq!(whole.ifds[1].u64(0x0101), Some(8));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_positions_never_overflow() {
|
||||
// a hand-built layout the reader never produces: the iterator stops instead of overflowing
|
||||
let hostile = DirectoryEntries { table: usize::MAX, read: 2, declared: 2, entry_size: 12 };
|
||||
assert_eq!(hostile.positions().collect::<Vec<_>>(), vec![usize::MAX]);
|
||||
assert_eq!(hostile.positions().nth(1), None);
|
||||
let wide = DirectoryEntries { table: usize::MAX - 30, read: 4, declared: 4, entry_size: 12 };
|
||||
assert_eq!(wide.positions().collect::<Vec<_>>(), vec![usize::MAX - 30, usize::MAX - 18, usize::MAX - 6]);
|
||||
// a directory read from data lists its entries
|
||||
let mut data = Vec::new();
|
||||
ByteOrder::Little.put_u16(&mut data, 2);
|
||||
data.extend([0; 24]);
|
||||
let dir = directory_entries(&data, 0, ByteOrder::Little, false, 8192).unwrap();
|
||||
assert_eq!(dir.positions().collect::<Vec<_>>(), vec![2, 14]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bigtiff_directory_declaring_u64_max_entries_parses_without_overflow() {
|
||||
// 44 bytes: BigTIFF header (first IFD at 16), an entry count of u64::MAX, one 20-byte entry
|
||||
let mut data = b"II\x2b\x00\x08\x00\x00\x00".to_vec();
|
||||
data.extend(16u64.to_le_bytes());
|
||||
data.extend(u64::MAX.to_le_bytes());
|
||||
data.extend(0x0100u16.to_le_bytes());
|
||||
data.extend(3u16.to_le_bytes());
|
||||
data.extend(1u64.to_le_bytes());
|
||||
data.extend([8, 0, 0, 0, 0, 0, 0, 0]);
|
||||
assert_eq!(data.len(), 44);
|
||||
let tiff = Tiff::parse(&data).unwrap();
|
||||
assert_eq!(tiff.ifds.len(), 1);
|
||||
assert_eq!(tiff.ifds[0].u64(0x0100), Some(8));
|
||||
}
|
||||
}
|
||||
@@ -215,7 +215,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts).
|
||||
| LR-EDIT-DETAIL-SUPERRES | Super resolution | P2 | ⬜ | | |
|
||||
| LR-EDIT-DETAIL-AISHARPEN | AI sharpen | OOS | 🚫 | | |
|
||||
| LR-EDIT-OPTICS-CA | Remove chromatic aberration | P1 | ✅ | `crates/ui-egui/src/panels/edit.rs` (checkbox), `ctl:optics.caRed`, `ctl:optics.caBlue` | |
|
||||
| LR-EDIT-OPTICS-PROFILE | Lens profile corrections | P1 | 🟡 | `ctl:optics.profileDistortion`, `ctl:optics.profileVignetting`, `crates/pipeline/src/optics.rs`, `crates/raw/src/vendor/rw2.rs`, `crates/raw/src/vendor/raf_lens.rs`, `crates/engine/src/crs.rs` | uses corrections embedded in the file: DNG `WarpRectilinear` / `WarpRectilinear2` (opcode 14, DNG 1.6: degree-14 radial polynomial, valid-radius clamp, reciprocal option, applied in list order with the spec's skip rule for a `WarpRectilinear` after an optional one, so Pixel DNGs' two-stage warp is corrected; up to 3 chained stages, sampled on the CPU because the polynomials need `f64`; issue #635) / `FixVignetteRadial`, and the camera's own distortion correction in Panasonic / Leica RW2 / RWL files (tag `0x0119`, read as the equivalent `WarpRectilinear`; on 79 photos from 13 bodies and 11 lenses the corrected render lines up with the camera JPEG within 0.75 px at 1440 px at the corners, from up to 13 % of the half diagonal uncorrected; issue #256); an imported preset's `LensProfileEnable=0` leaves them on, as Lightroom applies built-in corrections regardless (inferred); Sony ILCE-7RM4A Bayer ARW raw-IFD distortion tables (`0x7037`, 16-sample layout, native 3:2 framing) use the same warp path, enabled only for the independently validated ILCE-7RM4A with FE 24–105 and FE 200–600 lenses; ILCE-7RM6 verified against the camera JPEG on 54 photos with three lenses (`docs/sony-ilce7rm6-lens-corrections.md`); other Sony models remain uncorrected (see `docs/sony-lens-corrections.md`); Nikon NEF distortion from NikonNEFInfo (`0xc7d5`: polynomial, frame-filling zoom and the optical-centre offset left by in-body stabilisation; `crates/raw/src/vendor/nef_lens.rs`, `docs/nikon-lens-corrections.md`) for the Z 6II, Z 8 and Z f, full-frame image area only: F-mount lenses on the FTZ match the camera JPEG within 0.2 px median, Z lenses within 0.5–1.5 px except the 24-70 f/4's wide end, where the camera applies a further correction that is not in the readable data (7 px median at 24 mm, from 16 px / 106 px p95 uncorrected); Fujifilm RAF raw-IFD tables (`0xf00b` distortion, `0xf010` vignetting; `crates/raw/src/vendor/raf_lens.rs`, `docs/fuji-lens-corrections.md`) become a `WarpRectilinear` with the camera's frame-filling zoom and a `FixVignetteRadial` gain for 59 body models with sample files checked against the camera JPEG (the table indexed by the source radius with last knot at the corner; on 67 body/lens combinations the corrected render's radial shape error against the camera JPEG falls from a median of 0.72 % to 0.05 % of the half diagonal, largest 0.42 %, and the vignetting gain of the JPEG tracks the table at a median correlation of 0.987); files whose table scale is not the recorded image's half diagonal (X-S20, X-T30 III, X-M5, X-T5 crops) and six unmatched bodies stay uncorrected; Fujifilm's lateral CA table (`0xf00f`) is real but is not applied, because our demosaic already registers the planes and the stored shift over-corrects; vignetting and lateral CA data of Olympus and Sony raws not read yet; Reload from Disk reads the file's lens data for photos catalogued without it and turns the correction on for those still at import settings (issue #314); lens-profile database: a lensfun-format XML database in `LIGHTCRAFT_LENSFUN_DIR` (none is bundled) supplies distortion (poly3 / poly5 / PTLens), TCA and vignetting for lenses it lists (`crates/lensdb`, `crates/engine/src/lens_profiles.rs`, `docs/lens-profiles.md`); no settings switch or bundled database yet |
|
||||
| LR-EDIT-OPTICS-PROFILE | Lens profile corrections | P1 | 🟡 | `ctl:optics.profileDistortion`, `ctl:optics.profileVignetting`, `crates/pipeline/src/optics.rs`, `crates/raw/src/vendor/rw2.rs`, `crates/raw/src/vendor/raf_lens.rs`, `crates/engine/src/crs.rs` | uses corrections embedded in the file: DNG `WarpRectilinear` / `WarpRectilinear2` (opcode 14, DNG 1.6: degree-14 radial polynomial, valid-radius clamp, reciprocal option, applied in list order with the spec's skip rule for a `WarpRectilinear` after an optional one, so Pixel DNGs' two-stage warp is corrected; up to 3 chained stages, sampled on the CPU because the polynomials need `f64`; issue #635) / `FixVignetteRadial`, and the camera's own distortion correction in Panasonic / Leica RW2 / RWL files (tag `0x0119`, read as the equivalent `WarpRectilinear`; on 79 photos from 13 bodies and 11 lenses the corrected render lines up with the camera JPEG within 0.75 px at 1440 px at the corners, from up to 13 % of the half diagonal uncorrected; issue #256); an imported preset's `LensProfileEnable=0` leaves them on, as Lightroom applies built-in corrections regardless (inferred); Sony ILCE-7RM4A Bayer ARW raw-IFD distortion tables (`0x7037`, 16-sample layout, native 3:2 framing) use the same warp path, enabled only for the independently validated ILCE-7RM4A with FE 24–105 and FE 200–600 lenses; ILCE-7RM6 verified against the camera JPEG on 54 photos with three lenses (`docs/sony-ilce7rm6-lens-corrections.md`); other Sony models remain uncorrected (see `docs/sony-lens-corrections.md`); Nikon NEF distortion from NikonNEFInfo (`0xc7d5`: polynomial, frame-filling zoom and the optical-centre offset left by in-body stabilisation; `crates/raw/src/vendor/nef_lens.rs`, `docs/nikon-lens-corrections.md`) for the Z 6II, Z 8 and Z f, full-frame image area only: F-mount lenses on the FTZ match the camera JPEG within 0.2 px median, Z lenses within 0.5–1.5 px except the 24-70 f/4's wide end, where the camera applies a further correction that is not in the readable data (7 px median at 24 mm, from 16 px / 106 px p95 uncorrected); Fujifilm RAF raw-IFD tables (`0xf00b` distortion, `0xf010` vignetting; `crates/raw/src/vendor/raf_lens.rs`, `docs/fuji-lens-corrections.md`) become a `WarpRectilinear` with the camera's frame-filling zoom and a `FixVignetteRadial` gain for 59 body models with sample files checked against the camera JPEG (the table indexed by the source radius with last knot at the corner; on 67 body/lens combinations the corrected render's radial shape error against the camera JPEG falls from a median of 0.72 % to 0.05 % of the half diagonal, largest 0.42 %, and the vignetting gain of the JPEG tracks the table at a median correlation of 0.987); files whose table scale is not the recorded image's half diagonal (X-S20, X-T30 III, X-M5, X-T5 crops) and six unmatched bodies stay uncorrected; Fujifilm's lateral CA table (`0xf00f`) is real but is not applied, because our demosaic already registers the planes and the stored shift over-corrects; vignetting and lateral CA data of Olympus raws not read yet; Sony ARW shading, lateral CA and distortion tables (raw IFD `0x7031`–`0x7037`, with the encrypted `SR2SubIFD` copies as a checked fallback) are decoded with their camera settings, provenance and framing evidence for every body that records them (`crates/raw/src/lens_metadata.rs`), but only the ILCE-7RM4A and ILCE-7RM6 distortion above is applied: on the ILCE-7CR the unchanged distortion interpretation misses the camera JPEG at 24 mm (median 1.6 px, p95 6.2 px at 2376 px), and Sony shading and CA semantics are not validated beyond the ILCE-7RM6's (lateral CA off) (`docs/sony-lens-corrections.md`); Reload from Disk reads the file's lens data for photos catalogued without it and turns the correction on for those still at import settings (issue #314); lens-profile database: a lensfun-format XML database in `LIGHTCRAFT_LENSFUN_DIR` (none is bundled) supplies distortion (poly3 / poly5 / PTLens), TCA and vignetting for lenses it lists (`crates/lensdb`, `crates/engine/src/lens_profiles.rs`, `docs/lens-profiles.md`); no settings switch or bundled database yet |
|
||||
)
|
||||
| LR-EDIT-OPTICS-DEFRINGE | Defringe | P1 | ✅ | `ctl:optics.defringe*` | no fringe eyedropper |
|
||||
| LR-EDIT-OPTICS-MANUAL | Manual distortion / vignetting | P1 | ✅ | `ctl:optics.distortion`, `ctl:optics.vignetting`, `ctl:optics.vignettingMidpoint` | |
|
||||
@@ -797,6 +797,7 @@ Latin stays upright. Advanced Japanese composition remains open. Regression cove
|
||||
|
||||
| Date | Change | Summary |
|
||||
|---|---|---|
|
||||
| 2026-10-12 | minor | LR-EDIT-OPTICS-PROFILE: Sony ARW shading, CA and distortion tables decoded with settings and provenance (raw IFD, `SR2SubIFD` fallback); applied corrections unchanged (ILCE-7RM4A and ILCE-7RM6 distortion); status unchanged (🟡) |
|
||||
| 2026-10-14 | minor | LR-IMP-CAMERA-COVERAGE: 17 uncompressed Olympus and Samsung Adobe models verified on one CC0 file each; status unchanged (🟡) |
|
||||
| 2026-10-13 | minor | LR-EDIT-SECTION-SOLO: Solo Mode for Edit sections (Window ▸ Edit Sections, section-header menu, ⌥-click); KEYC-PANELS notes it |
|
||||
| 2026-10-14 | minor | LR-MERGE-HDR ✅ → 🟡: tested on real 67 MP Sony brackets; moving clipped highlights (black-hole stars) and dark-sky ghosts fixed, merge memory cut ~25 %; AI Denoise on merged DNGs, JPEG response curves and per-frame memory remain (gap 29). LR-MERGE-HEADLESS, MENU-PHOTO-MERGE: HDR for Each Bracket Set (`merge.hdrBatch`) |
|
||||
|
||||
+170
-37
@@ -1,30 +1,143 @@
|
||||
# Sony embedded distortion corrections
|
||||
# Sony embedded lens corrections
|
||||
|
||||
The **ILCE-7RM6** (A7R VI) has its own conversion, with other conventions measured against its camera JPEGs:
|
||||
[Sony ARW lens corrections](sony-ilce7rm6-lens-corrections.md). On ILCE-7RM6 photos the formula below misses the 24–70 GM II by
|
||||
up to 1.43 px at 1440 px while fitting the 100 mm and 200–600 better, and no single unit fits every lens under either
|
||||
convention; that page's *Two conventions* section has the comparison and what would settle it.
|
||||
|
||||
LightCraft reads the signed 16-sample distortion table in the raw image IFD (`0x7037`) of
|
||||
Sony **ILCE-7RM4A (A7R IVA)** Bayer ARWs. Other camera models are deliberately left uncorrected,
|
||||
even when they contain a table with the same layout.
|
||||
It converts that table into `OpcodeList3` / `WarpRectilinear`, so the existing profile correction control,
|
||||
CPU/GPU optics path, EXIF orientation handling, and DNG export use the same correction.
|
||||
Sony ARWs record three lens corrections per shot in the raw image IFD: shading (vignetting), lateral chromatic
|
||||
aberration (CA) and distortion, each as a camera setting and a radial table, with copies in the encrypted
|
||||
`SR2SubIFD`. LightCraft **decodes all three** wherever a file carries them and **applies one**: the distortion table
|
||||
of the **ILCE-7RM4A (A7R IVA)** in Bayer ARWs with native 3:2 framing, independently validated with the
|
||||
FE 24–105mm F4 G OSS and FE 200–600mm F5.6–6.3 G OSS (the ILCE-7RM6's distortion is applied too, by its own
|
||||
conversion above). Every other model,
|
||||
and shading and CA on every model, are decoded only: no warp, vignetting or CA correction is added for them. The decoded record stays with the image
|
||||
(`RawInfo::lens_metadata` / `RawImage::lens_metadata`, types in `crates/raw/src/lens_metadata.rs`, decoder in
|
||||
`crates/raw/src/vendor/arw_lens.rs`) for diagnostics and for the validation work described below.
|
||||
|
||||
This adds **distortion only**. Sony vignetting and lateral chromatic-aberration tables are not decoded.
|
||||
The enabled and independently validated model is ILCE-7RM4A with the FE 24–105mm F4 G OSS and
|
||||
FE 200–600mm F5.6–6.3 G OSS. Linear YCbCr ARWs and non-3:2 camera crops are excluded pending validation. An aspect crop may
|
||||
retain the full-frame radial normalization; the decoder does not guess that relationship. Older files that carry
|
||||
only encrypted correction metadata, different table lengths, and rejected tables remain uncorrected.
|
||||
The camera's distortion Off setting does not erase its table. Newly imported photos use LightCraft's existing
|
||||
embedded-profile default (enabled); the profile correction control can disable or adjust it.
|
||||
The applied distortion is converted into `OpcodeList3` / `WarpRectilinear`, so the existing profile correction
|
||||
control, CPU/GPU optics path, EXIF orientation handling, and DNG export use the same correction. Linear YCbCr ARWs
|
||||
and non-3:2 camera crops are excluded pending validation. An aspect crop may retain the full-frame radial
|
||||
normalization; the decoder does not guess that relationship. Tables of other lengths and rejected tables remain
|
||||
uncorrected, and a table found only in the encrypted `SR2SubIFD` is never applied. The camera's distortion Off
|
||||
setting does not erase its table. Newly imported photos use LightCraft's existing embedded-profile default
|
||||
(enabled); the profile correction control can disable or adjust it.
|
||||
|
||||
## What is decoded
|
||||
|
||||
| Component | Raw-IFD setting / table | `SR2SubIFD` setting / table | Accepted layout |
|
||||
|---|---|---|---|
|
||||
| Shading | `0x7031` / `0x7032` | `0x797c` / `0x797d` | SHORT\[1\]; SSHORT\[17\]: 16, then 16 samples |
|
||||
| Lateral CA | `0x7034` / `0x7035` | `0x797f` / `0x7980` | SHORT\[1\]; SSHORT\[33\]: 32, then two 16-sample curves |
|
||||
| Distortion | `0x7036` / `0x7037` | `0x7981` / `0x7982` | SHORT\[1\]; SSHORT\[17\]: 16, then 16 samples |
|
||||
|
||||
- **Bounded, exact reading.** A raw-IFD field is the one the TIFF reader parsed, the copy the image itself uses:
|
||||
its first occurrence, within the reader's limits (8,192 entries per directory, a total value budget). The directory
|
||||
is also scanned, in either byte order and in BigTIFF, for diagnostics only: a tag that occurs more than once is
|
||||
flagged (`raw_ifd_repeated`, `sr2_repeated`), and an entry the reader left out is reported with its reason (values
|
||||
outside the file, an unknown type, or past the reader's limits) rather than treated as absent. `SR2SubIFD` fields
|
||||
are read straight from their first entry. A setting may also be a nonnegative SSHORT (the documented editor type
|
||||
differs from the type the cameras store). Anything else is refused with its reason (`SonyFieldError`: field type,
|
||||
count, leading sample count, values outside the file or block, past the reader's limits, a negative setting).
|
||||
Nothing is allocated from a count in the file.
|
||||
- **Raw IFD first, `SR2SubIFD` as a checked fallback.** The raw IFD's copy wins. The `SR2SubIFD`'s copy is used only
|
||||
when the raw IFD lacks the tag; a malformed raw-IFD copy is reported, never silently replaced. When both copies are
|
||||
well-formed their agreement is recorded (`copies_agree`), and on disagreement the raw IFD's copy is used. The block
|
||||
is decrypted (by the existing `vendor/sr2.rs`) for every ARW that has one, so the check always runs; a block that
|
||||
can't be found or decrypted gives no copies. Locating it selects the `SR2Private` directory's offset, length and
|
||||
key exactly as the TIFF reader would (the reader's own directory layout, 8,192-entry limit, readable-entry rules
|
||||
and value budget, then the first readable occurrence of each), but decodes only each field's first value, so the
|
||||
directory's other entries cost no memory however large they claim to be; the block itself is limited to 1 MiB. The
|
||||
plain tag numbers are not looked up inside the `SR2SubIFD`.
|
||||
- **Settings keep their number and a documented meaning:** shading 256 Off, 257 and 272 Auto, 511 unavailable; CA
|
||||
0 Off, 1 Auto, 255 unavailable; distortion 0 Off, 1 Auto, 17 Auto fixed by the lens, 255 unavailable. Any other value
|
||||
is `Unknown(value)`, never Off (the A7CR's Low shading setting has no documented number). Values are not read as
|
||||
bit masks. Off does not mean absent: the table is still recorded.
|
||||
- **Only the 16-sample (CA: 2 × 16) layout is interpreted.** Many bodies store 11 (CA: 22) samples padded into the
|
||||
same 17 (33) values: older compacts and bodies such as the DSC-RX100 III and ILCE-3500, and the ILCE-7M4's APS-C
|
||||
captures (its full-frame captures carry 16). Those tables are refused as `CountPrefix(11)` and stay
|
||||
uninterpreted. An all-zero table is kept: it is a valid no-op, not a sign of an unsupported lens.
|
||||
- **Framing evidence travels with the tables:** the stored frame (padding included), Sony's crop tags and the DNG
|
||||
default crop as recorded, the image crop, the reference rectangle the distortion conversion is centred on and
|
||||
normalized to (the default crop when present, else the image crop), compression, whether the data is linear
|
||||
YCbCr (lossless M/S), and whether both crops are native 3:2.
|
||||
- **Per-component status.** `Applied` (converted into an opcode), `NoTable`, `NotValidatedComponent` (shading and CA
|
||||
on every body), `NotValidatedModel`, `NotValidatedMode` (linear YCbCr, non-3:2 framing, or a distortion table found
|
||||
only in the `SR2SubIFD`), `Neutral` (all-zero table in the validated scope) and `Rejected` (the validated
|
||||
conversion refused the table). A readable table never implies a usable correction; only `Applied` components
|
||||
become opcodes.
|
||||
|
||||
The applied path is unchanged: the validated conversion still receives the raw IFD's `0x7037` table exactly as the
|
||||
TIFF reader parsed it, with the same model, framing and fit checks, and produces the same warp. A repeated tag keeps
|
||||
the reader's choice and is only flagged, and a table the reader never reads (past its entry limit) is reported but
|
||||
never applied, so already-catalogued photos, their previews and their stored lens data stay consistent with a fresh
|
||||
decode.
|
||||
|
||||
## What each component still needs
|
||||
|
||||
The following summarizes an investigation on the ILCE-7CR (A7CR): an owner's collection of 131 raws with four
|
||||
lenses (Voigtländer APO-LANTHAR 28mm F2 and 50mm F2, both manual focus; Sony FE 24–50mm F2.8 G; Tamron 35–150mm
|
||||
F2–2.8), plus the three public raw.pixls.us A7CR samples. No private photographs, names or derived images are
|
||||
part of this repository.
|
||||
|
||||
- **Inventory.** All 131 raws carry all three tables in both the raw IFD and the `SR2SubIFD`, and the copies agree
|
||||
in every file; all four lenses provide tables, the manual-focus ones included. Shading is 272 (Auto) and CA 1
|
||||
(Auto) everywhere; distortion is 17 (Auto, fixed by the lens) on the nine FE 24–50 files and 0 (Off) on the rest.
|
||||
Every image crop is 9504 × 6336 at (32, 20), in uncompressed, compressed and lossless-L (padded to 9728 × 6656)
|
||||
storage. The decoder reproduces this inventory exactly (see *Local inventory check*).
|
||||
- **Distortion.** The table layout and crop tags match the A7R IVA's, but its interpretation applied unchanged does
|
||||
not reproduce the A7CR camera JPEG at 24 mm: at 2376 px wide, median / 95th-percentile feature residuals were
|
||||
1.61 / 6.17 px (12.98 / 51.19 px uncorrected, 6,539 matches), with a signed radial pattern (about +1.7 px at
|
||||
radii 0.25–0.5, −7.4 px at 0.8–1) that no translation removes. At 50 mm it agrees (0.23 / 1.05 px, from
|
||||
2.21 / 12.25 px). Scaling the raw table by about 10/9, or using the MakerNote's distortion array instead, brings
|
||||
24 mm to about 0.37 / 1.10 and 0.33 / 0.97 px. Those are exploratory comparisons on two files against the camera
|
||||
JPEG, not a model: the MakerNote values are roughly 1.11 times the raw table without one exact ratio, and the
|
||||
A7R IVA was validated against a different reference renderer (Imaging Edge exports). Needed: A7CR Imaging Edge
|
||||
On/Off exports at 24 and 50 mm and with another lens, compared with the raw table at its fixed scale; a decision on
|
||||
the reference (camera JPEG or Imaging Edge); then ILCE-7CR added to the validated models for the full-frame Bayer
|
||||
3:2 mode only. Neither a 10/9 multiplier nor the MakerNote table is used.
|
||||
- **Shading.** The layout decodes, but no public specification gives the sample-to-gain function, its radius
|
||||
convention on the A7CR, or whether the raw samples already contain part of the correction. Brightness ratios on six
|
||||
natural scenes show no consistent missing peripheral brightening. Needed: evenly lit flat fields with shading
|
||||
Off / Auto / Low at several apertures and focal lengths, raw CFA ratios between them, and Imaging Edge On/Off
|
||||
exports, to fix the gain function and processing stage and rule out double correction.
|
||||
- **Lateral CA.** A public analysis reads the table as a red, then a blue curve relative to green, at a 2⁻²¹
|
||||
relative radial scale. On three distortion-Off files that candidate reduced red and blue feature residuals
|
||||
modestly (for example 0.35 → 0.29 px median for red with the 28 mm), which supports investigating it but certifies
|
||||
neither scale, interpolation nor processing stage. Two product changes come before it: embedded CA must be
|
||||
independent of the distortion amount (today `pipeline::optics` scales the whole embedded per-plane warp with the
|
||||
profile distortion control), and the red and blue planes must be resampled in camera RGB, before the camera colour
|
||||
model mixes them (`engine::files::load_bytes_now` applies the colour model before the optics stage). Then
|
||||
validation on full-resolution achromatic edges.
|
||||
- **Other modes.** APS-C, aspect crops and lossless M/S (linear YCbCr) captures were not measured and stay
|
||||
excluded.
|
||||
|
||||
## Local inventory check
|
||||
|
||||
`crates/raw/tests/sony_lens_inventory.rs` is opt-in and does nothing in CI. With `LIGHTCRAFT_SONY_LENS_DIR` naming a
|
||||
directory of ARWs it decodes each from its headers and prints its record (settings, sources, copies' agreement,
|
||||
sample ranges, statuses); `LIGHTCRAFT_SONY_LENS_FULL=1` also compares header and full decode. With
|
||||
`LIGHTCRAFT_SONY_LENS_INVENTORY` naming a CSV inventory it checks every file's table counts, sizes, sample counts,
|
||||
minima and maxima, raw-IFD versus `SR2SubIFD` agreement, settings (numbers and labels), stored size, compression and
|
||||
crop tags. A file with a row must decode, and every file must have a row, whether it decodes or not. Any decode
|
||||
error fails the check except an unsupported variant, which is counted and must equal `LIGHTCRAFT_SONY_LENS_UNSUPPORTED`
|
||||
(default 0). On the 131 A7CR raws above, all 131 inventory rows reproduce and every file decodes. On 133 public Sony
|
||||
ARWs (127 raw.pixls.us samples from 100 bodies, four corpus copies and two files attached to a public issue), 110 carry
|
||||
a record, 14 decode without one, and 9 are unsupported variants (eight packed ARWs of older bodies and one ARW
|
||||
without a CFA image); header and full decode agree on the 124 that decode, and no record has disagreeing copies.
|
||||
|
||||
## Clean-room evidence
|
||||
|
||||
The [ExifTool EXIF tag-name documentation](https://exiftool.org/TagNames/EXIF.html) identifies
|
||||
`0x7037` as 17 signed 16-bit integers. The first value is the count, 16, followed by radial samples.
|
||||
No external photo decoder implementation or Adobe lens profile was consulted.
|
||||
Sony Imaging Edge Edit 4.1 was used only as a black-box reference on local scratch copies.
|
||||
The [ExifTool EXIF tag-name documentation](https://exiftool.org/TagNames/EXIF.html) names the raw-IFD tags
|
||||
`0x7031` to `0x7037` (VignettingCorrection / VignettingCorrParams, ChromaticAberrationCorrection /
|
||||
ChromaticAberrationCorrParams, DistortionCorrection / DistortionCorrParams) with their setting values, and gives the
|
||||
tables as 17 or 33 signed 16-bit integers whose first value is the count. The
|
||||
[Sony SR2SubIFD documentation](https://exiftool.org/TagNames/Sony.html#SR2SubIFD) names the encrypted copies
|
||||
`0x797d`, `0x7980` and `0x7982`; the setting tags next to them (`0x797c`, `0x797f`, `0x7981`) are unnamed there and
|
||||
were identified by inspecting files, where they equal the raw-IFD settings. The corpus test's expected tables are the
|
||||
values ExifTool, run as a black-box tool, prints for the public samples. No external photo decoder implementation,
|
||||
lens database or Adobe lens profile was consulted. Sony Imaging Edge Edit 4.1 was used only as a black-box reference
|
||||
on local scratch copies.
|
||||
|
||||
The observed inverse mapping, with radius `r` normalized to the default crop's half diagonal, is:
|
||||
|
||||
@@ -50,7 +163,8 @@ so the warp stays on the optical centre either way. On the A7R IVA that framing
|
||||
pixels left of Sony's exports (eight pixels in a 2376-pixel-wide export).
|
||||
|
||||
The main raw-IFD table is important: the similarly named MakerNote table has different values and
|
||||
must not be substituted into this formula. Main raw-IFD and SR2SubIFD values agreed in the test files.
|
||||
must not be substituted into this formula. Main raw-IFD and SR2SubIFD values agreed in the test files, and the
|
||||
`SR2SubIFD` copy is never converted.
|
||||
|
||||
## Validation
|
||||
|
||||
@@ -95,7 +209,18 @@ The distortion geometry remains aligned while every body's image framing follows
|
||||
|
||||
Tests cover signedness/length rejection, zero and malformed tables, independent 600 mm reference
|
||||
geometry, barrel framing, offset crop normalization, both TIFF byte orders, header/full consistency,
|
||||
and preservation through DNG export. No private photographs or reference TIFFs are committed.
|
||||
and preservation through DNG export. The decoder's synthetic tests (`crates/raw/src/vendor/arw_lens.rs`) add both
|
||||
byte orders and BigTIFF, exact SHORT / SSHORT types, negative samples, missing, short, long, huge and zero counts,
|
||||
wrong leading counts, values outside the file, a table past the TIFF reader's entry limit (not applied), repeated
|
||||
tags (the reader's copy kept, as before), unknown and negative settings, raw-IFD versus
|
||||
`SR2SubIFD` precedence and agreement, the correct `SR2SubIFD` tag numbers, a wrong decryption key, hostile entries
|
||||
inside the decrypted block, linear YCbCr, framing evidence (Sony's crop at x = 0 against the default crop, agreeing
|
||||
crops, padded storage, aspect crops, crop tags outside the frame), all eight EXIF orientations leaving the record
|
||||
unchanged, and neutral, out-of-range and folding tables. On a small classic-TIFF fixture in both byte orders, every
|
||||
truncation and four mutations of each byte outside the pixel data (set to 0x00 and 0xff, top and low bit flipped)
|
||||
decode or fail without a panic. `tests/sr2_heap.rs` measures the peak heap of decoding a 16 MiB file whose
|
||||
`SR2Private` directory holds eight fields aliasing the whole file: 37 kB, against 134 MB when those fields were
|
||||
decoded. No private photographs or reference TIFFs are committed.
|
||||
|
||||
## Camera coverage and regression boundary
|
||||
|
||||
@@ -104,27 +229,35 @@ model names, and every other model remain unsupported by this distortion decoder
|
||||
numbers and table lengths do not establish the scale, interpolation or crop normalization on another
|
||||
body. Support can expand after independent geometry validation; do not add model-name prefix matching.
|
||||
|
||||
The current `cargo xtask corpus --download` manifest contains 23 Sony CC0 samples across 17 bodies,
|
||||
including samples 4822 (A7R IVA / 24–105 mm) and 3989 (A9 II / 200–600 mm). All 23 files have
|
||||
pinned SHA-256 hashes and are checked with `sony_embedded_distortion_is_limited_to_validated_models`:
|
||||
The current `cargo xtask corpus --download` manifest contains 25 Sony CC0 samples across 18 bodies,
|
||||
including samples 4822 (A7R IVA / 24–105 mm) and 3989 (A9 II / 200–600 mm), each with a pinned SHA-256 that the
|
||||
download checks. `sony_embedded_distortion_is_limited_to_validated_models` checks 23 of them (the two ILCE-7CM2
|
||||
codings have their own test), and `sony_lens_records_of_the_public_samples` verifies all 25 against their pins
|
||||
(`SONY_SAMPLES` in `crates/raw/tests/corpus.rs`, itself anchored by a pinned digest and compared with the manifest)
|
||||
before decoding. **No CI job fetches the corpus**: these fixture tests run only where it has been downloaded, and
|
||||
otherwise skip each absent file. `LIGHTCRAFT_CORPUS_REQUIRED=1` makes `sony_lens_records_of_the_public_samples`
|
||||
fail on any absent pinned file instead.
|
||||
|
||||
| Camera | Corpus samples | Raw-IFD table observed | Expected correction |
|
||||
| Camera | Corpus samples | Decoded record: tables, settings (shading / CA / distortion) | Applied |
|
||||
|---|---|---|---|
|
||||
| ILCE-7RM4A | Compressed (4822) | 16 samples | One distortion warp |
|
||||
| ILCE-9M2 | Compressed (3989) | 16 samples | None |
|
||||
| ILCE-7M3 | Compressed, uncompressed | 16 samples | None |
|
||||
| ILCE-7M4 | 14-bit, lossless L/M/S | 11 samples in the 14-bit file; 16 in L/M/S | None |
|
||||
| ILCE-7RM2 | 12-bit uncompressed | 16 samples | None |
|
||||
| DSC-RX100 | One | No raw-IFD table | None |
|
||||
| DSC-RX100M3 | One | 11 samples | None |
|
||||
| ILCE-7CR | Compressed, lossless L/M | Not used; model excluded | None |
|
||||
| ILCE-7RM4 | 14-bit compressed | Not used; model excluded | None |
|
||||
| DSLR-A500, DSLR-A700, SLT-A33 | One each | Not used; model excluded | None |
|
||||
| ILCE-3500, ILCE-7S | 5600 K, Shade respectively | Not used; model excluded | None |
|
||||
| ILCE-7SM2, DSLR-A580 | 16:9 framing, one each | Not used; model excluded | None |
|
||||
| DSLR-A900 | Packed 12-bit | Not used; model excluded | None |
|
||||
| ILCE-7RM4A | Compressed (4822) | All three tables; 257 / 1 / 17 | One distortion warp |
|
||||
| ILCE-9M2 | Compressed (3989) | All three; 257 / 1 / 0 | None |
|
||||
| ILCE-7M3 | Compressed, uncompressed | All three; 257 / 1 / 0 | None |
|
||||
| ILCE-7M4 | 14-bit, lossless L/M/S | L/M/S: all three, 272 / 1 / 17; the 14-bit file (APS-C framing, 4608 × 3072): settings only, its 11-sample tables are not interpreted | None |
|
||||
| ILCE-7RM2 | 12-bit uncompressed | All three; 256 / 0 / 1 | None |
|
||||
| DSC-RX100 | One | Shading, from the `SR2SubIFD` (the raw IFD has no tables); CA and distortion in the 11-sample layout | None |
|
||||
| DSC-RX100M3 | One | Shading; CA and distortion in the 11-sample layout | None |
|
||||
| ILCE-7CR | Compressed, lossless L/M | All three; 272 / 1 / 0 | None |
|
||||
| ILCE-7RM4 | 14-bit compressed | All three; 257 / 1 / 17 | None |
|
||||
| ILCE-3500 | 5600 K | Settings only (unavailable) | None |
|
||||
| ILCE-7S | Shade | An all-zero CA table; settings unavailable | None |
|
||||
| ILCE-7SM2 | 16:9 framing | All-zero shading and CA tables; settings unavailable | None |
|
||||
| DSLR-A500, DSLR-A700, SLT-A33, DSLR-A580, DSLR-A900 | One each | No record | None |
|
||||
|
||||
These are **model-boundary regression checks, not new geometric validation of those models**. In particular,
|
||||
These are **model-boundary regression checks, not new geometric validation of those models**.
|
||||
`sony_lens_records_of_the_public_samples` also checks the exact tables, settings, provenance and framing evidence of
|
||||
the three A7CR codings and the A7R IVA sample, and that across the Sony corpus only the A7R IVA's distortion is ever
|
||||
applied. In particular,
|
||||
refusing the A7 III tables resolves the unverified corner warp identified in review. The lossless M/S
|
||||
files are linear YCbCr and remain excluded independently of the model check. Header and full decode
|
||||
must agree for every sample. Synthetic tests also verify both TIFF byte orders, exact model matching,
|
||||
|
||||
Reference in new issue
Block a user