From 3e42f2e05c5b2779e381f4117e14f53aa34d1b2f Mon Sep 17 00:00:00 2001 From: Chad Elliott Date: Sun, 11 Oct 2026 16:33:54 -0500 Subject: [PATCH 1/3] feat(testkit): #1037 tests record why they skip libtest has no skipped status and hides the output of passing tests, so a test that returns early without a GPU, a corpus file, a craft-fonts face or a model passed silently. `lightcraft-testkit` (the testkit slot the layering table already reserves, dev-dependency only, std only) adds `skip!(Need, "reason")`: it prints the note as before and, when LIGHTCRAFT_SKIP_LOG names a file, appends one JSON line (need, reason, test, binary, file:line). Plain `cargo test` is unchanged. The 61 existing skip sites use it. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 12 ++ Cargo.toml | 1 + apps/lightcraft/Cargo.toml | 4 + apps/lightcraft/src/window_gpu.rs | 4 +- crates/codecs/Cargo.toml | 2 + crates/codecs/tests/corpus.rs | 8 +- crates/engine/Cargo.toml | 2 + crates/engine/src/camera_preview.rs | 18 +- crates/engine/src/cmd/convert.rs | 2 +- crates/engine/src/export.rs | 8 +- crates/engine/src/files.rs | 4 +- crates/engine/src/tests_face_models.rs | 2 +- crates/engine/src/tests_face_recognize.rs | 2 +- crates/engine/src/tests_xmp.rs | 4 +- crates/engine/tests/preview_read.rs | 2 +- crates/faces/Cargo.toml | 2 + crates/faces/src/yunet.rs | 2 +- crates/gpu/Cargo.toml | 2 + crates/gpu/src/nn.rs | 2 +- crates/gpu/tests/equivalence.rs | 2 +- crates/gpu/tests/fallback.rs | 2 +- crates/gpu/tests/memory.rs | 2 +- crates/gpu/tests/shutdown.rs | 3 +- crates/raw/Cargo.toml | 2 + crates/raw/tests/corpus.rs | 42 ++--- crates/raw/tests/cr3_corpus.rs | 4 +- crates/raw/tests/sparse_corpus.rs | 2 +- crates/segment/Cargo.toml | 4 + crates/segment/tests/reference.rs | 2 +- crates/testkit/Cargo.toml | 12 ++ crates/testkit/src/lib.rs | 190 ++++++++++++++++++++++ crates/ui-egui/Cargo.toml | 2 + crates/ui-egui/src/i18n.rs | 4 +- crates/ui-egui/src/theme.rs | 2 +- 34 files changed, 297 insertions(+), 61 deletions(-) create mode 100644 crates/testkit/Cargo.toml create mode 100644 crates/testkit/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 5f2609a5b..43d5f119c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3081,6 +3081,7 @@ dependencies = [ "lightcraft-engine", "lightcraft-raster", "lightcraft-sysmem", + "lightcraft-testkit", "lightcraft-ui-egui", "log", "muda", @@ -3144,6 +3145,7 @@ dependencies = [ "lightcraft-geom", "lightcraft-heif", "lightcraft-raster", + "lightcraft-testkit", "moxcms 0.9.1", "png", "proptest", @@ -3239,6 +3241,7 @@ dependencies = [ "lightcraft-raw", "lightcraft-scenes", "lightcraft-segment", + "lightcraft-testkit", "lightcraft-tiff", "log", "miniz_oxide 0.8.9", @@ -3254,6 +3257,7 @@ dependencies = [ name = "lightcraft-faces" version = "0.6.0" dependencies = [ + "lightcraft-testkit", "matrixmultiply", "ndarray", "proptest", @@ -3304,6 +3308,7 @@ dependencies = [ "lightcraft-raster", "lightcraft-raw", "lightcraft-scenes", + "lightcraft-testkit", "log", "pollster 1.0.1", "wgpu", @@ -3424,6 +3429,7 @@ dependencies = [ "lightcraft-geom", "lightcraft-meta", "lightcraft-raster", + "lightcraft-testkit", "lightcraft-tiff", "miniz_oxide 0.8.9", "proptest", @@ -3454,6 +3460,7 @@ dependencies = [ "candle-core", "candle-nn", "lightcraft-fetch", + "lightcraft-testkit", "log", "serde", "serde_json", @@ -3464,6 +3471,10 @@ dependencies = [ name = "lightcraft-sysmem" version = "0.6.0" +[[package]] +name = "lightcraft-testkit" +version = "0.6.0" + [[package]] name = "lightcraft-tiff" version = "0.6.0" @@ -3493,6 +3504,7 @@ dependencies = [ "lightcraft-pipeline", "lightcraft-preview", "lightcraft-raster", + "lightcraft-testkit", "log", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 7163c757c..feaee1d54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ lightcraft-fetch = { path = "crates/fetch" } lightcraft-heif = { path = "crates/heif" } lightcraft-segment = { path = "crates/segment" } lightcraft-geocode = { path = "crates/geocode" } +lightcraft-testkit = { path = "crates/testkit" } # foundation thiserror = "2" diff --git a/apps/lightcraft/Cargo.toml b/apps/lightcraft/Cargo.toml index bbe495c7d..26122c3e6 100644 --- a/apps/lightcraft/Cargo.toml +++ b/apps/lightcraft/Cargo.toml @@ -48,3 +48,7 @@ muda = { workspace = true } objc2 = { workspace = true } objc2-app-kit = { workspace = true } objc2-foundation = { workspace = true } + +[dev-dependencies] +# tests record why they skip (`cargo xtask ci` lists the skips) +lightcraft-testkit = { workspace = true } diff --git a/apps/lightcraft/src/window_gpu.rs b/apps/lightcraft/src/window_gpu.rs index 66e5a78d7..1f84ec477 100644 --- a/apps/lightcraft/src/window_gpu.rs +++ b/apps/lightcraft/src/window_gpu.rs @@ -278,11 +278,11 @@ mod tests { ..wgpu::InstanceDescriptor::new_without_display_handle() }); let Ok(adapter) = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) else { - eprintln!("no GPU adapter: skipped"); + lightcraft_testkit::skip!(Gpu, "no GPU adapter"); return; }; let Ok((device, _queue)) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())) else { - eprintln!("no GPU device: skipped"); + lightcraft_testkit::skip!(Gpu, "no GPU device"); return; }; install_error_handler(&device); diff --git a/crates/codecs/Cargo.toml b/crates/codecs/Cargo.toml index 166fbf4dc..1ae016bc8 100644 --- a/crates/codecs/Cargo.toml +++ b/crates/codecs/Cargo.toml @@ -48,6 +48,8 @@ heif = ["dep:lightcraft-heif"] avif = ["dep:ravif", "dep:rav1e", "dep:avif-serialize"] [dev-dependencies] +# tests record why they skip (`cargo xtask ci` lists the skips) +lightcraft-testkit = { workspace = true } # JPEG XL tests read encoded samples without our colour conversion jxl-oxide = { workspace = true } proptest = { workspace = true } diff --git a/crates/codecs/tests/corpus.rs b/crates/codecs/tests/corpus.rs index d3b2ceba3..9a86bc5e2 100644 --- a/crates/codecs/tests/corpus.rs +++ b/crates/codecs/tests/corpus.rs @@ -31,7 +31,7 @@ fn files(dir: &Path) -> Vec { fn corpus_images_decode() { let dir = corpus_root().join("images"); if !dir.is_dir() { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; } let (mut ok, mut unsupported) = (0, 0); @@ -57,7 +57,7 @@ fn corpus_images_decode() { fn corpus_pngsuite() { let dir = corpus_root().join("pngsuite"); if !dir.is_dir() { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; } let mut n = 0; @@ -83,7 +83,7 @@ fn corpus_pngsuite() { fn corpus_raw_is_routed() { let dir = corpus_root().join("raw"); if !dir.is_dir() { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; } for p in files(&dir) { @@ -103,7 +103,7 @@ fn system_icc_profiles() { use lightcraft_codecs::icc::{IccColorModel, IccKind, parse}; let dir = Path::new("/System/Library/ColorSync/Profiles"); if !dir.is_dir() { - eprintln!("skip: no system ICC profiles"); + lightcraft_testkit::skip!(System, "no system ICC profiles"); return; } let expect = [ diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 2229c42a9..203135867 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -51,6 +51,8 @@ denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime", "lightcraft-g rawnind-model = ["denoise", "lightcraft-denoise/rawnind-model", "lightcraft-denoise-core/rawnind-model"] [dev-dependencies] +# tests record why they skip (`cargo xtask ci` lists the skips) +lightcraft-testkit = { workspace = true } # HDR JPEG XL tests read the file's colour encoding jxl-oxide = { workspace = true } # Lossless synthetic HEIC files (the import test; decoded when the codecs' `heif` feature is on). diff --git a/crates/engine/src/camera_preview.rs b/crates/engine/src/camera_preview.rs index 858f3b4cf..758714a75 100644 --- a/crates/engine/src/camera_preview.rs +++ b/crates/engine/src/camera_preview.rs @@ -1828,7 +1828,7 @@ mod tests { .join("raw"); for name in ["arw-sony-a7m4-14bit.arw", "cr2-canon-5d3.cr2", "nef-nikon-d5100-lossless.nef", "cr2-canon-7d.cr2", "dng-canon-5d3-16bit.dng"] { let Ok(bytes) = std::fs::read(dir.join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; }; let mut raw = lightcraft_raw::decode(&bytes).unwrap(); @@ -1929,13 +1929,13 @@ mod tests { .unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")) .join("raw/cr3-canon-r100-raw.cr3"); let Ok(bytes) = std::fs::read(&path) else { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); return; }; let raw = match lightcraft_raw::decode(&bytes) { Ok(raw) => raw, Err(lightcraft_raw::RawError::Unsupported(why)) => { - eprintln!("skip CR3 sensor decoder: {why}"); + lightcraft_testkit::skip!(Feature, "CR3 sensor decoder: {why}"); return; } Err(error) => panic!("CR3 corpus failed to decode: {error}"), @@ -1964,7 +1964,7 @@ mod tests { let bytes = match std::fs::read(&path) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); return; } Err(e) => panic!("{}: {e}", path.display()), @@ -1995,7 +1995,7 @@ mod tests { .unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")) .join("raw/nef-nikon-d7500-lossless14.nef"); let Ok(bytes) = std::fs::read(&path) else { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); return; }; let (_, info) = crate::files::load_bytes(&bytes, 400).unwrap(); @@ -2015,7 +2015,7 @@ mod tests { let bytes = match std::fs::read(dir.join(name)) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; } Err(e) => panic!("{name}: {e}"), @@ -2036,7 +2036,7 @@ mod tests { .unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")) .join("raw/rw2-panasonic-fz1000m2-4x3.rw2"); let Ok(bytes) = std::fs::read(&path) else { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); return; }; let (img, info) = crate::files::load_bytes(&bytes, 400).unwrap(); @@ -2055,7 +2055,7 @@ mod tests { .unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")) .join("raw/arw-sony-a7rm2-12bit-uncompressed.arw"); let Ok(bytes) = std::fs::read(&path) else { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); return; }; let (_, info) = crate::files::load_bytes(&bytes, 400).unwrap(); @@ -2213,7 +2213,7 @@ mod tests { let bytes = match std::fs::read(&path) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); continue; } Err(e) => panic!("{}: {e}", path.display()), diff --git a/crates/engine/src/cmd/convert.rs b/crates/engine/src/cmd/convert.rs index 4d7f13aeb..1eaec749d 100644 --- a/crates/engine/src/cmd/convert.rs +++ b/crates/engine/src/cmd/convert.rs @@ -494,7 +494,7 @@ mod tests { .unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")); let f = root.join("raw/rw2-panasonic-gx80.rw2"); if !f.exists() { - eprintln!("skip: {} absent", f.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", f.display()); return; } let mut s = Session::new().with_fs(); diff --git a/crates/engine/src/export.rs b/crates/engine/src/export.rs index f356d6e4d..4b91e5b78 100644 --- a/crates/engine/src/export.rs +++ b/crates/engine/src/export.rs @@ -2414,7 +2414,7 @@ mod tests { fn biz_ud_vertical_watermarks_place_punctuation_at_the_top_right() { use ab_glyph::Font; let Some(entry) = crate::fonts::CRAFT_FONTS.iter().find(|f| f.family == "BIZ UDMincho") else { - eprintln!("skipped: built without BIZ UDMincho from craft-fonts"); + lightcraft_testkit::skip!(Fonts, "built without BIZ UDMincho from craft-fonts"); return; }; let font = ab_glyph::FontRef::try_from_slice(entry.bytes).unwrap(); @@ -2486,7 +2486,7 @@ mod tests { compare(composed, decomposed, crate::fonts::CRAFT_FONTS); } } else { - eprintln!("skipped Japanese comparisons: built without BIZ UDMincho from craft-fonts"); + lightcraft_testkit::skip!(Fonts, "Japanese comparisons: built without BIZ UDMincho from craft-fonts"); } } @@ -2510,7 +2510,7 @@ mod tests { use ab_glyph::Font; let fonts = watermark_fonts(crate::fonts::CRAFT_FONTS); if fonts.len() < 2 { - eprintln!("skipped: built without CRAFT_FONTS_DIR, so there is no Japanese watermark face"); + lightcraft_testkit::skip!(Fonts, "built without CRAFT_FONTS_DIR, so there is no Japanese watermark face"); return; } for c in "日本語の文字".chars() { @@ -3090,7 +3090,7 @@ mod tests { #[test] fn a_batch_reports_a_gpu_fallback_on_the_file_it_hit() { if !lightcraft_gpu::available() { - eprintln!("skipped: no GPU adapter ({:?})", lightcraft_gpu::unavailable_reason()); + lightcraft_testkit::skip!(Gpu, "no GPU adapter ({})", lightcraft_gpu::unavailable_reason().unwrap_or_default()); return; } let mut s = crate::Session::with_demo(); diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs index c48aaff77..2ab7c21f0 100644 --- a/crates/engine/src/files.rs +++ b/crates/engine/src/files.rs @@ -1133,7 +1133,7 @@ mod tests { .unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")) .join("raw"); let Ok(gh1) = std::fs::read(dir.join("rw2-panasonic-gh1.rw2")) else { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; }; assert_eq!(probe_bytes("gh1.rw2", &gh1).unwrap().embedded_lens, None, "correction off: nothing to apply"); @@ -1484,7 +1484,7 @@ mod tests { .unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")) .join("raw"); let Ok(bytes) = std::fs::read(dir.join("arw-sony-a7rm4-14bit-compressed.arw")) else { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; }; let (img, _) = load_bytes(&bytes, 1600).unwrap(); diff --git a/crates/engine/src/tests_face_models.rs b/crates/engine/src/tests_face_models.rs index f1c27c849..49532c471 100644 --- a/crates/engine/src/tests_face_models.rs +++ b/crates/engine/src/tests_face_models.rs @@ -179,7 +179,7 @@ fn hostile_ids_unsupported_models_and_odd_folders_are_handled() { /// Settings > Faces downloads; nothing is committed). Without it those tests say so and pass. fn demo_with_yunet() -> Option { let Some(file) = std::env::var_os("LC_YUNET_MODEL") else { - eprintln!("LC_YUNET_MODEL is not set: skipping"); + lightcraft_testkit::skip!(Model, "LC_YUNET_MODEL is not set"); return None; }; let dir = temp("yunet"); diff --git a/crates/engine/src/tests_face_recognize.rs b/crates/engine/src/tests_face_recognize.rs index aabcb03c3..5aab8f9a4 100644 --- a/crates/engine/src/tests_face_recognize.rs +++ b/crates/engine/src/tests_face_recognize.rs @@ -34,7 +34,7 @@ pub(crate) fn setup(d: &std::path::Path, dim: u64) -> (Session, String) { /// committed) and says whether it did: the parts of a test that need to find faces run only then. pub(crate) fn install_yunet(s: &mut Session) -> bool { let Some(file) = std::env::var_os("LC_YUNET_MODEL") else { - eprintln!("LC_YUNET_MODEL is not set: skipping the face-finding part"); + lightcraft_testkit::skip!(Model, "LC_YUNET_MODEL is not set, so the face-finding part is not checked"); return false; }; let r = s.execute("faces.models.install", &json!({"path": file.to_string_lossy(), "acknowledged": true})); diff --git a/crates/engine/src/tests_xmp.rs b/crates/engine/src/tests_xmp.rs index c768078ce..9c2b40849 100644 --- a/crates/engine/src/tests_xmp.rs +++ b/crates/engine/src/tests_xmp.rs @@ -581,7 +581,7 @@ fn convert_raw_to_dng() { let r = s.execute("photo.convertToDng", &json!({})).unwrap(); assert_eq!((r["converted"].as_array().unwrap().len(), r["skipped"].as_array().unwrap().len()), (0, 1)); if !corpus.exists() { - eprintln!("skip: {} absent", corpus.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", corpus.display()); return; } let nef = dir.join("shot.nef"); @@ -843,7 +843,7 @@ fn dng_export_compression_choices() { .unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")) .join("raw/nef-nikon-d5100-uncompressed.nef"); if !corpus.exists() { - eprintln!("skip: {} absent", corpus.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", corpus.display()); return; } let mut s = Session::new().with_fs(); diff --git a/crates/engine/tests/preview_read.rs b/crates/engine/tests/preview_read.rs index 11ffec44e..34aae9591 100644 --- a/crates/engine/tests/preview_read.rs +++ b/crates/engine/tests/preview_read.rs @@ -13,7 +13,7 @@ use std::time::Instant; #[ignore = "needs LC_PREVIEW_RAW pointing at a real raw file"] fn a_preview_reads_only_the_header_and_the_preview() { let Ok(path) = std::env::var("LC_PREVIEW_RAW") else { - eprintln!("skipped: set LC_PREVIEW_RAW="); + lightcraft_testkit::skip!(Input, "set LC_PREVIEW_RAW="); return; }; let file_len = std::fs::metadata(&path).expect("the file exists").len(); diff --git a/crates/faces/Cargo.toml b/crates/faces/Cargo.toml index 112967cee..78ea0b798 100644 --- a/crates/faces/Cargo.toml +++ b/crates/faces/Cargo.toml @@ -22,4 +22,6 @@ sha2 = { workspace = true } thiserror = { workspace = true } [dev-dependencies] +# tests record why they skip (`cargo xtask ci` lists the skips) +lightcraft-testkit = { workspace = true } proptest = { workspace = true } diff --git a/crates/faces/src/yunet.rs b/crates/faces/src/yunet.rs index b77f318bf..56f5d6474 100644 --- a/crates/faces/src/yunet.rs +++ b/crates/faces/src/yunet.rs @@ -263,7 +263,7 @@ mod tests { fn detector() -> Option { let Some(bytes) = model() else { - eprintln!("LC_YUNET_MODEL is not set: skipping"); + lightcraft_testkit::skip!(Model, "LC_YUNET_MODEL is not set"); return None; }; Some(Detector::new(&bytes).unwrap()) diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml index dc348ee88..5547c548d 100644 --- a/crates/gpu/Cargo.toml +++ b/crates/gpu/Cargo.toml @@ -29,5 +29,7 @@ default = [] denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime"] [dev-dependencies] +# tests record why they skip (`cargo xtask ci` lists the skips) +lightcraft-testkit = { workspace = true } lightcraft-scenes = { workspace = true } lightcraft-raw = { workspace = true } diff --git a/crates/gpu/src/nn.rs b/crates/gpu/src/nn.rs index 4f078ff93..5872bfaa9 100644 --- a/crates/gpu/src/nn.rs +++ b/crates/gpu/src/nn.rs @@ -907,7 +907,7 @@ mod tests { match runner(net, tile) { Ok(r) => Some(r), Err(e) => { - eprintln!("skipped: no GPU runner here ({e})"); + lightcraft_testkit::skip!(Gpu, "no GPU runner here ({e})"); None } } diff --git a/crates/gpu/tests/equivalence.rs b/crates/gpu/tests/equivalence.rs index 96771a971..16a9263c1 100644 --- a/crates/gpu/tests/equivalence.rs +++ b/crates/gpu/tests/equivalence.rs @@ -19,7 +19,7 @@ const MAX_LSB: u8 = 3; fn gpu() -> bool { let ok = lightcraft_gpu::available(); if !ok { - eprintln!("skipped: no GPU adapter"); + lightcraft_testkit::skip!(Gpu, "no GPU adapter"); } ok } diff --git a/crates/gpu/tests/fallback.rs b/crates/gpu/tests/fallback.rs index fe2ecad5e..05347b446 100644 --- a/crates/gpu/tests/fallback.rs +++ b/crates/gpu/tests/fallback.rs @@ -38,7 +38,7 @@ fn masked(s: &mut DevelopSettings, k: usize) { #[test] fn failed_gpu_renders_fall_back_with_a_reason() { if !lightcraft_gpu::available() { - eprintln!("skipped: no GPU adapter ({:?})", lightcraft_gpu::unavailable_reason()); + lightcraft_testkit::skip!(Gpu, "no GPU adapter ({})", lightcraft_gpu::unavailable_reason().unwrap_or_default()); return; } assert_eq!(lightcraft_gpu::unavailable_reason(), None); diff --git a/crates/gpu/tests/memory.rs b/crates/gpu/tests/memory.rs index 2ef958cb1..25784f001 100644 --- a/crates/gpu/tests/memory.rs +++ b/crates/gpu/tests/memory.rs @@ -9,7 +9,7 @@ use lightcraft_pipeline::{RenderRequest, SourceInfo, StageCache}; #[test] fn stages_are_counted_and_the_pool_is_bounded_and_trimmed() { if !lightcraft_gpu::available() { - eprintln!("skipped: no GPU adapter"); + lightcraft_testkit::skip!(Gpu, "no GPU adapter"); return; } let limit = 64u64 << 20; diff --git a/crates/gpu/tests/shutdown.rs b/crates/gpu/tests/shutdown.rs index b827a0711..a8d5cbd23 100644 --- a/crates/gpu/tests/shutdown.rs +++ b/crates/gpu/tests/shutdown.rs @@ -21,7 +21,8 @@ fn nothing_renders_on_the_gpu_once_the_process_is_ending() { let req = RenderRequest::fit(w, h); let has_gpu = lightcraft_gpu::available(); if !has_gpu { - eprintln!("no GPU adapter ({:?}): only the CPU side is checked", lightcraft_gpu::unavailable_reason()); + let why = lightcraft_gpu::unavailable_reason().unwrap_or_default(); + lightcraft_testkit::skip!(Gpu, "no GPU adapter ({why}), so only the CPU side is checked"); } assert!(!lightcraft_gpu::shutting_down()); assert!(lightcraft_gpu::wait_idle(Duration::ZERO), "nothing is in flight yet"); diff --git a/crates/raw/Cargo.toml b/crates/raw/Cargo.toml index 2ca9ef246..84f9fa91a 100644 --- a/crates/raw/Cargo.toml +++ b/crates/raw/Cargo.toml @@ -28,6 +28,8 @@ default = ["jxl"] jxl = ["dep:jxl-oxide"] [dev-dependencies] +# tests record why they skip (`cargo xtask ci` lists the skips) +lightcraft-testkit = { workspace = true } jpeg-encoder = { workspace = true } proptest = { workspace = true } zune-jpegxl = { workspace = true } diff --git a/crates/raw/tests/corpus.rs b/crates/raw/tests/corpus.rs index 5ca4777b7..cf5bc5290 100644 --- a/crates/raw/tests/corpus.rs +++ b/crates/raw/tests/corpus.rs @@ -25,7 +25,7 @@ const KNOWN_UNSUPPORTED: &[&str] = &[ fn corpus_raw_decodes() { let dir = corpus_root().join("raw"); let Ok(rd) = std::fs::read_dir(&dir) else { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; }; let mut paths: Vec = rd.flatten().map(|e| e.path()).filter(|p| p.is_file()).collect(); @@ -159,7 +159,7 @@ fn corpus_cr2_cfa_patterns() { let mut seen = 0; for (name, want) in cases { let Ok(bytes) = std::fs::read(dir.join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; }; let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); @@ -189,7 +189,7 @@ fn corpus_cr2_as_shot_white_balance() { "cr2-canon-80d.cr2", ] { let Ok(bytes) = std::fs::read(dir.join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; }; let wb = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")).wb_multipliers.unwrap_or_else(|| panic!("{name}: no as-shot WB")); @@ -206,7 +206,7 @@ fn corpus_nef_compressed_matches_uncompressed() { let dir = corpus_root().join("raw"); let (Ok(a), Ok(b)) = (std::fs::read(dir.join("nef-nikon-d5100-lossless.nef")), std::fs::read(dir.join("nef-nikon-d5100-uncompressed.nef"))) else { - eprintln!("skip: D5100 NEF pair absent"); + lightcraft_testkit::skip!(Corpus, "D5100 NEF pair absent"); return; }; let (a, b) = (decode(&a).unwrap(), decode(&b).unwrap()); @@ -231,7 +231,7 @@ fn corpus_nef_compressed_matches_uncompressed() { fn corpus_nef_high_efficiency_is_named() { let dir = corpus_root().join("raw"); let (Ok(he), Ok(lossless)) = (std::fs::read(dir.join("nef-nikon-zf-he.nef")), std::fs::read(dir.join("nef-nikon-zf-lossless.nef"))) else { - eprintln!("skip: Z f samples absent"); + lightcraft_testkit::skip!(Corpus, "Z f samples absent"); return; }; for result in [decode(&he).map(|_| ()), probe_info(&he).map(|_| ())] { @@ -248,7 +248,7 @@ fn corpus_nef_12_bit_black_level_matches_14_bit() { let dir = corpus_root().join("raw"); let (Ok(a), Ok(b)) = (std::fs::read(dir.join("nef-nikon-d7500-lossless12.nef")), std::fs::read(dir.join("nef-nikon-d7500-lossless14.nef"))) else { - eprintln!("skip: D7500 NEF pair absent"); + lightcraft_testkit::skip!(Corpus, "D7500 NEF pair absent"); return; }; let (a, b) = (decode(&a).unwrap(), decode(&b).unwrap()); @@ -278,7 +278,7 @@ fn corpus_nef_12_bit_black_level_matches_14_bit() { fn corpus_dngs_carry_profile_looks() { let dir = corpus_root().join("raw"); let Ok(rd) = std::fs::read_dir(&dir) else { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; }; let mut seen = 0; @@ -327,7 +327,7 @@ fn corpus_dngs_carry_profile_looks() { fn corpus_proraw_sky_matte() { let path = corpus_root().join("raw/dng-apple-iphone12pro-proraw.dng"); let Ok(bytes) = std::fs::read(&path) else { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); return; }; let t0 = Instant::now(); @@ -365,7 +365,7 @@ fn corpus_sony_pre2017_colour_metadata() { let mut seen = 0; for (name, black, [r, b], (cw, ch)) in cases { let Ok(bytes) = std::fs::read(dir.join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; }; let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); @@ -397,7 +397,7 @@ fn corpus_sony_a7cr_codings() { let bytes = match std::fs::read(dir.join(name)) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; } Err(e) => panic!("{name}: {e}"), @@ -464,7 +464,7 @@ fn corpus_sony_a7cm2_codings() { let bytes = match std::fs::read(dir.join(name)) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; } Err(e) => panic!("{name}: {e}"), @@ -509,7 +509,7 @@ fn corpus_sony_black_levels_sit_at_the_data_floor() { let bytes = match std::fs::read(dir.join(name)) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; } Err(e) => panic!("{name}: {e}"), @@ -567,7 +567,7 @@ fn corpus_sony_sr2_white_balance() { let mut seen = 0; for (name, [r, b]) in cases { let Ok(bytes) = std::fs::read(dir.join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; }; let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); @@ -585,7 +585,7 @@ fn corpus_sony_sr2_white_balance() { fn corpus_sony_a900_packed12() { let name = "arw-sony-a900-packed12.arw"; let Ok(bytes) = std::fs::read(corpus_root().join("raw").join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); return; }; let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); @@ -607,7 +607,7 @@ fn corpus_sony_16x9_crops() { let bytes = match std::fs::read(dir.join(name)) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; } Err(e) => panic!("{name}: {e}"), @@ -727,7 +727,7 @@ fn corpus_panasonic_encodings() { let mut seen = 0; for (name, bits, black, (cw, ch), cfa) in cases { let Ok(bytes) = std::fs::read(dir.join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; }; let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); @@ -777,7 +777,7 @@ fn corpus_orf_cfa_patterns() { let mut seen = 0; for (name, cfa) in cases { let Ok(bytes) = std::fs::read(dir.join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; }; let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); @@ -812,7 +812,7 @@ fn corpus_samsung_srw() { let mut seen = 0; for (name, bits, active, crop, cfa, whole) in cases { let Ok(bytes) = std::fs::read(dir.join(name)) else { - eprintln!("skip: {name} absent"); + lightcraft_testkit::skip!(Corpus, "{name} absent"); continue; }; let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); @@ -941,7 +941,7 @@ fn corpus_sony_arw6_samples() { fn corpus_raws_keep_their_container_and_are_not_thumbnail_shells() { let dir = corpus_root().join("raw"); let Ok(rd) = std::fs::read_dir(&dir) else { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; }; let mut checked = 0; @@ -1004,7 +1004,7 @@ fn sony_embedded_distortion_is_limited_to_validated_models() { ] { let path = root.join(name); if !path.exists() { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); continue; } let bytes = std::fs::read(&path).unwrap(); @@ -1026,7 +1026,7 @@ fn corpus_kodak_dcs520c() { let name = "2573_Kodak - DCS520C - 12bit (3_2).TIF"; let candidates = [corpus_root().join("raw").join("kodak-dcs520c.tif"), Path::new("P:/raw-pixls/Kodak/DCS520C").join(name)]; let Some(bytes) = candidates.iter().find_map(|p| std::fs::read(p).ok()) else { - eprintln!("skip: DCS520C sample absent"); + lightcraft_testkit::skip!(Corpus, "DCS520C sample absent"); return; }; assert_eq!(probe(&bytes), Some(RawFormat::KodakDcs)); diff --git a/crates/raw/tests/cr3_corpus.rs b/crates/raw/tests/cr3_corpus.rs index ebf669307..06f7150a8 100644 --- a/crates/raw/tests/cr3_corpus.rs +++ b/crates/raw/tests/cr3_corpus.rs @@ -22,7 +22,7 @@ fn check(name: &str, hash: u64, sensor: (usize, usize), developed: (usize, usize fn check_against(name: &str, hash: u64, sensor: (usize, usize), developed: Option<(usize, usize)>, reference: &str) { let path = corpus().join("raw").join(name); let Ok(bytes) = std::fs::read(&path) else { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); return; }; let raw = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); @@ -118,7 +118,7 @@ fn canon_m50_craw_crop_is_the_shot_aspect() { // which is (12, 12) inside the valid sensor area. let path = corpus().join("raw").join("cr3-canon-m50-craw.cr3"); let Ok(bytes) = std::fs::read(&path) else { - eprintln!("skip: {} absent", path.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", path.display()); return; }; let raw = decode(&bytes).unwrap(); diff --git a/crates/raw/tests/sparse_corpus.rs b/crates/raw/tests/sparse_corpus.rs index 3ab735299..4d9112bfb 100644 --- a/crates/raw/tests/sparse_corpus.rs +++ b/crates/raw/tests/sparse_corpus.rs @@ -58,7 +58,7 @@ fn partial_reads_describe_raws_as_whole_files_do() { .filter(|p| p.is_file() && !p.extension().is_some_and(|e| e == "part" || e == "txt" || e == "md")) .collect(); if paths.is_empty() { - eprintln!("skip: {} absent", dir.display()); + lightcraft_testkit::skip!(Corpus, "{} absent", dir.display()); return; } paths.sort(); diff --git a/crates/segment/Cargo.toml b/crates/segment/Cargo.toml index 11dff5c77..b4614a81d 100644 --- a/crates/segment/Cargo.toml +++ b/crates/segment/Cargo.toml @@ -23,3 +23,7 @@ lightcraft-fetch = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] candle-core = { workspace = true, features = ["metal"] } candle-nn = { workspace = true, features = ["metal"] } + +[dev-dependencies] +# tests record why they skip (`cargo xtask ci` lists the skips) +lightcraft-testkit = { workspace = true } diff --git a/crates/segment/tests/reference.rs b/crates/segment/tests/reference.rs index a73fcaa7c..315d36cd1 100644 --- a/crates/segment/tests/reference.rs +++ b/crates/segment/tests/reference.rs @@ -38,7 +38,7 @@ fn mask_disagreement(a: &[f32], b: &[f32]) -> f32 { #[test] fn matches_reference() { let Some((dir, r)) = setup() else { - eprintln!("skipped: set LIGHTCRAFT_SAM3_DIR and LIGHTCRAFT_SAM3_REF"); + lightcraft_testkit::skip!(Model, "set LIGHTCRAFT_SAM3_DIR and LIGHTCRAFT_SAM3_REF"); return; }; let mut model = Sam3::load(&dir).unwrap(); diff --git a/crates/testkit/Cargo.toml b/crates/testkit/Cargo.toml new file mode 100644 index 000000000..cefa1ef5c --- /dev/null +++ b/crates/testkit/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "lightcraft-testkit" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "LightCraft test tooling (dev-dependency only): a test that can't run here records why, so `cargo xtask ci` reports it" +publish = false + +[lints] +workspace = true + +[dependencies] diff --git a/crates/testkit/src/lib.rs b/crates/testkit/src/lib.rs new file mode 100644 index 000000000..7a2322245 --- /dev/null +++ b/crates/testkit/src/lib.rs @@ -0,0 +1,190 @@ +//! Test tooling shared by the workspace's tests. A dev-dependency only: `cargo xtask layers` +//! refuses it as a normal dependency. Std only, so every crate can use it at no build cost. +//! +//! **Skips.** libtest has no "skipped" status and hides the output of passing tests, so a test that +//! needs a GPU, a corpus file, the craft-fonts faces or a model, and returns early without them, +//! passes silently: "all steps passed" could mean the GPU tests never ran. Such a test calls +//! [`skip!`] before it returns: +//! +//! ```no_run +//! # let gpu_available = false; +//! if !gpu_available { +//! lightcraft_testkit::skip!(Gpu, "no GPU adapter"); +//! return; +//! } +//! ``` +//! +//! It prints the note to stderr as before and, when the [`SKIP_LOG_ENV`] variable names a file +//! (`cargo xtask ci` sets it), appends one JSON line there: +//! `{"need":"gpu","reason":"no GPU adapter","test":"camera_tone","binary":"equivalence","location":"crates/gpu/tests/equivalence.rs:22"}`. +//! `cargo xtask ci` lists the skips by need and reason in its summary and in +//! `target/ci/report.json`. A test that runs part of its checks (e.g. only the CPU half) records +//! the part it could not run the same way. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::fmt::Write as _; +use std::io::Write as _; +use std::path::Path; + +/// The environment variable naming the file skips are appended to (JSON lines). +pub const SKIP_LOG_ENV: &str = "LIGHTCRAFT_SKIP_LOG"; + +/// What a skipped test needed and didn't find. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Need { + /// A usable GPU adapter (`lightcraft_gpu::available()`). + Gpu, + /// A file from the gitignored `corpus/` (`cargo xtask corpus --download`) or a private corpus. + Corpus, + /// Faces from craft-fonts, embedded only in builds with `CRAFT_FONTS_DIR`. + Fonts, + /// A model file named by an environment variable (SAM 3, YuNet, …). + Model, + /// An input file named by an environment variable (opt-in tests). + Input, + /// Something the operating system provides (system ICC profiles, …). + System, + /// Support this build doesn't have yet (a decoder that refuses a file variant, an optional + /// cargo feature that is off). + Feature, +} + +impl Need { + /// The name in the skip log and the CI summary. + pub fn as_str(self) -> &'static str { + match self { + Need::Gpu => "gpu", + Need::Corpus => "corpus", + Need::Fonts => "fonts", + Need::Model => "model", + Need::Input => "input", + Need::System => "system", + Need::Feature => "feature", + } + } +} + +/// Record that this test (or part of it) can't run here: `skip!(Gpu, "no GPU adapter ({why})")`. +/// The first argument names a [`Need`] variant; the rest is a `format!` message. Doesn't return +/// from the test: follow it with `return` (or `continue`) as the test needs. +#[macro_export] +macro_rules! skip { + ($need:ident, $($arg:tt)+) => { + $crate::record($crate::Need::$need, &::std::format!($($arg)+), ::std::file!(), ::std::line!()) + }; +} + +/// What [`skip!`] calls: print the note and append it to the skip log, if one is set. Never fails +/// the test: a log that can't be written loses the entry, not the run. +pub fn record(need: Need, reason: &str, file: &str, line: u32) { + eprintln!("skipped ({}): {reason}", need.as_str()); + let log = std::env::var_os(SKIP_LOG_ENV).filter(|p| !p.is_empty()); + if let Some(log) = log { + let thread = std::thread::current(); + let binary = std::env::current_exe().ok().and_then(|p| p.file_stem().map(|s| binary_name(&s.to_string_lossy()).to_owned())); + let entry = entry(need, reason, thread.name().unwrap_or(""), binary.as_deref().unwrap_or(""), file, line); + append(Path::new(&log), &entry); + } +} + +/// Append one line in a single write, so lines from parallel tests don't interleave. +fn append(path: &Path, line: &str) { + let file = std::fs::OpenOptions::new().create(true).append(true).open(path); + if let Err(e) = file.and_then(|mut f| f.write_all(line.as_bytes())) { + eprintln!("{SKIP_LOG_ENV}: can't append to {}: {e}", path.display()); + } +} + +/// A test binary's name without the hash cargo appends (`equivalence-0123abcd` → `equivalence`). +fn binary_name(stem: &str) -> &str { + match stem.rsplit_once('-') { + Some((name, hash)) if !name.is_empty() && hash.len() == 16 && hash.bytes().all(|b| b.is_ascii_hexdigit()) => name, + _ => stem, + } +} + +/// One skip-log line (JSON, newline-terminated). +fn entry(need: Need, reason: &str, test: &str, binary: &str, file: &str, line: u32) -> String { + let mut s = String::from("{"); + for (i, (key, value)) in [("need", need.as_str()), ("reason", reason), ("test", test), ("binary", binary)].into_iter().enumerate() { + if i > 0 { + s.push(','); + } + let _ = write!(s, "\"{key}\":\"{}\"", escape(value)); + } + let _ = writeln!(s, ",\"location\":\"{}:{line}\"}}", escape(&file.replace('\\', "/"))); + s +} + +/// A JSON string's contents. +fn escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if u32::from(c) < 0x20 => { + let _ = write!(out, "\\u{:04x}", u32::from(c)); + } + c => out.push(c), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_entry_is_one_json_line_with_its_text_escaped() { + let line = entry(Need::Corpus, "skip: C:\\corpus\\raw \"absent\"\n\u{1}", "corpus::decodes", "corpus", "crates\\raw\\tests\\corpus.rs", 28); + assert!(line.ends_with('\n')); + assert_eq!(line.matches('\n').count(), 1, "{line}"); + assert_eq!( + line, + "{\"need\":\"corpus\",\"reason\":\"skip: C:\\\\corpus\\\\raw \\\"absent\\\"\\n\\u0001\",\"test\":\"corpus::decodes\",\ + \"binary\":\"corpus\",\"location\":\"crates/raw/tests/corpus.rs:28\"}\n" + ); + } + + #[test] + fn binary_names_lose_cargos_hash_only() { + assert_eq!(binary_name("equivalence-0f3a9c2b1d4e5f60"), "equivalence"); + assert_eq!(binary_name("lightcraft_engine-0f3a9c2b1d4e5f60"), "lightcraft_engine"); + assert_eq!(binary_name("sparse_corpus"), "sparse_corpus"); + assert_eq!(binary_name("cr3-corpus"), "cr3-corpus"); + assert_eq!(binary_name("-0f3a9c2b1d4e5f60"), "-0f3a9c2b1d4e5f60"); + assert_eq!(binary_name("tool-cafe"), "tool-cafe"); + } + + #[test] + fn appended_lines_accumulate() { + let dir = std::env::temp_dir().join(format!("lightcraft-testkit-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let log = dir.join("skips.jsonl"); + let _ = std::fs::remove_file(&log); + let threads: Vec<_> = (0..8) + .map(|i| { + let log = log.clone(); + std::thread::spawn(move || append(&log, &entry(Need::Gpu, &format!("no GPU {i}"), "t", "b", "f.rs", i))) + }) + .collect(); + for t in threads { + t.join().unwrap(); + } + let text = std::fs::read_to_string(&log).unwrap(); + assert_eq!(text.lines().count(), 8); + assert!(text.lines().all(|l| l.starts_with("{\"need\":\"gpu\"") && l.ends_with('}'))); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_missing_log_directory_does_not_fail_the_test() { + append(Path::new("/nonexistent-lightcraft-dir/skips.jsonl"), "{}\n"); + } +} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 74bc80058..38322d834 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -27,6 +27,8 @@ serde_json = { workspace = true } log = { workspace = true } [dev-dependencies] +# tests record why they skip (`cargo xtask ci` lists the skips) +lightcraft-testkit = { workspace = true } egui_kittest = { workspace = true, features = ["wgpu", "snapshot"] } lightcraft-denoise = { workspace = true, features = ["runtime"] } lightcraft-codecs = { workspace = true } diff --git a/crates/ui-egui/src/i18n.rs b/crates/ui-egui/src/i18n.rs index 109accfa9..c040492a3 100644 --- a/crates/ui-egui/src/i18n.rs +++ b/crates/ui-egui/src/i18n.rs @@ -597,7 +597,7 @@ mod tests { lightcraft_engine::CRAFT_FONTS.iter().filter(|font| font.serves(language.script())).map(|font| font.family).collect(); faces.dedup(); if faces.is_empty() { - eprintln!("skipped {}: built without a craft-fonts face for {}", language.code(), language.script()); + lightcraft_testkit::skip!(Fonts, "{}: built without a craft-fonts face for {}", language.code(), language.script()); continue; } ctx.fonts_mut(|fonts| { @@ -1200,7 +1200,7 @@ mod tests { let samples = [(Locale::Ja, "日本語の文字"), (Locale::ZhHans, "简体中文字"), (Locale::ZhHant, "繁體中文字")]; for (language, sample) in samples { if !lightcraft_engine::CRAFT_FONTS.iter().any(|font| font.serves(language.script())) { - eprintln!("skipped {}: built without a craft-fonts face for {}", language.code(), language.script()); + lightcraft_testkit::skip!(Fonts, "{}: built without a craft-fonts face for {}", language.code(), language.script()); continue; } set_language(language); diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index a87a196ae..30a9823d1 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -415,7 +415,7 @@ mod tests { #[test] fn thai_face_is_a_fallback_in_every_ui_family_when_craft_fonts_carry_one() { let Some(thai) = lightcraft_engine::CRAFT_FONTS.iter().find(|f| f.covers("Thai")) else { - eprintln!("no Thai face in craft-fonts: nothing to check"); + lightcraft_testkit::skip!(Fonts, "no Thai face in craft-fonts"); return; }; set_language(Locale::En); From 82045f4d2cd0c99b3763c406a10cd9caf9295e20 Mon Sep 17 00:00:00 2001 From: Chad Elliott Date: Sun, 11 Oct 2026 16:33:54 -0500 Subject: [PATCH 2/3] feat(xtask): #1037 ci reports the machine, GPU adapters and skipped tests `cargo xtask ci` points each test step at target/ci/skips-.jsonl, then probes the machine: OS, CPUs, RAM, toolchain, corpus and craft-fonts, and the GPU through the ignored `gpu_probe` test of lightcraft-gpu (run right after the workspace tests with --workspace, so it reuses their build; xtask stays without wgpu). The run ends with the skips grouped by need and reason and writes target/ci/report.json (schema 1: commit, dirty flag, machine, capabilities, step results and times, skips) and target/ci/summary.md for a PR (`--summary-md` prints it). `--hardware` also runs the lightcraft-gpu tests once per backend that has a hardware adapter. Nothing is uploaded; reasons are cut free of the workspace root and home directory. Co-Authored-By: Claude Opus 5.5 --- crates/gpu/tests/gpu_probe.rs | 87 +++++ xtask/src/ci_report.rs | 599 ++++++++++++++++++++++++++++++++++ xtask/src/main.rs | 194 +++++++++-- 3 files changed, 849 insertions(+), 31 deletions(-) create mode 100644 crates/gpu/tests/gpu_probe.rs create mode 100644 xtask/src/ci_report.rs diff --git a/crates/gpu/tests/gpu_probe.rs b/crates/gpu/tests/gpu_probe.rs new file mode 100644 index 000000000..e006fbc80 --- /dev/null +++ b/crates/gpu/tests/gpu_probe.rs @@ -0,0 +1,87 @@ +//! What GPU this machine offers the develop pipeline, as one JSON line: the capability probe of +//! `cargo xtask ci`, which is std + serde_json and has no wgpu of its own. Ignored in normal runs; +//! xtask runs it after the workspace tests, so it costs no extra build: +//! `cargo test --workspace --test gpu_probe -- --ignored --nocapture`. +//! +//! - `LIGHTCRAFT_GPU_PROBE` unset: the compute device the pipeline picks (`available()`, +//! `adapter_name()`, `unavailable_reason()`), the backends it may use and the backends this build +//! contains. +//! - `LIGHTCRAFT_GPU_PROBE=` (`vulkan`, `metal`, `dx12`, `gl`): every adapter that +//! backend lists, software ones (lavapipe, llvmpipe, WARP) included. One backend per process, so +//! a driver that crashes while it is enumerated (issue #136) costs only its own line. +//! +//! Only reports: which adapters the pipeline accepts is `ctx.rs`'s policy. +#![cfg(not(target_arch = "wasm32"))] + +use lightcraft_gpu::backend::{self, Backends}; + +const NAMES: [(&str, Backends); 4] = [("vulkan", Backends::VULKAN), ("metal", Backends::METAL), ("dx12", Backends::DX12), ("gl", Backends::GL)]; + +#[test] +#[ignore = "the capability probe of `cargo xtask ci`"] +fn gpu_probe() { + let line = match std::env::var("LIGHTCRAFT_GPU_PROBE").ok().filter(|b| !b.is_empty()) { + None => pipeline(), + Some(name) => adapters(&name), + }; + // its own line, prefixed: libtest may have printed `test gpu_probe ... ` on this one + println!("\ngpu_probe: {line}"); +} + +fn names(set: Backends) -> String { + let list: Vec = NAMES.iter().filter(|(_, b)| set.contains(*b)).map(|(n, _)| json_str(n)).collect(); + format!("[{}]", list.join(",")) +} + +fn pipeline() -> String { + let available = lightcraft_gpu::available(); + let opt = |s: Option| s.map_or_else(|| "null".to_owned(), |s| json_str(&s)); + let compiled = wgpu::Instance::enabled_backend_features(); + format!( + "{{\"pipeline\":{{\"available\":{available},\"adapter\":{},\"reason\":{},\"backends\":{},\"compiled\":{}}}}}", + opt(lightcraft_gpu::adapter_name()), + opt(lightcraft_gpu::unavailable_reason()), + names(backend::compute_backends().unwrap_or(Backends::empty()) & compiled), + names(compiled), + ) +} + +fn adapters(name: &str) -> String { + let (_, set) = NAMES.iter().find(|(n, _)| *n == name).unwrap_or_else(|| panic!("unknown backend `{name}` (vulkan, metal, dx12, gl)")); + let mut desc = wgpu::InstanceDescriptor::new_without_display_handle(); + desc.backends = *set; + desc.backend_options = backend::backend_options(); + desc.flags = backend::instance_flags_from_env(desc.flags); + let instance = wgpu::Instance::new(desc); + let adapters: Vec = pollster::block_on(instance.enumerate_adapters(*set)) + .iter() + .map(|a| { + let i = a.get_info(); + format!( + "{{\"name\":{},\"type\":{},\"vendor\":\"0x{:04x}\",\"device\":\"0x{:04x}\",\"driver\":{},\"driver_info\":{}}}", + json_str(&i.name), + json_str(&format!("{:?}", i.device_type)), + i.vendor, + i.device, + json_str(&i.driver), + json_str(&i.driver_info), + ) + }) + .collect(); + format!("{{\"backend\":{},\"adapters\":[{}]}}", json_str(name), adapters.join(",")) +} + +/// A JSON string literal. +fn json_str(s: &str) -> String { + let mut out = String::from("\""); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + c if u32::from(c) < 0x20 => out.push_str(&format!("\\u{:04x}", u32::from(c))), + c => out.push(c), + } + } + out.push('"'); + out +} diff --git a/xtask/src/ci_report.rs b/xtask/src/ci_report.rs new file mode 100644 index 000000000..dfbd9f5f9 --- /dev/null +++ b/xtask/src/ci_report.rs @@ -0,0 +1,599 @@ +//! What a `cargo xtask ci` run covered on this machine: the capability probe (OS, CPUs, RAM, GPU +//! adapters per backend, corpus, craft-fonts), each step's result and time, and the tests that +//! skipped for want of hardware or data. Written to `target/ci/report.json` and +//! `target/ci/summary.md` (a block to paste into a PR) and printed after the CI summary. +//! +//! Tests record skips with `lightcraft_testkit::skip!`; each test step points +//! `LIGHTCRAFT_SKIP_LOG` at `target/ci/skips-.jsonl`. The GPU adapters come from the ignored +//! `gpu_probe` test of `lightcraft-gpu` (xtask has no wgpu). Nothing here is sent anywhere. +//! +//! The report holds no host name, user name or absolute path: the workspace root and the home +//! directory are cut from skip reasons. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::{Value, json}; + +/// Bumped when a field changes meaning. +pub const SCHEMA: u64 = 1; + +/// The environment variable `lightcraft-testkit` appends skips to. +pub const SKIP_LOG_ENV: &str = "LIGHTCRAFT_SKIP_LOG"; + +/// `target/ci` under the workspace root (not `CARGO_TARGET_DIR`: one place to look, like +/// `target/bench`). +pub fn dir(root: &Path) -> PathBuf { + root.join("target").join("ci") +} + +/// Create `target/ci` and remove the previous run's skip logs, report and summary. +pub fn prepare(root: &Path) -> Result { + let dir = dir(root); + std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; + for entry in std::fs::read_dir(&dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + if (name.starts_with("skips-") && name.ends_with(".jsonl")) || name == "report.json" || name == "summary.md" { + let _ = std::fs::remove_file(entry.path()); + } + } + Ok(dir) +} + +/// The skip log of one step. +pub fn skip_log(root: &Path, step: &str) -> PathBuf { + let safe: String = step.chars().map(|c| if c.is_ascii_alphanumeric() || c == '-' { c } else { '-' }).collect(); + dir(root).join(format!("skips-{safe}.jsonl")) +} + +fn output(program: &str, args: &[&str], cwd: Option<&Path>) -> Option { + let mut c = Command::new(program); + c.args(args); + if let Some(cwd) = cwd { + c.current_dir(cwd); + } + let out = c.output().ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).trim().to_owned()) +} + +/// Commit, branch and whether the work tree has uncommitted changes. +pub fn git(root: &Path) -> Value { + let commit = output("git", &["rev-parse", "HEAD"], Some(root)); + let branch = output("git", &["rev-parse", "--abbrev-ref", "HEAD"], Some(root)); + let dirty = output("git", &["status", "--porcelain"], Some(root)).map(|s| !s.is_empty()); + json!({"commit": commit, "branch": branch, "dirty": dirty}) +} + +/// OS, version, architecture, CPUs and RAM. +pub fn machine(ram_total_gb: Option, ram_available_mb: Option) -> Value { + json!({ + "os": std::env::consts::OS, + "os_version": os_version(), + "arch": std::env::consts::ARCH, + "cpus": std::thread::available_parallelism().map(|n| n.get()).ok(), + "ram_total_gb": ram_total_gb, + "ram_available_mb": ram_available_mb, + }) +} + +fn os_version() -> Option { + if cfg!(target_os = "linux") { + let release = std::fs::read_to_string("/etc/os-release") + .ok() + .and_then(|s| s.lines().find_map(|l| l.strip_prefix("PRETTY_NAME=")).map(|v| v.trim_matches('"').to_owned())); + let kernel = output("uname", &["-r"], None); + match (release, kernel) { + (Some(r), Some(k)) => Some(format!("{r}, kernel {k}")), + (r, k) => r.or(k), + } + } else if cfg!(target_os = "macos") { + output("sw_vers", &["-productVersion"], None).map(|v| format!("macOS {v}")) + } else if cfg!(windows) { + output("cmd", &["/C", "ver"], None) + } else { + output("uname", &["-sr"], None) + } +} + +/// `rustc -V` of the toolchain cargo uses. +pub fn toolchain() -> Option { + output(&std::env::var("RUSTC").unwrap_or_else(|_| "rustc".into()), &["-V"], None) +} + +/// Test data present: how many of the pinned raw samples, other corpus files, craft-fonts. +pub fn data(root: &Path, pinned_raw: &[&str]) -> Value { + let corpus = std::env::var_os("LIGHTCRAFT_CORPUS").map(PathBuf::from).unwrap_or_else(|| root.join("corpus")); + let count = |sub: &str| -> usize { std::fs::read_dir(corpus.join(sub)).map(|d| d.flatten().filter(|e| e.path().is_file()).count()).unwrap_or(0) }; + let pinned = pinned_raw.iter().filter(|name| corpus.join("raw").join(name).is_file()).count(); + let fonts = std::env::var_os("CRAFT_FONTS_DIR").filter(|d| !d.is_empty()).map(PathBuf::from); + json!({ + "corpus": { + "custom_dir": std::env::var_os("LIGHTCRAFT_CORPUS").is_some(), + "raw_pinned": pinned, + "raw_pinned_total": pinned_raw.len(), + "raw_files": count("raw"), + "images": count("images"), + "pngsuite": count("pngsuite"), + }, + "craft_fonts": { + "set": fonts.is_some(), + // crates/engine/build.rs embeds the fonts of `fonts/manifest.txt` (a relative dir may mean the root) + "found": fonts.as_deref().is_some_and(|d| [d.to_path_buf(), root.join(d)].iter().any(|d| d.join("fonts/manifest.txt").is_file())), + }, + }) +} + +/// Backends wgpu can probe, in the order reports list them. +const BACKENDS: [&str; 4] = ["vulkan", "metal", "dx12", "gl"]; + +/// Run `gpu_probe` (the pipeline's device), then once per backend: the backends the pipeline may +/// use, or with `all` every backend in the build (`--hardware`; on Windows that loads the Vulkan +/// driver, which the app avoids by default, issue #136). +pub fn gpu(mut probe: impl FnMut(Option<&str>) -> Result, all: bool) -> Value { + let pipeline = match probe(None).and_then(|s| first_json(&s)) { + Ok(v) => v.get("pipeline").cloned().unwrap_or(Value::Null), + Err(e) => return json!({"error": e}), + }; + let names = |key: &str| -> Vec { + pipeline[key].as_array().map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()).unwrap_or_default() + }; + let (compiled, used) = (names("compiled"), names("backends")); + let mut backends = Vec::new(); + for b in BACKENDS.iter().filter(|b| compiled.iter().any(|c| c == *b) && (all || used.iter().any(|u| u == *b))) { + backends.push(match probe(Some(b)).and_then(|s| first_json(&s)) { + Ok(v) => v, + Err(e) => json!({"backend": b, "adapters": [], "error": e}), + }); + } + json!({"pipeline": pipeline, "backends": backends}) +} + +/// The JSON of the `gpu_probe: {…}` line in the probe's output (libtest prints around it). +fn first_json(s: &str) -> Result { + let line = s.lines().find_map(|l| l.trim().strip_prefix("gpu_probe: ")).ok_or_else(|| format!("gpu_probe printed no result: {s:?}"))?; + serde_json::from_str(line).map_err(|e| format!("gpu_probe: bad JSON ({e}): {line}")) +} + +/// Backends with at least one hardware (not software) adapter: `--hardware` runs the GPU tests +/// on each. +pub fn hardware_backends(gpu: &Value) -> Vec<(String, String)> { + let mut out = Vec::new(); + for b in gpu["backends"].as_array().into_iter().flatten() { + let Some(name) = b["backend"].as_str() else { continue }; + let hw = b["adapters"].as_array().into_iter().flatten().find(|a| a["type"].as_str().is_some_and(|t| t != "Cpu")); + if let Some(a) = hw { + out.push((name.to_owned(), a["name"].as_str().unwrap_or("?").to_owned())); + } + } + out +} + +/// One test that skipped, possibly in several steps. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Skip { + pub need: String, + pub reason: String, + pub test: String, + pub binary: String, + pub location: String, + pub steps: Vec, +} + +/// The skips of every `skips-.jsonl` in `dir`, deduplicated (the HEIF step reruns some test +/// binaries), sorted by need, then reason. Unreadable lines are dropped. +pub fn read_skips(dir: &Path, root: &Path) -> Vec { + let mut by_key: BTreeMap<(String, String, String, String, String), Vec> = BTreeMap::new(); + let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() }; + let mut files: Vec = entries.flatten().map(|e| e.path()).collect(); + files.sort(); + for path in files { + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let Some(step) = name.strip_prefix("skips-").and_then(|n| n.strip_suffix(".jsonl")) else { continue }; + let Ok(text) = std::fs::read_to_string(&path) else { continue }; + for line in text.lines() { + let Ok(v) = serde_json::from_str::(line) else { continue }; + let field = |k: &str| v[k].as_str().unwrap_or("").to_owned(); + let key = (field("need"), sanitize(&field("reason"), root), field("test"), field("binary"), field("location")); + let steps = by_key.entry(key).or_default(); + if !steps.iter().any(|s| s == step) { + steps.push(step.to_owned()); + } + } + } + by_key.into_iter().map(|((need, reason, test, binary, location), steps)| Skip { need, reason, test, binary, location, steps }).collect() +} + +/// Cut the workspace root and the home directory from `s`, and fold `dir/../` in paths. +pub fn sanitize(s: &str, root: &Path) -> String { + let mut out = s.to_owned(); + let root = root.display().to_string(); + if !root.is_empty() { + for sep in ['/', '\\'] { + out = out.replace(&format!("{root}{sep}"), ""); + } + out = out.replace(&root, "."); + } + let home = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")).map(|h| h.to_string_lossy().into_owned()); + if let Some(home) = home.filter(|h| h.len() > 1) { + out = out.replace(&home, "~"); + } + out.split(' ').map(fold_parent_dirs).collect::>().join(" ") +} + +/// `crates/raw/../../corpus/raw` → `corpus/raw` (lexically; leading `..` stay). +fn fold_parent_dirs(token: &str) -> String { + if !token.contains("/../") && !token.contains("\\..\\") { + return token.to_owned(); + } + let sep = if token.contains("/../") { '/' } else { '\\' }; + let mut parts: Vec<&str> = Vec::new(); + for part in token.split(sep) { + if part == ".." && parts.last().is_some_and(|p| *p != ".." && !p.is_empty()) { + parts.pop(); + } else { + parts.push(part); + } + } + parts.join(&sep.to_string()) +} + +pub fn skips_json(skips: &[Skip]) -> Value { + Value::Array( + skips + .iter() + .map(|s| json!({"need": s.need, "reason": s.reason, "test": s.test, "binary": s.binary, "location": s.location, "steps": s.steps})) + .collect(), + ) +} + +/// What an empty skip list means: no test skipped, or the tests never ran. +fn no_skips(report: &Value) -> &'static str { + let test_ran = report["steps"].as_array().into_iter().flatten().any(|s| s["name"] == "test" && s["status"] != "not run"); + if test_ran { "none recorded" } else { "unknown, the test step did not run" } +} + +fn skips_from(report: &Value) -> Vec { + report["skips"] + .as_array() + .into_iter() + .flatten() + .map(|v| { + let f = |k: &str| v[k].as_str().unwrap_or("").to_owned(); + let steps = v["steps"].as_array().map(|a| a.iter().filter_map(|s| s.as_str().map(str::to_owned)).collect()).unwrap_or_default(); + Skip { need: f("need"), reason: f("reason"), test: f("test"), binary: f("binary"), location: f("location"), steps } + }) + .collect() +} + +/// Skip counts per need, most first: `[("corpus", 40), ("gpu", 5)]`. +pub fn counts_by_need(skips: &[Skip]) -> Vec<(String, usize)> { + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for s in skips { + *counts.entry(&s.need).or_default() += 1; + } + let mut v: Vec<(String, usize)> = counts.into_iter().map(|(k, n)| (k.to_owned(), n)).collect(); + v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + v +} + +/// What gets the skipped tests of `need` running. +fn hint(need: &str) -> &'static str { + match need { + "gpu" => "a hardware GPU adapter the pipeline accepts", + "corpus" => "the corpus (`cargo xtask corpus --download`; some files are private)", + "fonts" => "craft-fonts (`CRAFT_FONTS_DIR`, see the README)", + "model" => "the model files the reasons name", + "input" => "the input files the reasons name", + "system" => "what the operating system provides", + "feature" => "support this build doesn't have yet", + _ => "what the reasons name", + } +} + +/// The skips of `need`, grouped by reason, largest group first. +fn by_reason<'a>(skips: &'a [Skip], need: &str) -> Vec<(&'a str, Vec<&'a Skip>)> { + let mut groups: BTreeMap<&str, Vec<&Skip>> = BTreeMap::new(); + for k in skips.iter().filter(|k| k.need == need) { + groups.entry(&k.reason).or_default().push(k); + } + let mut groups: Vec<(&str, Vec<&Skip>)> = groups.into_iter().collect(); + groups.sort_by(|a, b| b.1.len().cmp(&a.1.len()).then_with(|| a.0.cmp(b.0))); + groups +} + +pub fn seconds(s: f64) -> String { + if s < 60.0 { format!("{s:.1} s") } else { format!("{}m {:02}s", (s / 60.0).floor() as u64, (s % 60.0).floor() as u64) } +} + +fn test_name(s: &Skip) -> String { + if s.test.is_empty() || s.test == "main" { format!("{} ({})", s.binary, s.location) } else { format!("{}::{}", s.binary, s.test) } +} + +fn adapter_line(a: &Value) -> String { + let name = a["name"].as_str().unwrap_or("?"); + let kind = a["type"].as_str().unwrap_or("?"); + let driver = [a["driver"].as_str().unwrap_or(""), a["driver_info"].as_str().unwrap_or("")] + .into_iter() + .filter(|s| !s.is_empty()) + .collect::>() + .join(" "); + if driver.is_empty() { format!("{name} ({kind})") } else { format!("{name} ({kind}; {driver})") } +} + +/// "the pipeline's device" line. +fn gpu_line(report: &Value) -> String { + let gpu = &report["capabilities"]["gpu"]; + if let Some(e) = gpu["error"].as_str() { + return format!("probe failed: {e}"); + } + let p = &gpu["pipeline"]; + match (p["available"].as_bool(), p["adapter"].as_str(), p["reason"].as_str()) { + (Some(true), Some(a), _) => a.to_owned(), + (_, _, Some(r)) => format!("none ({r})"), + _ => "not probed".to_owned(), + } +} + +fn adapters_lines(report: &Value) -> Vec { + let mut out = Vec::new(); + for b in report["capabilities"]["gpu"]["backends"].as_array().into_iter().flatten() { + let name = b["backend"].as_str().unwrap_or("?"); + let list: Vec = b["adapters"].as_array().into_iter().flatten().map(adapter_line).collect(); + let text = match (b["error"].as_str(), list.is_empty()) { + (Some(e), _) => format!("probe failed: {e}"), + (None, true) => "no adapter".to_owned(), + (None, false) => list.join(", "), + }; + out.push(format!("{name}: {text}")); + } + out +} + +fn machine_line(report: &Value) -> String { + let m = &report["machine"]; + let mut s = format!("{} {}", m["os"].as_str().unwrap_or("?"), m["arch"].as_str().unwrap_or("?")); + if let Some(v) = m["os_version"].as_str() { + let _ = write!(s, " ({v})"); + } + if let Some(n) = m["cpus"].as_u64() { + let _ = write!(s, ", {n} CPUs"); + } + if let Some(gb) = m["ram_total_gb"].as_u64() { + let _ = write!(s, ", {gb} GB RAM"); + } + if let Some(mb) = m["ram_available_mb"].as_u64() { + let _ = write!(s, " ({:.1} GB available)", mb as f64 / 1024.0); + } + s +} + +fn data_line(report: &Value) -> String { + let d = &report["capabilities"]["data"]; + let c = &d["corpus"]; + let n = |k: &str| c[k].as_u64().unwrap_or(0); + let fonts = match (d["craft_fonts"]["set"].as_bool(), d["craft_fonts"]["found"].as_bool()) { + (Some(true), Some(true)) => "CRAFT_FONTS_DIR set", + (Some(true), _) => "CRAFT_FONTS_DIR set but not found", + _ => "no CRAFT_FONTS_DIR", + }; + format!("corpus: {} of {} pinned raws, {} images, {} PngSuite files; {fonts}", n("raw_pinned"), n("raw_pinned_total"), n("images"), n("pngsuite")) +} + +fn commit_line(report: &Value) -> String { + let g = &report["git"]; + let commit = g["commit"].as_str().map_or("unknown commit".to_owned(), |c| c.chars().take(10).collect()); + let state = match g["dirty"].as_bool() { + Some(true) => "with uncommitted changes", + Some(false) => "clean", + None => "state unknown", + }; + format!("{commit} ({state})") +} + +/// The lines printed after the CI summary. +pub fn summary_text(report: &Value) -> String { + let mut s = String::new(); + let _ = writeln!(s, "\nMachine: {}", machine_line(report)); + let _ = writeln!(s, "GPU for the pipeline: {}", gpu_line(report)); + for line in adapters_lines(report) { + let _ = writeln!(s, " adapters on {line}"); + } + let _ = writeln!(s, "Data: {}", data_line(report)); + for h in report["hardware"].as_array().into_iter().flatten() { + let _ = writeln!( + s, + "GPU tests on {} ({}): {}", + h["backend"].as_str().unwrap_or("?"), + h["adapter"].as_str().unwrap_or("?"), + h["status"].as_str().unwrap_or("?") + ); + } + let skips = skips_from(report); + if skips.is_empty() { + let _ = writeln!(s, "Skipped tests: {}", no_skips(report)); + } else { + let counts: Vec = counts_by_need(&skips).iter().map(|(k, n)| format!("{k} {n}")).collect(); + let _ = writeln!(s, "Skipped tests: {} passed without running their checks ({}):", skips.len(), counts.join(", ")); + for (need, n) in counts_by_need(&skips) { + let _ = writeln!(s, " {need} ({n}): needs {}", hint(&need)); + let groups = by_reason(&skips, &need); + const SHOWN: usize = 4; + for (reason, tests) in groups.iter().take(SHOWN) { + let first = tests.first().map(|t| test_name(t)).unwrap_or_default(); + let more = if tests.len() > 1 { ", …" } else { "" }; + let _ = writeln!(s, " {:>3}× {reason} ({first}{more})", tests.len()); + } + if groups.len() > SHOWN { + let rest: usize = groups.iter().skip(SHOWN).map(|(_, t)| t.len()).sum(); + let _ = writeln!(s, " {rest:>3}× {} other reasons (target/ci/report.json)", groups.len() - SHOWN); + } + } + } + s +} + +fn cell(s: &str) -> String { + s.replace('|', "\\|").replace('\n', " ") +} + +/// A Markdown block for a PR description or comment. +pub fn summary_md(report: &Value) -> String { + let mut s = String::new(); + let result = report["result"].as_str().unwrap_or("?"); + let command = report["command"].as_str().unwrap_or("cargo xtask ci"); + let _ = writeln!(s, ""); + let _ = writeln!(s, "**`{command}`: {result}** at {}\n", commit_line(report)); + let _ = writeln!(s, "| | |\n|---|---|"); + let _ = writeln!(s, "| Machine | {} |", cell(&machine_line(report))); + if let Some(t) = report["toolchain"].as_str() { + let _ = writeln!(s, "| Rust | {} |", cell(t)); + } + let _ = writeln!(s, "| GPU for the pipeline | {} |", cell(&gpu_line(report))); + let adapters = adapters_lines(report); + if !adapters.is_empty() { + let _ = writeln!(s, "| Adapters | {} |", cell(&adapters.join("; "))); + } + let _ = writeln!(s, "| Data | {} |", cell(&data_line(report))); + let _ = writeln!(s, "\n| Step | Result | Time |\n|---|---|---|"); + for step in report["steps"].as_array().into_iter().flatten() { + let time = step["seconds"].as_f64().map(seconds).unwrap_or_default(); + let _ = writeln!(s, "| {} | {} | {time} |", cell(step["name"].as_str().unwrap_or("?")), step["status"].as_str().unwrap_or("?")); + } + for h in report["hardware"].as_array().into_iter().flatten() { + let time = h["seconds"].as_f64().map(seconds).unwrap_or_default(); + let name = format!("GPU tests on {} ({})", h["backend"].as_str().unwrap_or("?"), h["adapter"].as_str().unwrap_or("?")); + let _ = writeln!(s, "| {} | {} | {time} |", cell(&name), h["status"].as_str().unwrap_or("?")); + } + let skips = skips_from(report); + if skips.is_empty() { + let _ = writeln!(s, "\nSkipped tests: {}.", no_skips(report)); + } else { + let counts: Vec = counts_by_need(&skips).iter().map(|(k, n)| format!("{k} {n}")).collect(); + let _ = writeln!(s, "\n
{} tests skipped ({})\n", skips.len(), counts.join(", ")); + let _ = writeln!(s, "| Need | Tests | Reason | For example |\n|---|---|---|---|"); + for (need, _) in counts_by_need(&skips) { + let groups = by_reason(&skips, &need); + const SHOWN: usize = 12; + for (reason, tests) in groups.iter().take(SHOWN) { + let first = tests.first().map(|t| test_name(t)).unwrap_or_default(); + let _ = writeln!(s, "| {} | {} | {} | `{}` |", cell(&need), tests.len(), cell(reason), cell(&first)); + } + if groups.len() > SHOWN { + let rest: usize = groups.iter().skip(SHOWN).map(|(_, t)| t.len()).sum(); + let _ = writeln!(s, "| {} | {rest} | {} other reasons | |", cell(&need), groups.len() - SHOWN); + } + } + let _ = writeln!(s, "\n
"); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("xtask-ci-report-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + #[test] + fn skips_are_read_per_step_deduplicated_and_cut_free_of_the_root() { + let root = Path::new("/work/lightcraft"); + let dir = tmp("skips"); + let gpu = + r#"{"need":"gpu","reason":"no GPU adapter","test":"camera_tone","binary":"equivalence","location":"crates/gpu/tests/equivalence.rs:22"}"#; + let corpus = r#"{"need":"corpus","reason":"skip /work/lightcraft/crates/raw/../../corpus/raw absent","test":"decodes","binary":"corpus","location":"crates/raw/tests/corpus.rs:28"}"#; + std::fs::write(dir.join("skips-heif.jsonl"), format!("{corpus}\n")).unwrap(); + std::fs::write(dir.join("skips-test.jsonl"), format!("{gpu}\n{corpus}\nnot json\n")).unwrap(); + std::fs::write(dir.join("other.jsonl"), format!("{gpu}\n")).unwrap(); + let skips = read_skips(&dir, root); + assert_eq!(skips.len(), 2, "{skips:?}"); + assert_eq!(skips[0].need, "corpus"); + assert_eq!(skips[0].reason, "skip corpus/raw absent"); + assert_eq!(skips[0].steps, ["heif", "test"]); + assert_eq!(skips[1].steps, ["test"]); + assert_eq!(counts_by_need(&skips), [("corpus".to_owned(), 1), ("gpu".to_owned(), 1)]); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn prepare_clears_the_last_runs_files_only() { + let root = tmp("prepare"); + let dir = dir(&root); + std::fs::create_dir_all(&dir).unwrap(); + for f in ["skips-test.jsonl", "report.json", "summary.md", "keep.txt"] { + std::fs::write(dir.join(f), "x").unwrap(); + } + prepare(&root).unwrap(); + let left: Vec = std::fs::read_dir(&dir).unwrap().flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(); + assert_eq!(left, ["keep.txt"]); + assert_eq!(skip_log(&root, "gpu:dx12").file_name().unwrap(), "skips-gpu-dx12.jsonl"); + std::fs::remove_dir_all(&root).unwrap(); + } + + #[test] + fn parent_dirs_fold_in_path_tokens_only() { + assert_eq!(fold_parent_dirs("crates/raw/../../corpus/raw"), "corpus/raw"); + assert_eq!(fold_parent_dirs("../corpus/raw"), "../corpus/raw"); + assert_eq!(fold_parent_dirs("C:\\w\\crates\\raw\\..\\..\\corpus"), "C:\\w\\corpus"); + assert_eq!(fold_parent_dirs("absent"), "absent"); + } + + #[test] + fn the_probe_asks_the_pipelines_backends_or_all_with_hardware() { + let pipeline = r#"{"pipeline":{"available":false,"adapter":null,"reason":"software adapter (llvmpipe) skipped","backends":["vulkan","dx12"],"compiled":["vulkan","dx12","gl"]}}"#; + let asked = std::cell::RefCell::new(Vec::new()); + let probe = |b: Option<&str>| -> Result { + asked.borrow_mut().push(b.map(str::to_owned)); + Ok(match b { + None => format!("running 1 test\ntest gpu_probe ... \ngpu_probe: {pipeline}\nok\n"), + Some("vulkan") => r#"gpu_probe: {"backend":"vulkan","adapters":[{"name":"llvmpipe","type":"Cpu","driver":"llvmpipe","driver_info":"Mesa"}]}"#.to_owned(), + Some("dx12") => r#"gpu_probe: {"backend":"dx12","adapters":[{"name":"Intel(R) UHD Graphics 620","type":"IntegratedGpu","driver":"","driver_info":"31.0.101"}]}"#.to_owned(), + Some(_) => return Err("crashed".into()), + }) + }; + let gpu_default = gpu(probe, false); + assert_eq!(*asked.borrow(), [None, Some("vulkan".to_owned()), Some("dx12".to_owned())]); + assert_eq!(hardware_backends(&gpu_default), [("dx12".to_owned(), "Intel(R) UHD Graphics 620".to_owned())]); + asked.borrow_mut().clear(); + let gpu_all = gpu(probe, true); + assert_eq!(asked.borrow().len(), 4, "gl too"); + assert_eq!(gpu_all["backends"][2]["error"], "crashed"); + + let report = json!({ + "command": "cargo xtask ci", + "result": "pass", + "git": {"commit": "0123456789abcdef", "dirty": false}, + "machine": {"os": "windows", "arch": "x86_64", "cpus": 8, "ram_total_gb": 16}, + "capabilities": {"gpu": gpu_all, "data": {"corpus": {"raw_pinned": 3, "raw_pinned_total": 30}, "craft_fonts": {"set": false}}}, + "steps": [{"name": "fmt", "status": "ok", "seconds": 1.25}, {"name": "test", "status": "ok", "seconds": 125.0}], + "skips": [{"need": "gpu", "reason": "no GPU | adapter", "test": "memory", "binary": "memory", "location": "x.rs:1", "steps": ["test"]}], + }); + let text = summary_text(&report); + assert!(text.contains("GPU for the pipeline: none (software adapter (llvmpipe) skipped)"), "{text}"); + assert!(text.contains("adapters on dx12: Intel(R) UHD Graphics 620 (IntegratedGpu; 31.0.101)"), "{text}"); + assert!(text.contains("adapters on gl: probe failed: crashed"), "{text}"); + assert!(text.contains("Skipped tests: 1 passed without running their checks (gpu 1)"), "{text}"); + assert!(text.contains(" gpu (1): needs a hardware GPU adapter"), "{text}"); + assert!(text.contains(" 1× no GPU | adapter (memory::memory)"), "{text}"); + assert!(text.contains("corpus: 3 of 30 pinned raws"), "{text}"); + let md = summary_md(&report); + assert!(md.contains("**`cargo xtask ci`: pass** at 0123456789 (clean)"), "{md}"); + assert!(md.contains("| test | ok | 2m 05s |"), "{md}"); + assert!(md.contains("| gpu | 1 | no GPU \\| adapter | `memory::memory` |"), "{md}"); + assert!(md.contains("1 tests skipped (gpu 1)"), "{md}"); + } + + #[test] + fn no_skips_is_not_claimed_when_the_tests_never_ran() { + let stopped = json!({"steps": [{"name": "clippy", "status": "fail"}, {"name": "test", "status": "not run"}], "skips": []}); + assert!(summary_text(&stopped).contains("Skipped tests: unknown, the test step did not run")); + let ran = json!({"steps": [{"name": "test", "status": "ok"}], "skips": []}); + assert!(summary_md(&ran).contains("Skipped tests: none recorded.")); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 08ef435c5..0bc3631ba 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -5,6 +5,7 @@ mod assets; mod bench; +mod ci_report; mod ico; mod layers; mod parity; @@ -33,7 +34,11 @@ commands: web [--serve [port]] [--dev] build the browser app (apps/lightcraft-web) into /web-site/; --serve serves it on http://127.0.0.1: (default 8080) - ci fmt --check, clippy -D warnings, heif, test, parity refs, layers, assets, wasm (stops at first failure) + ci [--hardware] [--summary-md] + fmt --check, clippy -D warnings, heif, test, GPU probe, parity refs, layers, assets, wasm + (stops at first failure); writes target/ci/report.json (machine, GPU adapters, data, + steps, skipped tests) and target/ci/summary.md; --hardware also runs the GPU tests on + every backend with a hardware adapter; --summary-md prints the Markdown summary corpus [--download] show where test corpora live; --download fetches PngSuite and CC0 raw samples (raw.pixls.us) into corpus/ and checks their sha256 stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`) @@ -51,7 +56,7 @@ fn main() -> ExitCode { Some("parity") => parity::run(&root(), rest.contains(&"--write")), Some("wasm") => cmd_wasm(), Some("web") => web::run(&rest), - Some("ci") => cmd_ci(), + Some("ci") => cmd_ci(&rest), Some("corpus") => cmd_corpus(rest.contains(&"--download")), Some("stats") => stats::run(&root(), rest.contains(&"--exact")), Some("-h" | "--help" | "help") | None => { @@ -240,9 +245,38 @@ fn cmd_wasm() -> Result<(), String> { if failed == 0 { Ok(()) } else { Err(format!("{failed} crate(s) failed the wasm check")) } } -fn cmd_ci() -> Result<(), String> { - type Step = (&'static str, Box Result<(), String>>); - let steps: Vec = vec![ +fn cmd_ci(args: &[&str]) -> Result<(), String> { + let hardware = args.contains(&"--hardware"); + let print_md = args.contains(&"--summary-md"); + if let Some(other) = args.iter().find(|a| !matches!(**a, "--hardware" | "--summary-md")) { + return Err(format!("ci: unknown option `{other}`\n\n{USAGE}")); + } + let root = root(); + let report_dir = ci_report::prepare(&root)?; + let started = std::time::Instant::now(); + let started_unix = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); + // test runs append the skips their tests record (`lightcraft_testkit::skip!`) to target/ci/skips-.jsonl + let test = |cargo_args: &[&str], step: &str| { + let mut c = cargo(); + c.args(cargo_args).env(ci_report::SKIP_LOG_ENV, ci_report::skip_log(&root, step)); + run(c, &format!("cargo {}", cargo_args.join(" "))) + }; + let gpu = std::cell::RefCell::new(serde_json::Value::Null); + let hardware_runs = std::cell::RefCell::new(Vec::new()); + // What this machine offers the GPU tests, once, after the workspace tests (whose build the + // probe reuses). Never fails the run: the report says what went wrong. + let probe = || { + if gpu.borrow().is_null() { + eprintln!("\n=== ci: GPU probe ==="); + let t = std::time::Instant::now(); + let mut found = ci_report::gpu(probe_gpu, hardware); + found["seconds"] = serde_json::json!(t.elapsed().as_secs_f64()); + *gpu.borrow_mut() = found; + } + }; + + type Step<'a> = (&'static str, Box Result<(), String> + 'a>); + let mut steps: Vec = vec![ ( "fmt", Box::new(|| { @@ -266,46 +300,144 @@ fn cmd_ci() -> Result<(), String> { let mut c = cargo(); c.args(["clippy", "-p", "lightcraft-codecs", "--features", "heif", "--all-targets", "--", "-D", "warnings"]); run(c, "cargo clippy -p lightcraft-codecs --features heif --all-targets -- -D warnings")?; - let mut c = cargo(); - c.args(["test", "-p", "lightcraft-codecs", "-p", "lightcraft-heif", "--features", "lightcraft-codecs/heif"]); - run(c, "cargo test -p lightcraft-codecs -p lightcraft-heif --features lightcraft-codecs/heif")?; + test(&["test", "-p", "lightcraft-codecs", "-p", "lightcraft-heif", "--features", "lightcraft-codecs/heif"], "heif")?; // the engine's HEIC import test, with the decoder (the workspace run checks the error) - let mut c = cargo(); - c.args(["test", "-p", "lightcraft-engine", "--lib", "--features", "lightcraft-codecs/heif", "heic"]); - run(c, "cargo test -p lightcraft-engine --lib --features lightcraft-codecs/heif heic") + test(&["test", "-p", "lightcraft-engine", "--lib", "--features", "lightcraft-codecs/heif", "heic"], "heif") }), ), - ( - "test", - Box::new(|| { - let mut c = cargo(); - c.args(["test", "--workspace"]); - run(c, "cargo test --workspace") - }), - ), - ("parity", Box::new(|| parity::run(&root(), false))), - ("layers", Box::new(cmd_layers)), - ("assets", Box::new(|| assets::run(&root()))), - ("wasm", Box::new(cmd_wasm)), + ("test", Box::new(|| test(&["test", "--workspace"], "test"))), ]; - let mut done = Vec::new(); + if hardware { + steps.push(( + "gpu-backends", + Box::new(|| { + // the GPU tests once per backend with a hardware adapter (`LIGHTCRAFT_GPU_BACKEND`) + probe(); + let backends = ci_report::hardware_backends(&gpu.borrow()); + if backends.is_empty() { + eprintln!("no hardware GPU adapter on any backend: nothing to run"); + } + let mut failed = Vec::new(); + for (backend, adapter) in backends { + let step = format!("gpu-{backend}"); + let mut c = cargo(); + c.args(["test", "-p", "lightcraft-gpu", "--features", "denoise"]) + .env("LIGHTCRAFT_GPU_BACKEND", &backend) + .env(ci_report::SKIP_LOG_ENV, ci_report::skip_log(&root, &step)); + let t = std::time::Instant::now(); + let ok = run(c, &format!("LIGHTCRAFT_GPU_BACKEND={backend} cargo test -p lightcraft-gpu --features denoise")).is_ok(); + hardware_runs.borrow_mut().push(serde_json::json!({ + "backend": backend, + "adapter": adapter, + "status": if ok { "ok" } else { "fail" }, + "seconds": t.elapsed().as_secs_f64(), + })); + if !ok { + failed.push(backend); + } + } + if failed.is_empty() { Ok(()) } else { Err(format!("the GPU tests failed on {}", failed.join(", "))) } + }), + )); + } + steps.push(("parity", Box::new(|| parity::run(&root, false)))); + steps.push(("layers", Box::new(cmd_layers))); + steps.push(("assets", Box::new(|| assets::run(&root)))); + steps.push(("wasm", Box::new(cmd_wasm))); + + let mut results = Vec::new(); + let mut failure = None; for (name, f) in &steps { eprintln!("\n=== ci: {name} ==="); - if let Err(e) = f() { + let t = std::time::Instant::now(); + let r = f(); + let status = if r.is_ok() { "ok" } else { "fail" }; + results.push(serde_json::json!({"name": name, "status": status, "seconds": t.elapsed().as_secs_f64()})); + if *name == "test" { + // right after the workspace tests, before parity and wasm build other feature sets + probe(); + } + if let Err(e) = r { + failure = Some((*name, e)); + break; + } + } + if gpu.borrow().is_null() { + *gpu.borrow_mut() = serde_json::json!({"error": "not probed: the test step did not run"}); + } + let done: Vec<&str> = steps.iter().take(results.len()).map(|(n, _)| *n).collect(); + match &failure { + Some((name, e)) => { println!("\nCI summary:"); - for d in &done { + for d in done.iter().filter(|d| *d != name) { println!(" ok {d}"); } println!(" FAIL {name}: {e}"); - for (n, _) in steps.iter().skip(done.len() + 1) { + for (n, _) in steps.iter().skip(results.len()) { println!(" skip {n}"); } - return Err(format!("ci failed at `{name}`")); } - done.push(*name); + None => println!("\nCI summary: all {} steps passed ({})", done.len(), done.join(", ")), } - println!("\nCI summary: all {} steps passed ({})", done.len(), done.join(", ")); - Ok(()) + for (n, _) in steps.iter().skip(results.len()) { + results.push(serde_json::json!({"name": n, "status": "not run"})); + } + + let skips = ci_report::read_skips(&report_dir, &root); + let ram_mb = available_ram_mb(); + let report = serde_json::json!({ + "schema": ci_report::SCHEMA, + "command": std::iter::once("cargo xtask ci").chain(args.iter().copied()).collect::>().join(" "), + "result": if failure.is_some() { "fail" } else { "pass" }, + "failed_step": failure.as_ref().map(|(n, _)| *n), + "started_unix": started_unix, + "seconds": started.elapsed().as_secs_f64(), + "git": ci_report::git(&root), + "machine": ci_report::machine(total_ram_gb(), ram_mb), + "toolchain": ci_report::toolchain(), + "capabilities": {"gpu": gpu.take(), "data": ci_report::data(&root, &RAW_SAMPLES.iter().map(|s| s.0).collect::>())}, + "steps": results, + "hardware": hardware_runs.take(), + "skips": ci_report::skips_json(&skips), + }); + print!("{}", ci_report::summary_text(&report)); + let md = ci_report::summary_md(&report); + let written = serde_json::to_string_pretty(&report) + .map_err(|e| e.to_string()) + .and_then(|json| std::fs::write(report_dir.join("report.json"), json + "\n").map_err(|e| e.to_string())) + .and_then(|()| std::fs::write(report_dir.join("summary.md"), &md).map_err(|e| e.to_string())); + match written { + Ok(()) => println!("Report: target/ci/report.json; to paste into a PR: target/ci/summary.md (or run with --summary-md)"), + Err(e) => eprintln!("ci: could not write the report to {}: {e}", report_dir.display()), + } + if print_md { + println!("\n{md}"); + } + match failure { + Some((name, _)) => Err(format!("ci failed at `{name}`")), + None => Ok(()), + } +} + +/// Run the ignored `gpu_probe` test of lightcraft-gpu: the pipeline's device (`None`) or one +/// backend's adapters. `--workspace`, like the test step, so its build is reused. Its stdout, or why +/// it failed (a driver that crashes while it is enumerated, issue #136, ends up here, not in xtask). +fn probe_gpu(backend: Option<&str>) -> Result { + let mut c = cargo(); + c.args(["test", "-q", "--workspace", "--test", "gpu_probe", "--", "--ignored", "--exact", "gpu_probe", "--nocapture"]); + c.env("LIGHTCRAFT_GPU_PROBE", backend.unwrap_or("")); + let out = c.output().map_err(|e| format!("gpu_probe: {e}"))?; + if !out.status.success() { + let err = String::from_utf8_lossy(&out.stderr); + let tail: Vec<&str> = err.lines().rev().filter(|l| !l.trim().is_empty()).take(3).collect(); + return Err(format!( + "gpu_probe {}exited with {}: {}", + backend.map(|b| format!("{b} ")).unwrap_or_default(), + out.status, + tail.into_iter().rev().collect::>().join(" / ") + )); + } + Ok(String::from_utf8_lossy(&out.stdout).into_owned()) } /// CC0 raw samples (raw.pixls.us). Small, representative set; extend freely (CC0 only). From dbf2dcbfbd3cd75e901ce2ed6b2871397c3e7811 Mon Sep 17 00:00:00 2001 From: Chad Elliott Date: Sun, 11 Oct 2026 16:33:54 -0500 Subject: [PATCH 3/3] docs: #1037 visible skips and the ci report Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 +++++- CLAUDE.md | 6 +++++- README.md | 7 +++++++ docs/architecture.md | 15 +++++++++++++-- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8cd199ff6..3d615d00c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,11 +125,15 @@ of it. Full standard: `../craftrules/standards/never-crash.md` with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe ≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG. - Never commit media: test images are procedural (`lightcraft-scenes`) or CC0 downloads in the gitignored `corpus/`. +- A test that can't run here (no GPU, corpus file, craft-fonts face, model) calls `lightcraft_testkit::skip!(Gpu, "why")` + before it returns, not a bare `eprintln!`. `cargo xtask ci` then lists the skips and the machine's GPU adapters and + data, and writes `target/ci/report.json` and `target/ci/summary.md` (Markdown for a PR). ## Map of the code `geom`, `color`, `raster`, `tiff` (L0) → `raw`, `codecs`, `meta`, `develop` (L1) → `pipeline` → `catalog` → `engine` → `ui-egui`, `mcp` (L5) → apps `lightcraft` (desktop), `lightcraft-cli` (render/commands/MCP). `scenes` generates demo -photos. `xtask` = tooling (`ci`, `layers`, `assets`, `parity`, `wasm`, `corpus`, `stats`). `flake.nix` + +photos. `xtask` = tooling (`ci`, `layers`, `assets`, `parity`, `wasm`, `corpus`, `stats`); `testkit` = test helpers +(dev-dependency only). `flake.nix` + `nix/package.nix` = the Nix package (`nix build` builds both binaries with the craft-fonts input, installs the desktop file/icons/AppStream metadata and runs `cargo test --workspace`; `nix develop` = dev shell). Community-maintained and not in CI: it may lag behind the workspace; see README → Quick start. diff --git a/CLAUDE.md b/CLAUDE.md index 4a736f12a..5596bb95f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,11 +132,15 @@ of it. Full standard: `../craftrules/standards/never-crash.md` with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe ≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG. - Never commit media: test images are procedural (`lightcraft-scenes`) or CC0 downloads in the gitignored `corpus/`. +- A test that can't run here (no GPU, corpus file, craft-fonts face, model) calls `lightcraft_testkit::skip!(Gpu, "why")` + before it returns, not a bare `eprintln!`. `cargo xtask ci` then lists the skips and the machine's GPU adapters and + data, and writes `target/ci/report.json` and `target/ci/summary.md` (Markdown for a PR). ## Map of the code `geom`, `color`, `raster`, `tiff` (L0) → `raw`, `codecs`, `meta`, `develop` (L1) → `pipeline` → `catalog` → `engine` → `ui-egui`, `mcp` (L5) → apps `lightcraft` (desktop), `lightcraft-cli` (render/commands/MCP). `scenes` generates demo -photos. `xtask` = tooling (`ci`, `layers`, `assets`, `parity`, `wasm`, `corpus`, `stats`). `flake.nix` + +photos. `xtask` = tooling (`ci`, `layers`, `assets`, `parity`, `wasm`, `corpus`, `stats`); `testkit` = test helpers +(dev-dependency only). `flake.nix` + `nix/package.nix` = the Nix package (`nix build` builds both binaries with the craft-fonts input, installs the desktop file/icons/AppStream metadata and runs `cargo test --workspace`; `nix develop` = dev shell). Community-maintained and not in CI: it may lag behind the workspace; see README → Quick start. diff --git a/README.md b/README.md index 8884f2bd3..6f9ff1cba 100644 --- a/README.md +++ b/README.md @@ -370,6 +370,13 @@ per CPU, otherwise exhaust RAM before any test runs. GPU coverage is unchanged. `CARGO_PROFILE_DEV_DEBUG`, `CARGO_BUILD_JOBS` and `RUST_TEST_THREADS` settings override these defaults. Ordinary development commands keep their own settings. +Tests that need a GPU, the corpus, the craft-fonts faces or a model pass without checking anything when +those are missing (libtest has no "skipped" status), so `cargo xtask ci` lists them: tests record a skip +with `lightcraft_testkit::skip!`, and the run ends with what the machine offered (OS, CPUs, RAM, GPU +adapters per backend, corpus, fonts) and every skip by reason. It writes `target/ci/report.json` and +`target/ci/summary.md`, a block to paste into a PR (`--summary-md` prints it); `--hardware` also runs the +GPU tests on every backend that has a hardware adapter. Nothing is uploaded. + **Chinese and Japanese text** need the shared font repo, an optional build input (official releases always include it): ```sh diff --git a/docs/architecture.md b/docs/architecture.md index c50066a10..7d1b7a3ae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (created from the code; the full design notes stay in the local `plan/architecture.md`) · **Target:** Adobe Lightroom Classic 15.5 / Lightroom 9.6 +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (`cargo xtask ci` reports the machine and skipped tests; `testkit`) · **Target:** Adobe Lightroom Classic 15.5 / Lightroom 9.6 How LightCraft is built today. The longer design document (`plan/architecture.md`) is local-only (`plan/` is gitignored); this page is the committed, current summary. Sizes are Rust lines counted @@ -19,7 +19,7 @@ L5 depends on egui, eframe, winit or rfd, and that each crate only uses lower la | L3 | `catalog` (11.0k), `gpu` (5.1k), `preview` (1.3k), `merge` (3.8k), `segment` (2.3k) | Library model and persistence; wgpu compute twin of the pipeline; thumbnail/preview cache and job pool; HDR and panorama merge; SAM 3 segmentation | | L4 | `engine` (53.2k) | Session, command registry (everything is a command), history, render and export jobs, import, Lightroom catalog and preset import, faces index, view-models | | L5 | `ui-egui` (46.9k), `mcp` (2.3k) | The Lightroom-style egui interface (swappable); the MCP server | -| L6 | `apps/lightcraft` (2.8k), `apps/lightcraft-cli` (1.5k), `apps/lightcraft-web` (3.4k), `xtask` (2.7k) | Desktop app (eframe + wgpu, native menus, control server), CLI (render, commands, calibrate, snapshot, MCP), WASM build, tooling | +| L6 | `apps/lightcraft` (2.8k), `apps/lightcraft-cli` (1.5k), `apps/lightcraft-web` (3.4k), `xtask` (3.4k), `testkit` (0.2k, dev-dependency only) | Desktop app (eframe + wgpu, native menus, control server), CLI (render, commands, calibrate, snapshot, MCP), WASM build, tooling; test helpers (skips the CI report lists) | ## Data model @@ -63,8 +63,19 @@ dispatch by id, and headless snapshots render the UI without a window. `assets` (every asset attributed, no Adobe formats) and the WASM build. Other workflows build and test FreeBSD and Windows ARM64, lint packaging, and build releases. +`ci` also probes the machine: OS, CPUs, RAM, the GPU adapters of each backend (from the ignored +`gpu_probe` test of `lightcraft-gpu`, run after the workspace tests so it reuses their build; xtask +has no wgpu), the corpus and `CRAFT_FONTS_DIR`. Tests +that need what is missing call `lightcraft_testkit::skip!(need, reason)`, which appends a JSON line to +the file `LIGHTCRAFT_SKIP_LOG` names. The run ends with the machine and the skips by need and reason, +and writes `target/ci/report.json` (commit, dirty flag, machine, capabilities, each step's result and +time, skips) and `target/ci/summary.md` (Markdown for a PR; `--summary-md` prints it). The report holds +no host name, user name or absolute path, and nothing is uploaded. `--hardware` adds a step that runs +the `lightcraft-gpu` tests once per backend with a hardware adapter (`LIGHTCRAFT_GPU_BACKEND`). + ## Revision history | Date | Change | Summary | |---|---|---| +| 2026-10-11 | minor | Quality gates: the CI machine probe, visible skips (`testkit`) and `target/ci/report.json` | | 2026-10-10 | major | Created from the code (crate sizes, layers, data model, rendering, control, gates) |