ava1: release the console page's finished jobs once it has seen them

ava1_jobs_release_seen(me): every finished operation or copy owned by
the console page keeps only the grace (AVA1_OP_GRACE_MS) after the page
first read it finished, instead of a copy holding one of the 32 shared
job slots for the full 10-minute park age. Operations reuse
ava1_op_status_delivered; copies are restamped and the reaper keeps a
delivered copy only for the grace. A paired PC's jobs are never touched.

ava1-ctest: the 32-job cap with list_all (full table refuses a 33rd for
the page and a PC; releasing the page's own makes room, PC jobs stay)
and the grace (page jobs gone after ~10 s, PC op and copy still listed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RxCGb8Asp1A1AQYiv1kbgS

engine/crates/ava1-ctest/src/lib.rs

@@ -2507,6 +2507,42 @@ extern "C" {
     fn ava1_jobs_list_all(me: *const u8, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int;
     fn ava1_jobs_owner_is(id: *const u8, me: *const u8) -> c_int;
     fn ava1_job_cancel_any(id: *const u8) -> c_int;
+    fn ava1_jobs_release_seen(me: *const u8);
+    fn ava1_server_set_rpc_peer(peer: *const u8);
+    fn ava1_data_rpc(
+        method: u16,
+        body: *const u8,
+        len: u32,
+        out: *mut u8,
+        cap: usize,
+        out_len: *mut usize,
+    ) -> c_int;
+}
+
+/// `ava1_jobs_release_seen` (the console page read /api/jobs): `me`'s finished operations and
+/// copies keep only the grace.
+pub fn c_jobs_release_seen(me: &[u8; 32]) {
+    unsafe { ava1_jobs_release_seen(me.as_ptr()) }
+}
+
+/// A data-layer RPC (`job.copy`, `job.run`, ...) as `me`, the way the console page's routes call
+/// it (`ava1_server_set_rpc_peer` on the calling thread, then the data plane): (status, reply).
+pub fn c_data_rpc_as(me: &[u8; 32], method: u16, body: &[u8]) -> (i32, Vec<u8>) {
+    let mut out = vec![0u8; 64 * 1024];
+    let mut n = 0usize;
+    let rc = unsafe {
+        ava1_server_set_rpc_peer(me.as_ptr());
+        ava1_data_rpc(
+            method,
+            body.as_ptr(),
+            body.len() as u32,
+            out.as_mut_ptr(),
+            out.len(),
+            &mut n,
+        )
+    };
+    out.truncate(n);
+    (rc, out)
 }
 
 /// `ava1_jobs_list_all` (the console page's job list): every job the data layer holds, of

engine/crates/ava1-ctest/tests/jobs_any_owner.rs

@@ -184,3 +184,167 @@ async fn job_list_rpc_unchanged_for_pcs() {
     assert_eq!(all, vec![ja, jb]);
     drop(t.srv);
 }
+
+const FSCK_NOW: &[u8] = br#"{"device":"/dev/md1","loops":0}"#;
+
+fn fsck_body(job: [u8; 16]) -> Vec<u8> {
+    JobRun {
+        job_id: job,
+        op: gen::JOB_OP_FSCK,
+        args: FSCK_NOW.to_vec(),
+    }
+    .to_bytes()
+    .unwrap()
+}
+
+/// `job.run` of a quick fsck as the console page (the page's routes call the data layer with its
+/// identity set on the thread): the status.
+fn run_as_console(job: [u8; 16]) -> i32 {
+    c_data_rpc_as(&CONSOLE, gen::METHOD_JOB_RUN, &fsck_body(job)).0
+}
+
+/// `job.copy` as the console page.
+fn copy_as_console(job: [u8; 16], src: &std::path::Path, dest: &std::path::Path) -> i32 {
+    let body = JobCopy {
+        job_id: job,
+        src: src.to_str().unwrap().into(),
+        dest: dest.to_str().unwrap().into(),
+        flags: 0,
+    };
+    c_data_rpc_as(&CONSOLE, gen::METHOD_JOB_COPY, &body.to_bytes().unwrap()).0
+}
+
+/// Waits until every job in `ids` is listed and finished.
+async fn all_finished(ids: &[[u8; 16]]) {
+    for _ in 0..400 {
+        let all = list_all(&CONSOLE);
+        if ids
+            .iter()
+            .all(|id| all.iter().any(|e| e.job_id == *id && e.state != 0))
+        {
+            return;
+        }
+        tokio::time::sleep(Duration::from_millis(25)).await;
+    }
+    panic!("jobs did not finish");
+}
+
+/// Final review: the 32-slot job table is shared by every owner. Finished jobs hold their slot
+/// until they are released; the console page lists all 32 and, once it has read them finished,
+/// releases only its own, which frees room for new jobs.
+#[tokio::test(flavor = "multi_thread")]
+async fn table_holds_32_jobs_and_seen_console_jobs_make_room() {
+    let t = two("any-cap32", 0).await;
+    let mut pc = Vec::new();
+    let mut mine = Vec::new();
+    for n in 0..32u8 {
+        let id = [0x40 + n; 16];
+        // At most eight operations run at once: a quick one is retried while the others end.
+        let mut st = gen::ERR_BUSY as i32;
+        for _ in 0..200 {
+            st = if n % 2 == 0 {
+                run_fsck_now(&t.a, id).await as i32
+            } else {
+                run_as_console(id)
+            };
+            if st != gen::ERR_BUSY as i32 {
+                break;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+        assert_eq!(st, gen::STATUS_OK as i32, "job {n}");
+        if n % 2 == 0 {
+            pc.push(id)
+        } else {
+            mine.push(id)
+        }
+    }
+    let every: Vec<_> = pc.iter().chain(mine.iter()).copied().collect();
+    all_finished(&every).await;
+    // list_all fits all 32 in one reply, whoever owns them.
+    let all = list_all(&CONSOLE);
+    assert_eq!(all.len(), 32);
+    for id in &mine {
+        assert_eq!(c_jobs_owner_is(id, &CONSOLE), 1);
+    }
+    // Full, and nothing was delivered: a 33rd job is refused, for the page as for a PC.
+    assert_eq!(run_as_console([0x70; 16]), gen::ERR_BUSY as i32);
+    assert_eq!(run_fsck_now(&t.b, [0x71; 16]).await, gen::ERR_BUSY);
+    // The page read them finished: its own are released (oldest first when room is needed) ...
+    c_jobs_release_seen(&CONSOLE);
+    assert_eq!(list_all(&CONSOLE).len(), 32, "the grace keeps them listed");
+    assert_eq!(run_as_console([0x70; 16]), gen::STATUS_OK as i32);
+    assert_eq!(run_fsck_now(&t.b, [0x71; 16]).await, gen::STATUS_OK);
+    // ... and a PC's job never is.
+    let all = list_all(&CONSOLE);
+    assert_eq!(all.len(), 32);
+    for id in &pc {
+        assert!(
+            all.iter().any(|e| e.job_id == *id),
+            "a PC's job stays listed"
+        );
+    }
+    drop(t.srv);
+}
+
+/// A finished copy or operation the console page started is released a grace after the page first
+/// read it finished (not the 10-minute park age); a paired PC's finished jobs stay listed.
+#[tokio::test(flavor = "multi_thread")]
+async fn seen_console_jobs_leave_after_the_grace_pc_jobs_stay() {
+    let t = two("any-seen", 0).await;
+    write_tree(&t.d.join("usb/g"), 5, |_| 1024);
+    let (my_op, my_copy, pc_op, pc_copy) = ([0x81; 16], [0x82; 16], [0x83; 16], [0x84; 16]);
+    assert_eq!(run_as_console(my_op), gen::STATUS_OK as i32);
+    assert_eq!(
+        copy_as_console(my_copy, &t.d.join("usb/g"), &t.d.join("data/mine")),
+        gen::STATUS_OK as i32
+    );
+    assert_eq!(run_fsck_now(&t.a, pc_op).await, gen::STATUS_OK);
+    assert_eq!(
+        copy(&t.a, pc_copy, &t.d.join("usb/g"), &t.d.join("data/pc")).await,
+        gen::STATUS_OK
+    );
+    all_finished(&[my_op, my_copy, pc_op, pc_copy]).await;
+    assert_eq!(c_jobs_owner_is(&my_copy, &CONSOLE), 1);
+    c_jobs_release_seen(&CONSOLE);
+    let started = std::time::Instant::now();
+    let listed = |id: [u8; 16]| list_all(&CONSOLE).iter().any(|e| e.job_id == id);
+    assert!(
+        listed(my_op) && listed(my_copy),
+        "still listed during the grace"
+    );
+    // The page reads /api/jobs about once a second.
+    let mut gone = false;
+    while started.elapsed() < Duration::from_secs(20) {
+        c_jobs_release_seen(&CONSOLE);
+        if !listed(my_op) && !listed(my_copy) {
+            gone = true;
+            break;
+        }
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+    assert!(
+        gone,
+        "the page's finished jobs are released after the grace"
+    );
+    assert!(
+        started.elapsed() >= Duration::from_secs(9),
+        "not before the grace: {:?}",
+        started.elapsed()
+    );
+    assert!(listed(pc_op), "a PC's finished operation stays");
+    assert!(listed(pc_copy), "a PC's finished copy stays");
+    // The PC still reads its own results.
+    assert_eq!(
+        code(&t.a, gen::METHOD_JOB_STATUS, pc_copy).await,
+        gen::STATUS_OK
+    );
+    drop(t.srv);
+}
+
+async fn run_fsck_now(s: &Session, job: [u8; 16]) -> u16 {
+    s.rpc(gen::METHOD_JOB_RUN, &fsck_body(job))
+        .await
+        .unwrap()
+        .status
+}