From f6a3a3579f25ac65e2fd22a102b0404d7be79570 Mon Sep 17 00:00:00 2001 From: Mathew Beane Date: Thu, 1 Oct 2026 14:47:33 -0400 Subject: [PATCH] fix(firmware): confirm OTA'd images with a bounded health check Rollback-enabled nodes (CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE=y) boot an OTA'd image as PENDING_VERIFY, but nothing called esp_ota_mark_app_valid_cancel_rollback(), so every OTA reverted at the next reset, a second OTA was refused until then, and a bad image was never deliberately rolled back. - main/ota_health.{c,h}: at boot, if the running image is PENDING_VERIFY, poll for STA got-IP and >=1 CSI frame accepted by the stream sender. Confirm after CONFIG_OTA_HEALTH_MIN_UPTIME_S (30 s) once both are in; roll back via esp_ota_mark_app_invalid_rollback_and_reboot() if they are not in by CONFIG_OTA_HEALTH_TIMEOUT_S (120 s from boot). VALID, UNDEFINED, NEW and no-otadata/factory images are left alone. The decision is a pure header function, host-tested in test/test_ota_health.c (make host_tests). - CONFIG_OTA_HEALTH_FORCE_FAIL (test only) rolls back where it would have confirmed, for proving rollback on hardware. - GET /ota/status reports ota_state so durability is visible remotely. - RUNBOOK: replace the nonexistent ota_rollback_boot_check() claim and document the behaviour and the hardware check (2.1). Hardware-verified on an ESP32-S3 (push, confirm, hard reset, forced rollback) and an ESP32-C6 (push, confirm, hard reset); see ADR-379, next in this series. Host tests: make host_tests passes, including run_ota_health. Co-Authored-By: claude-flow --- firmware/esp32-csi-node/README.md | 7 + firmware/esp32-csi-node/RUNBOOK.md | 46 +++++- firmware/esp32-csi-node/main/CMakeLists.txt | 2 +- .../esp32-csi-node/main/Kconfig.projbuild | 44 ++++++ firmware/esp32-csi-node/main/csi_collector.c | 5 + firmware/esp32-csi-node/main/csi_collector.h | 12 ++ firmware/esp32-csi-node/main/main.c | 7 + firmware/esp32-csi-node/main/ota_health.c | 139 ++++++++++++++++ firmware/esp32-csi-node/main/ota_health.h | 110 +++++++++++++ firmware/esp32-csi-node/main/ota_update.c | 14 +- .../esp32-csi-node/sdkconfig.defaults.16mb | 4 +- firmware/esp32-csi-node/test/Makefile | 21 ++- .../esp32-csi-node/test/test_ota_health.c | 148 ++++++++++++++++++ 13 files changed, 547 insertions(+), 12 deletions(-) create mode 100644 firmware/esp32-csi-node/main/ota_health.c create mode 100644 firmware/esp32-csi-node/main/ota_health.h create mode 100644 firmware/esp32-csi-node/test/test_ota_health.c diff --git a/firmware/esp32-csi-node/README.md b/firmware/esp32-csi-node/README.md index 7448eedae..1625ee567 100644 --- a/firmware/esp32-csi-node/README.md +++ b/firmware/esp32-csi-node/README.md @@ -98,6 +98,12 @@ For an existing provisioned node, back up its current application and inspect `http://DEVICE_IP:8032/ota/status` before choosing an application-only update. Writing only offset `0x20000` is safe only when the status endpoint reports `running_partition` as `ota_0` and the downloaded image matches the board. + +After an OTA, the same endpoint's `ota_state` must read `valid` before you +power-cycle the node or push again. `pending_verify` means the image has not +passed its first-boot health check yet and would revert on reset. See +[RUNBOOK §2.1](RUNBOOK.md) and ADR-379. + The full bundles do not include NVS, so the documented four-offset install preserves WiFi and node configuration while replacing the boot and application images. @@ -632,6 +638,7 @@ cargo build -p wifi-densepose-wasm-edge --target wasm32-unknown-unknown --releas | `main/nvs_config.c` / `.h` | Runtime configuration: loads Kconfig defaults, overrides from NVS | | `main/edge_processing.c` / `.h` | Tier 0-2 DSP pipeline: SPSC ring buffer, biquad IIR filters, Welford stats, BPM extraction, presence, fall detection | | `main/ota_update.c` / `.h` | HTTP OTA firmware update server on port 8032 | +| `main/ota_health.c` / `.h` | First-boot health check that confirms an OTA'd image or rolls it back (ADR-379) | | `main/power_mgmt.c` / `.h` | Battery-aware light sleep duty cycling | | `main/wasm_runtime.c` / `.h` | WASM3 interpreter: module slots, host API bindings, budget guard, per-frame dispatch | | `main/wasm_upload.c` / `.h` | HTTP endpoints for WASM module upload, list, start, stop, delete | diff --git a/firmware/esp32-csi-node/RUNBOOK.md b/firmware/esp32-csi-node/RUNBOOK.md index 2cd103b5c..5603a0949 100644 --- a/firmware/esp32-csi-node/RUNBOOK.md +++ b/firmware/esp32-csi-node/RUNBOOK.md @@ -31,9 +31,9 @@ It does **not** pick up `sdkconfig.defaults.16mb`, and that file is where `CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE=y` lives. Build without it and you get an image that is correct in every visible respect -and **cannot roll back a bad OTA** — while `ota_rollback_boot_check()` sits in -the app expecting the capability. Silent, and only discoverable by pushing a -deliberately bad image. +and **cannot roll back a bad OTA**, while `main/ota_health.c` sits in the app +expecting the capability (it compiles to a no-op without it). Silent, and only +discoverable by pushing a deliberately bad image. See §2.1. That is why the command above concatenates all three explicitly rather than relying on discovery. @@ -104,6 +104,46 @@ Corollary: "every node reports health" proves the **app** is current and proves nothing about the bootloader. Do not answer a bootloader question with app evidence. +### 2.1 An OTA'd image must confirm itself (ADR-379) + +On a rollback-capable bootloader an OTA'd image boots **`PENDING_VERIFY`**. +Until it is confirmed, the **next reset reverts to the previous slot** and +`POST /ota` refuses a second update. `main/ota_health.c` does the confirming: + +- It runs only when the running image is `PENDING_VERIFY`. USB-flashed images + (the bootloader writes `VALID`), already-confirmed images, `UNDEFINED` and + `NEW` are left alone. +- It confirms once the STA has an IP **and** the stream sender has accepted at + least one CSI frame, but not before `CONFIG_OTA_HEALTH_MIN_UPTIME_S` + (default 30 s). Log: `ota_health: marked valid after ... in ms`. +- If both signals have not arrived by `CONFIG_OTA_HEALTH_TIMEOUT_S` (default + 120 s from boot), it logs `health check failed: , rolling back` and + reboots into the previous slot. A crash or reset before confirmation is also + a rollback. +- `GET /ota/status` reports `ota_state`. `pending_verify` means the image is + not yet durable. `valid` means it survives a power cycle. `new` after an OTA + means this board's bootloader has **no** rollback support. + +Consequence for rollouts: wait for `ota_state: "valid"` before pushing again, +and before power-cycling. If the AP is down when the node boots, a good image +rolls back. That is expected; re-push it. + +**Hardware check, one node.** It needs a node with the OTA PSK provisioned and +a rollback bootloader. + +1. Note `version` and `running_partition` from `/ota/status`. +2. OTA-push the new image. Watch the log for `OTA image pending verify`, then + `marked valid after ... in ms`. `/ota/status` must show the new + version, the other partition, and `ota_state: "valid"`. +3. Power-cycle it. It must come back on the same version and partition with + `ota_state: "valid"`. Before ADR-379 it came back on the old one. + +**Negative check.** Build with `CONFIG_OTA_HEALTH_FORCE_FAIL=y` and push it. +The log must show `health check failed: forced failure ..., rolling back` at +about `MIN_UPTIME_S`, and the node must come back on the step-2 version and +partition. Then push a normal image to restore it. Never leave a FORCE_FAIL +image in `release_bins`. + --- ## 3. Deploying diff --git a/firmware/esp32-csi-node/main/CMakeLists.txt b/firmware/esp32-csi-node/main/CMakeLists.txt index 23e70c793..f7b28fb1e 100644 --- a/firmware/esp32-csi-node/main/CMakeLists.txt +++ b/firmware/esp32-csi-node/main/CMakeLists.txt @@ -2,7 +2,7 @@ set(SRCS "main.c" "csi_collector.c" "node_log.c" "thermal.c" "stream_sender.c" "nvs_config.c" "serial_onboarding.c" "serial_onboarding_protocol.c" - "edge_processing.c" "ota_update.c" "power_mgmt.c" + "edge_processing.c" "ota_update.c" "ota_health.c" "power_mgmt.c" "wasm_runtime.c" "wasm_upload.c" "rvf_parser.c" "mmwave_sensor.c" "swarm_bridge.c" diff --git a/firmware/esp32-csi-node/main/Kconfig.projbuild b/firmware/esp32-csi-node/main/Kconfig.projbuild index 0235c2c3d..43ffcb459 100644 --- a/firmware/esp32-csi-node/main/Kconfig.projbuild +++ b/firmware/esp32-csi-node/main/Kconfig.projbuild @@ -693,6 +693,50 @@ config UPLINK_WATCHDOG_TIMEOUT_S endmenu +menu "OTA health check (ADR-379)" + +config OTA_HEALTH_TIMEOUT_S + int "Seconds an OTA'd image has to prove itself before rolling back" + depends on BOOTLOADER_APP_ROLLBACK_ENABLE + range 30 900 + default 120 + help + Only applies when the running image is ESP_OTA_IMG_PENDING_VERIFY, i.e. + it was just delivered by OTA to a node whose bootloader has + CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE. USB-flashed and already-confirmed + images are never checked. + + Healthy means: the STA got an IP, and the stream sender accepted at least + one CSI frame. Both must happen within this many seconds of boot, or the + image is marked invalid and the node reboots into the previous slot. + + Measured from boot, and the WiFi boot wait alone can take about two + minutes against an absent AP, so an AP outage during a rollout rolls a + good image back. That is the safe direction: re-push it later. + +config OTA_HEALTH_MIN_UPTIME_S + int "Minimum uptime before an OTA'd image may be confirmed" + depends on BOOTLOADER_APP_ROLLBACK_ENABLE + range 0 600 + default 30 + help + Holds confirmation back for this long after boot even once both signals + are in, so an image that crashes shortly after starting its pipelines + still reverts (a crash while PENDING_VERIFY is a rollback). Clamped to + OTA_HEALTH_TIMEOUT_S. + +config OTA_HEALTH_FORCE_FAIL + bool "TEST ONLY: fail the OTA health check to exercise rollback" + depends on BOOTLOADER_APP_ROLLBACK_ENABLE + default n + help + Builds an image that rolls itself back at the point it would otherwise + confirm. OTA-push it to a node to prove rollback works end to end: the + node should reboot into the previous image. Has no effect when USB + flashed (such an image is not PENDING_VERIFY). Never ship this. + +endmenu + config CSI_GATE_MESH_ALIGNED bool "Align the CSI rate gate to mesh time (enables cross-node pairing)" default y diff --git a/firmware/esp32-csi-node/main/csi_collector.c b/firmware/esp32-csi-node/main/csi_collector.c index e0666e797..378cfaf13 100644 --- a/firmware/esp32-csi-node/main/csi_collector.c +++ b/firmware/esp32-csi-node/main/csi_collector.c @@ -750,6 +750,11 @@ uint16_t csi_collector_get_send_fail_count(void) return (f > 0xFFFFu) ? 0xFFFFu : (uint16_t)f; } +uint32_t csi_collector_get_send_ok_count(void) +{ + return s_send_ok; +} + /* The gate here is fixed at compile time (mesh-aligned bucketing, falling * back to elapsed time when unsynced), so there is no mode or period to * report. Return the explicit sentinel rather than a number that reads as diff --git a/firmware/esp32-csi-node/main/csi_collector.h b/firmware/esp32-csi-node/main/csi_collector.h index 33fb86934..37934afa4 100644 --- a/firmware/esp32-csi-node/main/csi_collector.h +++ b/firmware/esp32-csi-node/main/csi_collector.h @@ -139,6 +139,18 @@ uint16_t csi_collector_get_pkt_yield_per_sec(void); */ uint16_t csi_collector_get_send_fail_count(void); +/** + * Get the cumulative count of CSI frames the stream sender accepted since boot. + * + * Nonzero means the whole capture path worked at least once: the driver + * delivered a CSI callback, the frame serialized, and sendto() accepted it. + * Used by the OTA health check (ota_health.c) as its "operational" signal. + * + * @return Number of successful stream_sender_send() calls from the CSI + * callback path. + */ +uint32_t csi_collector_get_send_ok_count(void); + /** Reported by the gate accessors when this build has no selectable gate. */ #define CSI_GATE_NOT_CONFIGURABLE 0xFF diff --git a/firmware/esp32-csi-node/main/main.c b/firmware/esp32-csi-node/main/main.c index f235eeba3..10c0e5ed5 100644 --- a/firmware/esp32-csi-node/main/main.c +++ b/firmware/esp32-csi-node/main/main.c @@ -28,6 +28,7 @@ #include "serial_onboarding.h" #include "edge_processing.h" #include "ota_update.h" +#include "ota_health.h" #include "power_mgmt.h" #include "wasm_runtime.h" #include "wasm_upload.h" @@ -142,6 +143,7 @@ static void event_handler(void *arg, esp_event_base_t event_base, ip_event_got_ip_t *event = (ip_event_got_ip_t *)event_data; ESP_LOGI(TAG, "Got IP: " IPSTR, IP2STR(&event->ip_info.ip)); s_retry_num = 0; + ota_health_note_got_ip(); xEventGroupSetBits(s_wifi_event_group, WIFI_CONNECTED_BIT); } } @@ -338,6 +340,11 @@ void app_main(void) ESP_LOGI(TAG, "%s CSI Node (ADR-018 / ADR-110) — v%s — Node ID: %d", target_name, app_desc->version, g_nvs_config.node_id); + /* ADR-379: an OTA'd image boots PENDING_VERIFY and reverts at the next + * reset unless it confirms itself. Start the clock before anything that + * can block (WiFi waits up to ~2 min for an AP), so the deadline is real. */ + ota_health_start(); + /* Apply the opt-in XIAO RF path before WiFi starts. Generic C6 boards keep * ownership of GPIO3 and GPIO14 because the default implementation is a * no-op unless CONFIG_C6_XIAO_ANTENNA_SELECT is enabled. */ diff --git a/firmware/esp32-csi-node/main/ota_health.c b/firmware/esp32-csi-node/main/ota_health.c new file mode 100644 index 000000000..5b190f4f2 --- /dev/null +++ b/firmware/esp32-csi-node/main/ota_health.c @@ -0,0 +1,139 @@ +/* First-boot health check for an OTA-delivered image. See ota_health.h and + * ADR-379 for why it exists; the decision itself is ota_health_step(). */ +#include "ota_health.h" + +#include "sdkconfig.h" + +#ifdef CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE + +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "esp_log.h" +#include "esp_ota_ops.h" +#include "esp_timer.h" + +#include "csi_collector.h" + +static const char *TAG = "ota_health"; + +#define OTA_HEALTH_POLL_MS 1000 + +/* Written from the event loop task, read from the health task. A single + * aligned bool is enough; it only ever goes false -> true. */ +static volatile bool s_got_ip; + +static const char *state_name(esp_ota_img_states_t st) +{ + switch (st) { + case ESP_OTA_IMG_NEW: return "NEW"; + case ESP_OTA_IMG_PENDING_VERIFY: return "PENDING_VERIFY"; + case ESP_OTA_IMG_VALID: return "VALID"; + case ESP_OTA_IMG_INVALID: return "INVALID"; + case ESP_OTA_IMG_ABORTED: return "ABORTED"; + case ESP_OTA_IMG_UNDEFINED: return "UNDEFINED"; + } + return "?"; +} + +static void ota_health_task(void *arg) +{ + (void)arg; + const ota_health_cfg_t cfg = { + .timeout_ms = (uint32_t)CONFIG_OTA_HEALTH_TIMEOUT_S * 1000u, + .min_uptime_ms = (uint32_t)CONFIG_OTA_HEALTH_MIN_UPTIME_S * 1000u, +#ifdef CONFIG_OTA_HEALTH_FORCE_FAIL + .force_fail = true, +#else + .force_fail = false, +#endif + }; + ota_health_sm_t sm; + ota_health_sm_init(&sm); + + for (;;) { + uint32_t now_ms = (uint32_t)(esp_timer_get_time() / 1000); + ota_health_verdict_t v = ota_health_step(&sm, &cfg, s_got_ip, + csi_collector_get_send_ok_count() > 0, + now_ms); + if (v == OTA_HEALTH_MARK_VALID) { + esp_err_t err = esp_ota_mark_app_valid_cancel_rollback(); + if (err == ESP_OK) { + ESP_LOGI(TAG, "marked valid after %s in %lu ms", + sm.reason, (unsigned long)now_ms); + } else { + ESP_LOGE(TAG, "mark valid failed: %s (image stays PENDING_VERIFY " + "and reverts on the next reset)", esp_err_to_name(err)); + } + break; + } + if (v == OTA_HEALTH_ROLLBACK) { + ESP_LOGE(TAG, "health check failed: %s, rolling back (%lu ms)", + sm.reason, (unsigned long)now_ms); + /* Returns only on failure: no other bootable slot. Rebooting then + * would just loop, so stay up; the next reset aborts this image. */ + esp_err_t err = esp_ota_mark_app_invalid_rollback_and_reboot(); + ESP_LOGE(TAG, "rollback not possible: %s -- staying on this image", + esp_err_to_name(err)); + break; + } + vTaskDelay(pdMS_TO_TICKS(OTA_HEALTH_POLL_MS)); + } + vTaskDelete(NULL); +} + +void ota_health_start(void) +{ + const esp_partition_t *running = esp_ota_get_running_partition(); + esp_ota_img_states_t st; + esp_err_t err = esp_ota_get_state_partition(running, &st); + + /* ESP_ERR_NOT_SUPPORTED: running from factory. ESP_ERR_NOT_FOUND: no + * otadata entry for this slot (erased otadata). Neither can be rolled + * back, so there is nothing to confirm. */ + if (err != ESP_OK) { + ESP_LOGI(TAG, "running %s has no OTA state (%s): no health check", + running ? running->label : "?", esp_err_to_name(err)); + return; + } + + /* VALID: USB-flashed with ota_data_initial (the bootloader writes VALID) + * or already confirmed. UNDEFINED: selected by a build without rollback; + * the bootloader boots it unconditionally. NEW: the bootloader has no + * rollback support, so it never promoted the image to PENDING_VERIFY. + * None of these revert on reset. */ + if (st != ESP_OTA_IMG_PENDING_VERIFY) { + ESP_LOGI(TAG, "running %s state %s: no health check", + running->label, state_name(st)); + return; + } + + ESP_LOGW(TAG, "OTA image pending verify on %s: need IP + CSI frame within %d s " + "(min uptime %d s)%s", + running->label, CONFIG_OTA_HEALTH_TIMEOUT_S, + CONFIG_OTA_HEALTH_MIN_UPTIME_S, +#ifdef CONFIG_OTA_HEALTH_FORCE_FAIL + " -- FORCE_FAIL test build, will roll back" +#else + "" +#endif + ); + + if (xTaskCreate(ota_health_task, "ota_health", 4096, NULL, 2, NULL) != pdPASS) { + /* Without the task nothing confirms the image, which is the same as + * failing the check: the next reset reverts it. Say so. */ + ESP_LOGE(TAG, "health task create failed; image will revert on next reset"); + } +} + +void ota_health_note_got_ip(void) +{ + s_got_ip = true; +} + +#else /* !CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE */ + +/* Without bootloader rollback no image is ever PENDING_VERIFY. */ +void ota_health_start(void) {} +void ota_health_note_got_ip(void) {} + +#endif /* CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE */ diff --git a/firmware/esp32-csi-node/main/ota_health.h b/firmware/esp32-csi-node/main/ota_health.h new file mode 100644 index 000000000..7613904df --- /dev/null +++ b/firmware/esp32-csi-node/main/ota_health.h @@ -0,0 +1,110 @@ +/* First-boot health check for an OTA-delivered image (ADR-379). + * + * With CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE the bootloader starts a freshly + * OTA'd image in ESP_OTA_IMG_PENDING_VERIFY. Until the app confirms itself, + * the next reset reverts to the previous slot and esp_ota_begin() refuses a + * second OTA. Nothing confirmed it before this module existed, so every OTA + * on a rollback-enabled node was undone by the next power cycle. + * + * The decision logic is a pure function kept in this header, like + * thermal_next_state(), so the host test exercises the exact code the device + * runs. ota_health.c only feeds it events and a clock and acts on the verdict. */ +#pragma once + +#include +#include + +typedef enum { + OTA_HEALTH_WAIT = 0, /* keep checking */ + OTA_HEALTH_MARK_VALID, /* confirm the image: esp_ota_mark_app_valid_cancel_rollback() */ + OTA_HEALTH_ROLLBACK, /* esp_ota_mark_app_invalid_rollback_and_reboot() */ +} ota_health_verdict_t; + +typedef struct { + uint32_t timeout_ms; /* deadline, measured from boot */ + uint32_t min_uptime_ms; /* earliest the image may be confirmed; clamped to timeout_ms */ + bool force_fail; /* test hook: roll back where it would have confirmed */ +} ota_health_cfg_t; + +typedef struct { + bool got_ip; /* STA associated and holds an IPv4 address (latched) */ + bool csi_delivered; /* stream sender accepted >= 1 CSI frame (latched) */ + ota_health_verdict_t verdict; + const char *reason; /* set once verdict leaves WAIT; static string */ +} ota_health_sm_t; + +static inline void ota_health_sm_init(ota_health_sm_t *sm) +{ + sm->got_ip = false; + sm->csi_delivered = false; + sm->verdict = OTA_HEALTH_WAIT; + sm->reason = "waiting"; +} + +/** + * Advance the health check. + * + * Pure: no globals, no hardware. Both signals latch, because each proves a + * capability the image has, and a later AP outage is not evidence against the + * image. Once the verdict leaves WAIT it is terminal: later calls return it + * unchanged, so a caller that polls after acting cannot act twice. + * + * @param sm State, initialised with ota_health_sm_init(). + * @param cfg Timeout, minimum uptime, force-fail hook. + * @param got_ip Event seen since the last step: STA got an IP. + * @param csi_delivered Event seen since the last step: a CSI frame was sent. + * @param now_ms Milliseconds since boot. + * @return The verdict. sm->reason names why, for the log. + */ +static inline ota_health_verdict_t ota_health_step(ota_health_sm_t *sm, + const ota_health_cfg_t *cfg, + bool got_ip, + bool csi_delivered, + uint32_t now_ms) +{ + if (sm->verdict != OTA_HEALTH_WAIT) { + return sm->verdict; + } + + sm->got_ip = sm->got_ip || got_ip; + sm->csi_delivered = sm->csi_delivered || csi_delivered; + + /* A minimum uptime above the timeout could never be satisfied; that would + * turn every OTA into a rollback. Clamp rather than trust the config. */ + uint32_t min_up = cfg->min_uptime_ms < cfg->timeout_ms + ? cfg->min_uptime_ms : cfg->timeout_ms; + + if (sm->got_ip && sm->csi_delivered && now_ms >= min_up) { + if (cfg->force_fail) { + sm->verdict = OTA_HEALTH_ROLLBACK; + sm->reason = "forced failure (CONFIG_OTA_HEALTH_FORCE_FAIL)"; + } else { + sm->verdict = OTA_HEALTH_MARK_VALID; + sm->reason = "Wi-Fi got IP and CSI frame delivered"; + } + return sm->verdict; + } + + if (now_ms >= cfg->timeout_ms) { + sm->verdict = OTA_HEALTH_ROLLBACK; + if (!sm->got_ip && !sm->csi_delivered) { + sm->reason = "timeout: no IP and no CSI frame delivered"; + } else if (!sm->got_ip) { + sm->reason = "timeout: no IP"; + } else { + sm->reason = "timeout: no CSI frame delivered"; + } + return sm->verdict; + } + + return OTA_HEALTH_WAIT; +} + +/* Device side (ota_health.c). No-ops when the bootloader has no rollback. */ + +/** Call once, early in app_main. Starts the check only if the running image + * is ESP_OTA_IMG_PENDING_VERIFY; otherwise logs the state and returns. */ +void ota_health_start(void); + +/** Call from the IP_EVENT_STA_GOT_IP handler. Cheap; safe from any task. */ +void ota_health_note_got_ip(void); diff --git a/firmware/esp32-csi-node/main/ota_update.c b/firmware/esp32-csi-node/main/ota_update.c index 410ca00b1..2f0f4eb37 100644 --- a/firmware/esp32-csi-node/main/ota_update.c +++ b/firmware/esp32-csi-node/main/ota_update.c @@ -90,14 +90,26 @@ static esp_err_t ota_status_handler(httpd_req_t *req) const esp_partition_t *running = esp_ota_get_running_partition(); const esp_partition_t *update = esp_ota_get_next_update_partition(NULL); + /* ADR-379: lets an operator confirm remotely that an OTA'd image left + * pending_verify (and so survives a power cycle) without a serial log. */ + esp_ota_img_states_t st; + const char *ota_state = "none"; + if (running && esp_ota_get_state_partition(running, &st) == ESP_OK) { + ota_state = st == ESP_OTA_IMG_VALID ? "valid" : + st == ESP_OTA_IMG_PENDING_VERIFY ? "pending_verify" : + st == ESP_OTA_IMG_NEW ? "new" : + st == ESP_OTA_IMG_UNDEFINED ? "undefined" : "other"; + } + char response[512]; int len = snprintf(response, sizeof(response), "{\"version\":\"%s\",\"date\":\"%s\",\"time\":\"%s\"," "\"running_partition\":\"%s\",\"next_partition\":\"%s\"," - "\"max_size\":%lu}", + "\"ota_state\":\"%s\",\"max_size\":%lu}", app->version, app->date, app->time, running ? running->label : "unknown", update ? update->label : "none", + ota_state, (unsigned long)(update ? update->size : 0)); httpd_resp_set_type(req, "application/json"); diff --git a/firmware/esp32-csi-node/sdkconfig.defaults.16mb b/firmware/esp32-csi-node/sdkconfig.defaults.16mb index 124146e42..5f13b3c16 100644 --- a/firmware/esp32-csi-node/sdkconfig.defaults.16mb +++ b/firmware/esp32-csi-node/sdkconfig.defaults.16mb @@ -18,8 +18,8 @@ CONFIG_PARTITION_TABLE_CUSTOM=y CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_16mb.csv" # Firmware rollback: an OTA'd image boots PENDING_VERIFY and must confirm -# itself, or the bootloader reverts on the next boot. See ota_rollback_* in -# ota_update.c for when confirmation happens and why. +# itself, or the bootloader reverts on the next boot. main/ota_health.c does +# the confirming (IP + one CSI frame within OTA_HEALTH_TIMEOUT_S), ADR-379. CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE=y # Coredump to flash. The 64K `coredump` partition has existed in diff --git a/firmware/esp32-csi-node/test/Makefile b/firmware/esp32-csi-node/test/Makefile index ee32cc02a..12a7098c5 100644 --- a/firmware/esp32-csi-node/test/Makefile +++ b/firmware/esp32-csi-node/test/Makefile @@ -56,10 +56,11 @@ FUZZ_JOBS ?= 1 test_vitals run_vitals test_mmwave_detect run_mmwave_detect host_tests \ test_thermal run_thermal test_csi_sanitize run_csi_sanitize \ test_c6_antenna run_c6_antenna test_serial_onboarding run_serial_onboarding \ - run_delivery_contract + run_delivery_contract test_ota_health run_ota_health all: fuzz_serialize fuzz_edge fuzz_nvs test_adr110 test_vitals test_mmwave_detect \ - test_thermal test_csi_sanitize test_c6_antenna test_serial_onboarding + test_thermal test_csi_sanitize test_c6_antenna test_serial_onboarding \ + test_ota_health # --- XIAO ESP32-C6 RF switch selection --- # Host-side truth table for the GPIO14 antenna selector. GPIO3 is always driven @@ -133,12 +134,21 @@ test_serial_onboarding: test_serial_onboarding_protocol.c $(MAIN_DIR)/serial_onb run_serial_onboarding: test_serial_onboarding ./test_serial_onboarding +# --- OTA first-boot health check (ADR-379) --- +# Host-side. Exercises ota_health_step() from ../main/ota_health.h directly -- +# the decision the device runs to confirm or roll back an OTA'd image. +test_ota_health: test_ota_health.c $(MAIN_DIR)/ota_health.h + cc -std=c99 -Wall -Wextra -Werror -I$(MAIN_DIR) -o $@ $< + +run_ota_health: test_ota_health + ./test_ota_health + run_delivery_contract: python3 test_delivery_contract.py host_tests: run_adr110 run_vitals run_mmwave_detect run_thermal run_csi_sanitize \ - run_c6_antenna run_serial_onboarding run_delivery_contract - @echo "Host tests passed (ADR-110 + CSI sanitation + vitals + mmwave + thermal + C6 antenna + USB onboarding + delivery contract)" + run_c6_antenna run_serial_onboarding run_delivery_contract run_ota_health + @echo "Host tests passed (ADR-110 + CSI sanitation + vitals + mmwave + thermal + C6 antenna + USB onboarding + delivery contract + OTA health)" # --- Serialize fuzzer --- # Tests csi_serialize_frame() with random wifi_csi_info_t inputs. @@ -175,5 +185,6 @@ run_all: run_serialize run_edge run_nvs clean: rm -f fuzz_serialize fuzz_edge fuzz_nvs test_adr110 test_vitals test_mmwave_detect \ - test_thermal test_csi_sanitize test_c6_antenna test_serial_onboarding + test_thermal test_csi_sanitize test_c6_antenna test_serial_onboarding \ + test_ota_health rm -rf corpus_serialize/ corpus_edge/ corpus_nvs/ diff --git a/firmware/esp32-csi-node/test/test_ota_health.c b/firmware/esp32-csi-node/test/test_ota_health.c new file mode 100644 index 000000000..f1325a9ea --- /dev/null +++ b/firmware/esp32-csi-node/test/test_ota_health.c @@ -0,0 +1,148 @@ +/** + * @file test_ota_health.c + * @brief Pins the OTA first-boot health check decision (ADR-379). + * + * Exercises `ota_health_step()` from ../main/ota_health.h directly -- the same + * function the device build runs, not a copy -- so the test and the firmware + * cannot drift apart. + * + * The regression that motivated this: nothing ever called + * esp_ota_mark_app_valid_cancel_rollback(), so on rollback-enabled nodes every + * OTA'd image reverted at the next reset, and a bad image was never caught. + * + * Config under test: timeout 120 s, minimum uptime 30 s. + */ + +#include +#include + +#include "ota_health.h" + +#define TIMEOUT_MS 120000u +#define MIN_UP_MS 30000u + +static int g_failures = 0; + +static const char *name(ota_health_verdict_t v) +{ + switch (v) { + case OTA_HEALTH_WAIT: return "WAIT"; + case OTA_HEALTH_MARK_VALID: return "MARK_VALID"; + case OTA_HEALTH_ROLLBACK: return "ROLLBACK"; + } + return "?"; +} + +static void expect(ota_health_verdict_t got, ota_health_verdict_t want, + const char *why) +{ + if (got != want) { + printf(" FAIL: got %s, expected %s\n %s\n", name(got), name(want), why); + g_failures++; + } +} + +static void expect_reason(const ota_health_sm_t *sm, const char *needle, + const char *why) +{ + if (strstr(sm->reason, needle) == NULL) { + printf(" FAIL: reason \"%s\" lacks \"%s\"\n %s\n", + sm->reason, needle, why); + g_failures++; + } +} + +static const ota_health_cfg_t CFG = { TIMEOUT_MS, MIN_UP_MS, false }; + +int main(void) +{ + ota_health_sm_t sm; + + printf("ota health check (timeout %u ms, min uptime %u ms)\n", + TIMEOUT_MS, MIN_UP_MS); + + /* Healthy boot: IP at 4 s, first CSI frame at 6 s, confirmed at 30 s. */ + ota_health_sm_init(&sm); + expect(ota_health_step(&sm, &CFG, false, false, 1000), OTA_HEALTH_WAIT, + "nothing yet, well inside the deadline"); + expect(ota_health_step(&sm, &CFG, true, false, 4000), OTA_HEALTH_WAIT, + "IP alone is not enough"); + expect(ota_health_step(&sm, &CFG, false, true, 6000), OTA_HEALTH_WAIT, + "both signals in, but below the minimum uptime"); + expect(ota_health_step(&sm, &CFG, false, false, 29999), OTA_HEALTH_WAIT, + "one ms short of the minimum uptime"); + expect(ota_health_step(&sm, &CFG, false, false, 30000), OTA_HEALTH_MARK_VALID, + "latched IP + CSI at the minimum uptime must confirm the image"); + expect_reason(&sm, "IP", "the log names the signals that passed"); + + /* Terminal: a verdict never changes, even past the deadline. */ + expect(ota_health_step(&sm, &CFG, false, false, 500000), OTA_HEALTH_MARK_VALID, + "a confirmed image must not later be rolled back by the same check"); + + /* Signals arriving after the minimum uptime confirm immediately. */ + ota_health_sm_init(&sm); + expect(ota_health_step(&sm, &CFG, true, true, 45000), OTA_HEALTH_MARK_VALID, + "both signals after min uptime -> confirm now"); + + /* No IP by the deadline. */ + ota_health_sm_init(&sm); + expect(ota_health_step(&sm, &CFG, false, false, 119999), OTA_HEALTH_WAIT, + "one ms before the deadline"); + expect(ota_health_step(&sm, &CFG, false, false, 120000), OTA_HEALTH_ROLLBACK, + "no signals at the deadline -> roll back"); + expect_reason(&sm, "no IP and no CSI", "both missing"); + + /* IP but the CSI pipeline never delivers. */ + ota_health_sm_init(&sm); + ota_health_step(&sm, &CFG, true, false, 5000); + expect(ota_health_step(&sm, &CFG, false, false, 120000), OTA_HEALTH_ROLLBACK, + "associated but no CSI frame -> roll back"); + expect_reason(&sm, "no CSI frame", "names the missing CSI signal"); + + /* CSI sent (can happen only with IP in practice) but no IP event seen. */ + ota_health_sm_init(&sm); + ota_health_step(&sm, &CFG, false, true, 5000); + expect(ota_health_step(&sm, &CFG, false, false, 130000), OTA_HEALTH_ROLLBACK, + "no IP -> roll back"); + expect_reason(&sm, "no IP", "names the missing IP signal"); + + /* Rollback is terminal too. */ + expect(ota_health_step(&sm, &CFG, true, true, 131000), OTA_HEALTH_ROLLBACK, + "late signals must not resurrect a rolled-back verdict"); + + /* Force-fail hook rolls back exactly where it would have confirmed. */ + { + const ota_health_cfg_t ff = { TIMEOUT_MS, MIN_UP_MS, true }; + ota_health_sm_init(&sm); + expect(ota_health_step(&sm, &ff, true, true, 10000), OTA_HEALTH_WAIT, + "force-fail still waits for the minimum uptime"); + expect(ota_health_step(&sm, &ff, false, false, 30000), OTA_HEALTH_ROLLBACK, + "force-fail must roll back instead of confirming"); + expect_reason(&sm, "FORCE_FAIL", "names the test hook"); + } + + /* Minimum uptime above the timeout is clamped, not a guaranteed rollback. */ + { + const ota_health_cfg_t bad = { 60000u, 600000u, false }; + ota_health_sm_init(&sm); + expect(ota_health_step(&sm, &bad, true, true, 59999), OTA_HEALTH_WAIT, + "clamped min uptime: still waiting just before it"); + expect(ota_health_step(&sm, &bad, false, false, 60000), OTA_HEALTH_MARK_VALID, + "healthy at the deadline must confirm, not roll back"); + } + + /* Zero minimum uptime confirms on the first step with both signals. */ + { + const ota_health_cfg_t zero = { TIMEOUT_MS, 0u, false }; + ota_health_sm_init(&sm); + expect(ota_health_step(&sm, &zero, true, true, 0), OTA_HEALTH_MARK_VALID, + "no soak configured -> confirm immediately"); + } + + if (g_failures) { + printf("FAILED: %d case(s)\n", g_failures); + return 1; + } + printf("OK: all OTA health cases pass\n"); + return 0; +}