Phase 7a: dual OTA partitions + signed, self-updating firmware

.github/workflows/ci.yml

@@ -2,8 +2,14 @@ name: CI
 
 on:
   push:
+    branches: [main]
+    tags: ['v*']
   pull_request:
 
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   firmware-build:
     name: Build firmware (ESP32-S3)
@@ -30,6 +36,24 @@ jobs:
           constexpr const char* ADMIN_PASS = "ci-placeholder-password";
           EOF
 
+      - name: Write OTA signing key
+        env:
+          OTA_SIGNING_KEY_PEM: ${{ secrets.OTA_SIGNING_KEY_PEM }}
+        run: |
+          if [ -z "$OTA_SIGNING_KEY_PEM" ]; then
+            echo "::error::OTA_SIGNING_KEY_PEM secret is empty or unset. Generate a key with" \
+                 "'espsecure.py generate_signing_key --version 2 --scheme ecdsa256 secure_boot_signing_key.pem'" \
+                 "and paste its full contents into the repo's OTA_SIGNING_KEY_PEM secret (Settings > Secrets and variables > Actions)."
+            exit 1
+          fi
+          echo "$OTA_SIGNING_KEY_PEM" > secure_boot_signing_key.pem
+          if ! grep -q "BEGIN EC PRIVATE KEY" secure_boot_signing_key.pem; then
+            echo "::error::secure_boot_signing_key.pem does not look like an ECDSA key (missing 'BEGIN EC PRIVATE KEY')." \
+                 "Regenerate with --scheme ecdsa256 — this project's sdkconfig expects ECDSA, not the RSA key" \
+                 "espsecure.py generates by default without that flag."
+            exit 1
+          fi
+
       - name: Build firmware
         run: |
           . "$IDF_PATH/export.sh"
@@ -83,6 +107,24 @@ jobs:
           constexpr const char* ADMIN_PASS = "ci-placeholder-password";
           EOF
 
+      - name: Write OTA signing key
+        env:
+          OTA_SIGNING_KEY_PEM: ${{ secrets.OTA_SIGNING_KEY_PEM }}
+        run: |
+          if [ -z "$OTA_SIGNING_KEY_PEM" ]; then
+            echo "::error::OTA_SIGNING_KEY_PEM secret is empty or unset. Generate a key with" \
+                 "'espsecure.py generate_signing_key --version 2 --scheme ecdsa256 secure_boot_signing_key.pem'" \
+                 "and paste its full contents into the repo's OTA_SIGNING_KEY_PEM secret (Settings > Secrets and variables > Actions)."
+            exit 1
+          fi
+          echo "$OTA_SIGNING_KEY_PEM" > secure_boot_signing_key.pem
+          if ! grep -q "BEGIN EC PRIVATE KEY" secure_boot_signing_key.pem; then
+            echo "::error::secure_boot_signing_key.pem does not look like an ECDSA key (missing 'BEGIN EC PRIVATE KEY')." \
+                 "Regenerate with --scheme ecdsa256 — this project's sdkconfig expects ECDSA, not the RSA key" \
+                 "espsecure.py generates by default without that flag."
+            exit 1
+          fi
+
       - name: Build firmware
         run: |
           . "$IDF_PATH/export.sh"
@@ -96,6 +138,7 @@ jobs:
             build/bootloader/bootloader.bin
             build/partition_table/partition-table.bin
             build/mini_dns.bin
+            build/ota_data_initial.bin
           body: |
             Prebuilt binaries for `${{ github.ref_name }}`.
 
@@ -108,8 +151,16 @@ jobs:
             3. `idf.py set-target esp32s3 && idf.py build`
             4. `idf.py -p <PORT> flash monitor`
 
+            These binaries are signed with this repository's CI signing key. A
+            device will only accept a signed OTA image whose signature matches
+            the public key embedded in its *currently running* firmware —
+            since the Setup instructions above have you generate your own
+            signing key, a locally-built device will reject these CI-signed
+            release assets over OTA. Fresh-flash via the esptool command below
+            works regardless of signing.
+
             Flash command for these prebuilt binaries (placeholder credentials —
             device will boot but will not join your Wi-Fi):
             ```
-            python -m esptool --chip esp32s3 -b 460800 --before default_reset --after hard_reset write_flash --flash_mode dio --flash_size 16MB --flash_freq 80m 0x0 bootloader.bin 0x8000 partition-table.bin 0x20000 mini_dns.bin
+            python -m esptool --chip esp32s3 -b 460800 --before default_reset --after hard_reset write_flash --flash_mode dio --flash_size 16MB --flash_freq 80m 0x0 bootloader.bin 0x8000 partition-table.bin 0x19000 ota_data_initial.bin 0x30000 mini_dns.bin
             ```

.gitignore

@@ -1,5 +1,6 @@
 main/wifi_credentials.h
 main/admin_credentials.h
+secure_boot_signing_key.pem
 
 build/
 managed_components/

ARCHITECTURE.md

@@ -4,7 +4,7 @@
 
 `mini_dns` is ESP32-S3 firmware moving from proof-of-concept toward a marketable "edge DNS" appliance: a single device that connects to Wi-Fi, serves a runtime-managed set of hostnames authoritatively (A and/or AAAA), forwards everything else to an upstream recursive resolver with TTL caching and secondary-upstream failover, sinkholes ad/tracker domains, exposes Prometheus metrics, advertises itself via mDNS, and serves an HTTP page + a JSON CRUD API for managing that record table. It was built incrementally — Wi-Fi → raw UDP → DNS parsing → single-record response → multi-record + NXDOMAIN → HTTP server → JSON API → page wired to the API → **forwarding resolver + cache (Phase 1)** → **NVS-backed ad-block (Phase 2)** → **Prometheus `/metrics` (Phase 3)** → **mDNS responder (Phase 4)** → **record management: persistence + CRUD API + auth (Phase 5)** → **hardening: dual-stack records + secondary upstream + host tests (Phase 6)** — each step flashed and confirmed on real hardware before moving on.
 
-**As of Phase 5, records are runtime-managed** — persisted in NVS, editable via `POST`/`PUT`/`DELETE /api/records` (Basic-auth-gated), no longer a reflash-only compile-time table. **As of Phase 6, a record can hold an A, an AAAA, or both**, and a forwarded query that times out against the primary upstream gets one retry against a secondary before SERVFAIL. There is still no provisioning UI, no OTA, no TLS. If you're extending this, read the Non-Goals section before adding anything that smells like a "real" feature.
+**As of Phase 5, records are runtime-managed** — persisted in NVS, editable via `POST`/`PUT`/`DELETE /api/records` (Basic-auth-gated), no longer a reflash-only compile-time table. **As of Phase 6, a record can hold an A, an AAAA, or both**, and a forwarded query that times out against the primary upstream gets one retry against a secondary before SERVFAIL. There is still no provisioning UI, no TLS. If you're extending this, read the Non-Goals section before adding anything that smells like a "real" feature.
 
 ## Target hardware / toolchain
 
@@ -140,7 +140,6 @@ These are the things most likely to confuse future-you or bite an extension:
 These were ruled out deliberately, not overlooked — don't reintroduce them without reopening the scoping conversation:
 
 - Wi-Fi provisioning UI / captive portal
-- OTA updates
 - TLS/HTTPS — Basic auth (Phase 5) runs over plaintext HTTP; see the gotcha above
 - NVS-stored/rotatable admin credentials, rate limiting on auth failures — see the Phase 5 design doc's Open threads
 - True iterative/recursive DNS resolution (root-server walking) — forwarding + caching was built instead (Phase 1); see the design doc for the rationale
@@ -159,7 +158,9 @@ Roughly ordered by how naturally each extends the current design, not by priorit
 4. ~~**mDNS responder.**~~ **Done (Phase 4).** Device self-advertisement as `edge-dns.local` (A + AAAA) plus an `_http._tcp` service via ESP-IDF's `mdns` managed component — a genuinely different mechanism from the unicast resolver, running alongside it rather than replacing it. Does not delegate `DNS_RECORDS` as `.local` hosts (see the `.local` gotcha above and the design doc's Open threads). See `docs/superpowers/specs/2026-07-21-edge-dns-phase4-mdns-design.md`.
 5. ~~**Record management: NVS-backed persistence + a real add/edit/delete API + basic auth.**~~ **Done (Phase 5).** Scoped as one phase, not three, because the pieces are load-bearing for each other: `POST`/`PUT`/`DELETE` on `/api/records` is worthless without persistence behind it, and shipping either without auth would leave a mutating, unauthenticated endpoint exposed on the LAN — so none of the three shipped without the other two. Also the single biggest architectural jump so far: `DnsRecordStore` is the first cross-task state that's genuinely shared *and* mutable, guarded by a `std::mutex` — every prior phase (blocklist, metrics) avoided this by keeping cross-task state either immutable-after-boot or a single atomic word. Ships a JSON CRUD API only, no bundled frontend — the record-management UI is a separate project. See the concurrency-model section above and `docs/superpowers/specs/2026-07-21-edge-dns-phase5-record-management-design.md`.
 6. ~~**Hardening & reliability (Phase 6): AAAA record support for local-table names + host-side unit tests for the wire-format functions + a secondary/failover upstream resolver.**~~ **Done (Phase 6).** Bundled because none of the three touch the concurrency model or add new attack surface — unlike Phase 5, this phase rounds out correctness and resilience of what Phases 1–4 already shipped, rather than adding a new capability surface. `DnsRecordEntry` now holds an optional IPv4 *and* optional IPv6 address (never neither); `find()` returns the whole entry so the DNS task can tell "name exists, wrong family" (NODATA) apart from "name doesn't exist" (NXDOMAIN) — forwarded/cached AAAA for *non*-local names already worked as of Phase 1 (the forwarder and cache are qtype-agnostic). The pure functions in `dns_wire.cpp` are covered by a Unity suite building against ESP-IDF's `linux` target (`host_test/`, no board/QEMU needed). A slot whose primary-upstream attempt times out is retried once against a secondary before SERVFAIL, reusing the same transaction ID/socket. See `docs/superpowers/specs/2026-07-21-edge-dns-phase6-hardening-design.md`.
-7. **Leaving the dev bench (Phase 7): Wi-Fi provisioning (BLE/captive portal) + OTA updates.** Both explicit non-goals today. Sized as its own phase rather than folded into Phase 5 or 6 — this is the step that turns the device from "flash it from a laptop on the desk" into something that could plausibly leave the bench, and it doesn't share a dependency with either of the other two phases.
+7. ~~**Leaving the bench, part 1 (Phase 7a): dual OTA partitions + signed, self-updating firmware.**~~ **Done (Phase 7a).** `otadata`/`ota_0`/`ota_1` replace the single `factory` partition (see partitions.csv); a background task in `ota_updater.cpp` polls GitHub Releases every 6 hours (or on demand via `POST /api/ota/check`), downloads via `esp_https_ota()`, and only cancels the bootloader's rollback after a real health check (30s uptime + at least one DNS query answered, or 10 minutes uptime regardless of traffic) — not unconditionally at boot. Until that gate passes, `esp_ota_begin()` itself refuses new OTA attempts, so the update endpoint rejects with 409 rather than failing mid-download. Signed with `CONFIG_SECURE_SIGNED_APPS_NO_SECURE_BOOT` (software-checked signature) rather than full Secure Boot v2, which would burn eFuses irreversibly. `GET /api/ota` / `POST /api/ota/check` expose status and a manual trigger.
+8. **Leaving the bench, part 2 (Phase 7b): Wi-Fi provisioning via SoftAP captive portal.** Still an explicit non-goal today (see above). Split from Phase 7a because the two share no dependency beyond the one-time partition-table rewrite Phase 7a already did.
+9. **Reliability & forensics (Phase 7c): coredump-to-flash, task watchdog, heap/uptime/reset-reason metrics.** Also split from 7a/7b for the same reason — independent of both beyond the partition table.
 
 Lower-value, opportunistic — worth doing if a specific need arises, not currently phase-scoped:
 - **Multiple questions per query (`qdcount > 1`)** — currently only the first question is parsed; real resolvers essentially never send more than one.

README.md

@@ -86,6 +86,25 @@ dig @<esp32-ip> AAAA dual.loc          # answered locally, not forwarded
 dig @<esp32-ip> AAAA foo.loc           # v4-only record: NOERROR, no answer (NODATA) — not NXDOMAIN
 ```
 
+## Monitoring
+
+A Prometheus + Grafana stack for the `/metrics` endpoint lives in
+[`monitoring/`](monitoring/) (docker-compose, scrape config, provisioned
+Grafana dashboard) — see `monitoring/README.md` for setup.
+
+![Grafana dashboard showing query breakdown, upstream health, latency percentiles, and saturation](docs/grafana-dashboard.png)
+
+## OTA updates
+
+The device checks GitHub Releases for a newer tagged version every 6 hours, and can also be triggered on demand, then updates itself over HTTPS (Phase 7a):
+
+```
+curl http://<esp32-ip>/api/ota                              # current status
+curl -u admin:<your-password> -X POST http://<esp32-ip>/api/ota/check   # trigger a check now
+```
+
+A newly-flashed or newly-updated image stays in "pending_verify" until it's been up ~30s and answered at least one DNS query, or until 10 minutes have passed regardless of traffic (so an idle-but-healthy device isn't stuck forever) — only then does it cancel the bootloader's rollback, and only then will it accept another OTA check (an attempt while still pending_verify is rejected with 409, matching how `esp_ota_begin()` itself refuses to start an update on an unverified image). If it crashes before the gate passes, the bootloader reverts to the previous image on next reset.
+
 ## Running host tests
 
 The pure DNS wire-format functions (`main/dns_wire.h/.cpp`) have no FreeRTOS/lwIP
@@ -126,3 +145,5 @@ exist as a CI-verified reference build, not a flash-and-go artifact.
 - **Metrics run for the life of the device.** `/metrics` counters reset only on reboot — there's no zero/reset endpoint.
 - **Basic auth runs over plaintext HTTP.** There's no TLS on this device, so credentials for the mutating `/api/records` routes are base64-encoded, not encrypted. Fine on a trusted LAN, not a real security boundary.
 - **CORS is effectively open.** The mutating routes reflect back whatever `Origin` a request sends (browsers disallow a wildcard alongside credentialed requests) — protection comes entirely from the Basic-auth check, not from origin filtering.
+- **OTA updates require a signing key you generate once.** `secure_boot_signing_key.pem` is gitignored like `wifi_credentials.h`; generate it with `espsecure.py generate_signing_key --version 2 --scheme ecdsa256 secure_boot_signing_key.pem` before your first build after Phase 7a — the `--scheme ecdsa256` flag matters, since `espsecure.py` defaults to an RSA key otherwise, which this project's ECDSA-based sdkconfig can't sign with. CI has its own copy in a repository secret (`OTA_SIGNING_KEY_PEM`) — see `.github/workflows/ci.yml`.
+- **Repartitioning (Phase 7a) requires `idf.py erase-flash`.** This wipes the NVS record store and blocklist — reflash and re-seed from `dns_records.h`/`dns_blocklist_defaults.h`, or re-add records via the CRUD API, after upgrading from a pre-Phase-7a build.

host_test/main/CMakeLists.txt

@@ -1,6 +1,7 @@
 # dns_wire.cpp is pulled in directly from the firmware's main/ component
 # (not linked as a library) so this test always exercises the exact same
 # source that ships — see the header comment on ../CMakeLists.txt.
-idf_component_register(SRCS "test_dns_wire.cpp" "../../main/dns_wire.cpp"
+idf_component_register(SRCS "test_dns_wire.cpp" "test_ota_version.cpp"
+                            "../../main/dns_wire.cpp" "../../main/ota_version.cpp"
                     INCLUDE_DIRS "." "../../main"
                     REQUIRES unity)

host_test/main/test_dns_wire.cpp

@@ -11,8 +11,26 @@
 #include <vector>
 
 #include "dns_wire.h"
+#include "ota_version.h"
 #include "unity.h"
 
+// test_ota_version.cpp's tests (Phase 7a) — plain (non-anonymous-namespace)
+// functions there so they're callable here; this file owns the single
+// merged app_main (see below), so its RUN_TEST calls need these declared.
+void test_parse_valid_with_v_prefix();
+void test_parse_valid_without_v_prefix();
+void test_parse_tolerates_git_describe_suffix();
+void test_parse_rejects_empty();
+void test_parse_rejects_garbage();
+void test_parse_rejects_missing_minor();
+void test_is_newer_minor_bump();
+void test_is_newer_false_when_older();
+void test_is_newer_false_when_equal();
+void test_is_newer_patch_bump();
+void test_is_newer_major_beats_minor_and_patch();
+void test_is_newer_false_on_unparseable_remote();
+void test_is_newer_false_on_unparseable_current();
+
 namespace {
 
 // Encodes a dotted name ("test.loc") as length-prefixed labels terminated
@@ -380,6 +398,22 @@ extern "C" void app_main(void)
 {
     UNITY_BEGIN();
     unity_run_all_tests();
+    // ota_version tests use plain RUN_TEST rather than TEST_CASE
+    // auto-registration (see test_ota_version.cpp), so they run here
+    // explicitly rather than via unity_run_all_tests() above.
+    RUN_TEST(test_parse_valid_with_v_prefix);
+    RUN_TEST(test_parse_valid_without_v_prefix);
+    RUN_TEST(test_parse_tolerates_git_describe_suffix);
+    RUN_TEST(test_parse_rejects_empty);
+    RUN_TEST(test_parse_rejects_garbage);
+    RUN_TEST(test_parse_rejects_missing_minor);
+    RUN_TEST(test_is_newer_minor_bump);
+    RUN_TEST(test_is_newer_false_when_older);
+    RUN_TEST(test_is_newer_false_when_equal);
+    RUN_TEST(test_is_newer_patch_bump);
+    RUN_TEST(test_is_newer_major_beats_minor_and_patch);
+    RUN_TEST(test_is_newer_false_on_unparseable_remote);
+    RUN_TEST(test_is_newer_false_on_unparseable_current);
     // ESP-IDF's linux-target port starts the FreeRTOS scheduler before
     // calling app_main and never tears it down when app_main returns —
     // the process would otherwise hang forever after printing results.

host_test/main/test_ota_version.cpp

@@ -0,0 +1,88 @@
+// Host-side unit tests for main/ota_version.h/.cpp (Phase 7a) — same
+// no-FreeRTOS-dependency, linux-target pattern as test_dns_wire.cpp:
+//
+//   idf.py --preview set-target linux -C host_test build
+//   ./host_test/build/host_test.elf
+
+#include "ota_version.h"
+#include "unity.h"
+
+// Not in an anonymous namespace, unlike test_dns_wire.cpp's helpers: these
+// need external linkage so test_dns_wire.cpp's app_main (the single merged
+// Unity entry point — see its file comment) can RUN_TEST() them via the
+// forward declarations there.
+
+void test_parse_valid_with_v_prefix()
+{
+    auto v = ota_version_parse("v1.2.3");
+    TEST_ASSERT_TRUE(v.has_value());
+    TEST_ASSERT_EQUAL_UINT(1, v->major);
+    TEST_ASSERT_EQUAL_UINT(2, v->minor);
+    TEST_ASSERT_EQUAL_UINT(3, v->patch);
+}
+
+void test_parse_valid_without_v_prefix()
+{
+    auto v = ota_version_parse("0.6.0");
+    TEST_ASSERT_TRUE(v.has_value());
+    TEST_ASSERT_EQUAL_UINT(0, v->major);
+    TEST_ASSERT_EQUAL_UINT(6, v->minor);
+    TEST_ASSERT_EQUAL_UINT(0, v->patch);
+}
+
+void test_parse_tolerates_git_describe_suffix()
+{
+    auto v = ota_version_parse("v0.6.0-3-gabc1234-dirty");
+    TEST_ASSERT_TRUE(v.has_value());
+    TEST_ASSERT_EQUAL_UINT(0, v->patch);
+}
+
+void test_parse_rejects_empty()
+{
+    TEST_ASSERT_FALSE(ota_version_parse("").has_value());
+}
+
+void test_parse_rejects_garbage()
+{
+    TEST_ASSERT_FALSE(ota_version_parse("notaversion").has_value());
+}
+
+void test_parse_rejects_missing_minor()
+{
+    TEST_ASSERT_FALSE(ota_version_parse("v1").has_value());
+}
+
+void test_is_newer_minor_bump()
+{
+    TEST_ASSERT_TRUE(ota_version_is_newer("v0.6.0", "v0.7.0"));
+}
+
+void test_is_newer_false_when_older()
+{
+    TEST_ASSERT_FALSE(ota_version_is_newer("v0.7.0", "v0.6.0"));
+}
+
+void test_is_newer_false_when_equal()
+{
+    TEST_ASSERT_FALSE(ota_version_is_newer("v0.6.0", "v0.6.0"));
+}
+
+void test_is_newer_patch_bump()
+{
+    TEST_ASSERT_TRUE(ota_version_is_newer("v0.6.0", "v0.6.1"));
+}
+
+void test_is_newer_major_beats_minor_and_patch()
+{
+    TEST_ASSERT_TRUE(ota_version_is_newer("v0.9.9", "v1.0.0"));
+}
+
+void test_is_newer_false_on_unparseable_remote()
+{
+    TEST_ASSERT_FALSE(ota_version_is_newer("v0.6.0", "garbage"));
+}
+
+void test_is_newer_false_on_unparseable_current()
+{
+    TEST_ASSERT_FALSE(ota_version_is_newer("garbage", "v1.0.0"));
+}