From a5720fef2a3c72d371702f697854f79ed2c4d177 Mon Sep 17 00:00:00 2001 From: Alexander Nguyen Date: Wed, 10 Jun 2026 01:22:28 -0700 Subject: [PATCH 01/21] docs(spec): rsync-over-ssh NAS transport design + implementation plan Co-Authored-By: Claude Fable 5 --- .../2026-06-10-rsync-ssh-nas-transport.md | 2081 +++++++++++++++++ ...26-06-10-rsync-ssh-nas-transport-design.md | 529 +++++ 2 files changed, 2610 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-10-rsync-ssh-nas-transport.md create mode 100644 docs/superpowers/specs/2026-06-10-rsync-ssh-nas-transport-design.md diff --git a/docs/superpowers/plans/2026-06-10-rsync-ssh-nas-transport.md b/docs/superpowers/plans/2026-06-10-rsync-ssh-nas-transport.md new file mode 100644 index 0000000..7daced4 --- /dev/null +++ b/docs/superpowers/plans/2026-06-10-rsync-ssh-nas-transport.md @@ -0,0 +1,2081 @@ +# rsync-over-ssh NAS Transport Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add an `rsync_ssh` NAS transport (selected per-instance by `nas.transport`) so cluster machines — where IT blocks SMB and rclone-SFTP but allows rsync-over-ssh — can push, reconcile, verify, and probe against the Synology NAS. + +**Architecture:** A new `RsyncSshDriver` mirrors `RcloneDriver`'s thin-subprocess-wrapper shape behind a shared `NasTransportDriver` protocol and a `build_nas_driver` factory. Targets reuse the existing `:` string (`user@host:/path` is valid rsync syntax), so `NASSyncClient`'s state machine, reconcile loop, and cleanup gate are untouched. All ops use rsync protocol primitives only (no remote command exec): push = `rsync -rt`, manifest = `--list-only` parse, verify = `--checksum` dry-run itemize, probe = non-recursive `--list-only`. + +**Spec:** `docs/superpowers/specs/2026-06-10-rsync-ssh-nas-transport-design.md` (all OQs resolved 2026-06-10). + +**Tech Stack:** Python 3.12, Pydantic v2, pytest (`asyncio_mode = "auto"`, so `async def` tests need no marker). Test runner: `uv run --extra test pytest`. Lint: `uvx ruff check`. **Run sync test suites one at a time** (they flake under CPU load). + +**Out of scope (deliberate):** renaming `SetupState.INCOMPLETE_NO_NAS_REMOTE` / `SetupNextAction.CONFIGURE_RCLONE_REMOTE` (kept; only UI copy changes); the `error_kind=None` → HASH_MISMATCH retry-routing debt; `ui/pages/wizard_equipment.py` copy polish; staging-hop rsync support. + +--- + +## File map + +| File | Change | +| --- | --- | +| `src/exlab_wizard/constants/enums.py` | new `SyncTransport` StrEnum | +| `src/exlab_wizard/constants/__init__.py` | export `SyncTransport` | +| `src/exlab_wizard/config/models.py` | `NasConfig.transport/ssh_port/ssh_identity_file` + validator | +| `src/exlab_wizard/sync/manifest.py` | new `parse_rsync_listing()` | +| `src/exlab_wizard/sync/transports/__init__.py` | `NasTransportDriver` protocol, `build_nas_driver` factory | +| `src/exlab_wizard/sync/transports/rclone.py` | new `lsjson_manifest()` shim | +| `src/exlab_wizard/sync/transports/rsync_ssh.py` | **new** `RsyncSshDriver` | +| `src/exlab_wizard/sync/nas_client.py` | factory routing + stage-mode bypass + `lsjson_manifest` | +| `src/exlab_wizard/sync/verifier.py` | protocol typing, drop no-arg default | +| `src/exlab_wizard/tray/dependencies.py` | transport-aware gate hydration + probe | +| `src/exlab_wizard/paths.py` | `_missing_nas_fields` transport branch | +| `src/exlab_wizard/ui/pages/settings.py` | transport-aware NAS-Remote section | +| `src/exlab_wizard/ui/pages/main.py` | transport-neutral banner subline | +| `tests/fixtures/stub_rsync.py` | **new** stub rsync binary | +| `tests/unit/...` | per-task test files (see tasks) | +| `tests/docker/*` | rsync-over-ssh leg + characterization tests | +| `docs/setup/`, `design_specs/design_spec_sections/{04,07,09}_*.md`, `README.md` | docs | + +--- + +### Task 1: `SyncTransport` enum + +**Files:** +- Modify: `src/exlab_wizard/constants/enums.py` (after `SyncMode`, ~line 176) +- Modify: `src/exlab_wizard/constants/__init__.py` (import list ~line 38, `__all__` ~line 264) +- Test: `tests/unit/constants/test_enum_literal_alignment.py`, `tests/unit/constants/test_enums.py` + +- [ ] **Step 1: Write the failing test** + +Add to the `@pytest.mark.parametrize` list in `tests/unit/constants/test_enum_literal_alignment.py` (alongside the `SetupNextAction` entry): + +```python + (enums.SyncTransport, frozenset({"rclone", "rsync_ssh"})), +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run --extra test pytest tests/unit/constants/test_enum_literal_alignment.py -v` +Expected: FAIL with `AttributeError: module 'exlab_wizard.constants.enums' has no attribute 'SyncTransport'` + +- [ ] **Step 3: Implement the enum** + +In `src/exlab_wizard/constants/enums.py`, directly after the `SyncMode` class: + +```python +class SyncTransport(StrEnum): + """Which binary the NAS sync subsystem shells out to. + + rsync-over-ssh NAS transport design (2026-06-10). Selected per + instance by ``nas.transport``: ``rclone`` (default) drives the named + remote in the operator's rclone.conf; ``rsync_ssh`` drives + ``rsync -e ssh`` against ``nas.remote`` as a ``user@host`` target. + Stage-mode equipment always uses rclone regardless of this value. + """ + + RCLONE = "rclone" + RSYNC_SSH = "rsync_ssh" +``` + +In `src/exlab_wizard/constants/__init__.py`: add `SyncTransport,` to the `from exlab_wizard.constants.enums import (...)` block (alphabetical, next to `SyncMode`) and `"SyncTransport",` to `__all__` (next to `"SyncMode"`). + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `uv run --extra test pytest tests/unit/constants/ -v` +Expected: PASS (alignment test plus the existing `test_enums.py` suite; if `test_enums.py` asserts a closed member list of the module, add `SyncTransport` there as well — check its failure output). + +- [ ] **Step 5: Commit** + +```bash +git add src/exlab_wizard/constants/ tests/unit/constants/ +git commit -m "feat(constants): add SyncTransport enum for NAS transport selection" +``` + +--- + +### Task 2: `NasConfig` transport fields + validator + +**Files:** +- Modify: `src/exlab_wizard/config/models.py` (`NasConfig`, ~line 273; import block ~line 36) +- Test: `tests/unit/config/test_models.py` + +- [ ] **Step 1: Write the failing tests** + +Add to `tests/unit/config/test_models.py`: + +```python +class TestNasTransport: + def test_transport_defaults_to_rclone(self) -> None: + nas = NasConfig() + assert nas.transport == SyncTransport.RCLONE + assert nas.ssh_port == 22 + assert nas.ssh_identity_file == "" + + def test_rsync_ssh_accepts_user_at_host(self) -> None: + nas = NasConfig( + transport="rsync_ssh", + remote="svc-sync@nas01.lab.example", + base_root="/volume1/lab", + ssh_identity_file="~/.ssh/id_exlab", + ) + assert nas.transport == SyncTransport.RSYNC_SSH + assert nas.remote == "svc-sync@nas01.lab.example" + + def test_rsync_ssh_rejects_empty_remote(self) -> None: + with pytest.raises(ValidationError, match="requires nas.remote"): + NasConfig(transport="rsync_ssh", remote="") + + @pytest.mark.parametrize("bad", ["nas01", "@nas01", "svc-sync@"]) + def test_rsync_ssh_rejects_non_user_at_host(self, bad: str) -> None: + with pytest.raises(ValidationError, match="user@host"): + NasConfig(transport="rsync_ssh", remote=bad) + + def test_rclone_mode_remote_shape_unrestricted(self) -> None: + assert NasConfig(remote="nas01").remote == "nas01" + + def test_transport_serializes_as_string(self) -> None: + dumped = NasConfig( + transport="rsync_ssh", remote="u@h" + ).model_dump() + assert dumped["transport"] == "rsync_ssh" +``` + +Imports needed at top of the test module (add to existing import blocks): `from pydantic import ValidationError` and `from exlab_wizard.constants import SyncTransport` (the module already imports `NasConfig` and `pytest`). + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run --extra test pytest tests/unit/config/test_models.py -k TestNasTransport -v` +Expected: FAIL with `ValidationError` ("Extra inputs are not permitted" — `extra="forbid"` rejects the unknown `transport` key). + +- [ ] **Step 3: Implement the fields** + +In `src/exlab_wizard/config/models.py`: add `SyncTransport,` to the existing `from exlab_wizard.constants import (...)` block. Then inside `NasConfig`, after the `rclone_config_path: str = ""` field: + +```python + # rsync-over-ssh NAS transport (2026-06-10 design). ``rsync_ssh`` drives + # ``rsync -e ssh`` with ``remote`` as a ``user@host`` target prefix; + # ``rclone_config_path`` and ``perf`` are rclone-only and ignored in + # rsync mode. Stage-mode equipment always uses rclone regardless. + transport: SyncTransport = SyncTransport.RCLONE + ssh_port: int = Field(default=22, ge=1, le=65535) + ssh_identity_file: str = "" +``` + +And after the existing fields, the serializer + validator (mirroring `EquipmentConfig._serialize_sync_mode`): + +```python + @field_serializer("transport") + def _serialize_transport(self, value: SyncTransport) -> str: + return value.value + + @model_validator(mode="after") + def _validate_rsync_ssh(self) -> NasConfig: + if self.transport != SyncTransport.RSYNC_SSH: + return self + if not self.remote: + msg = "nas.transport 'rsync_ssh' requires nas.remote ('user@host')" + raise ValueError(msg) + user, sep, host = self.remote.partition("@") + if not (sep and user and host): + msg = ( + f"nas.remote {self.remote!r} must be 'user@host' when " + "nas.transport is 'rsync_ssh'" + ) + raise ValueError(msg) + return self +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `uv run --extra test pytest tests/unit/config/test_models.py -v` +Expected: PASS (whole module — confirms no regression in existing `NasConfig` tests). + +- [ ] **Step 5: Commit** + +```bash +git add src/exlab_wizard/config/models.py tests/unit/config/test_models.py +git commit -m "feat(config): nas.transport selector + ssh fields on NasConfig" +``` + +--- + +### Task 3: `parse_rsync_listing` manifest parser + +**Files:** +- Modify: `src/exlab_wizard/sync/manifest.py` +- Test: `tests/unit/sync/test_manifest.py` + +- [ ] **Step 1: Write the failing tests** + +Add to `tests/unit/sync/test_manifest.py`: + +```python +from datetime import datetime + +from exlab_wizard.sync.manifest import parse_rsync_listing + + +def _stamp(epoch: float) -> str: + """Format ``epoch`` the way the rsync client formats listing mtimes + (local timezone, 1 s resolution).""" + return datetime.fromtimestamp(epoch).strftime("%Y/%m/%d %H:%M:%S") + + +class TestParseRsyncListing: + def test_plain_file(self) -> None: + raw = f"-rw-r--r-- 2048 {_stamp(1_750_000_000)} data/reading_001.csv\n" + manifest = parse_rsync_listing(raw) + assert manifest.has("data/reading_001.csv") + assert manifest.entries["data/reading_001.csv"].size == 2048 + + def test_filename_with_spaces_survives(self) -> None: + raw = f"-rw-r--r-- 1234 {_stamp(1_750_000_000)} image data 001.tif\n" + manifest = parse_rsync_listing(raw) + assert manifest.has("image data 001.tif") + + def test_comma_grouped_size(self) -> None: + raw = f"-rw-r--r-- 1,234,567 {_stamp(1_750_000_000)} big.bin\n" + assert parse_rsync_listing(raw).entries["big.bin"].size == 1234567 + + def test_mtime_roundtrips_through_local_timezone(self) -> None: + epoch = 1_750_000_000.0 + raw = f"-rw-r--r-- 100 {_stamp(epoch)} f.txt\n" + manifest = parse_rsync_listing(raw) + assert manifest.matches("f.txt", 100, epoch, tolerance_s=2) + assert not manifest.matches("f.txt", 100, epoch + 3600, tolerance_s=2) + + def test_directories_and_dot_dropped(self) -> None: + stamp = _stamp(1_750_000_000) + raw = ( + f"drwxr-xr-x 4096 {stamp} .\n" + f"drwxr-xr-x 4096 {stamp} data\n" + f"-rw-r--r-- 10 {stamp} data/x.csv\n" + ) + manifest = parse_rsync_listing(raw) + assert set(manifest.entries) == {"data/x.csv"} + + def test_octal_escapes_unescaped(self) -> None: + raw = f"-rw-r--r-- 10 {_stamp(1_750_000_000)} weird\\#012name.txt\n" + assert parse_rsync_listing(raw).has("weird\nname.txt") + + def test_garbage_lines_ignored(self) -> None: + assert parse_rsync_listing("sending incremental file list\n\n") .entries == {} + + def test_empty_input(self) -> None: + assert parse_rsync_listing("").entries == {} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run --extra test pytest tests/unit/sync/test_manifest.py -k TestParseRsyncListing -v` +Expected: FAIL with `ImportError: cannot import name 'parse_rsync_listing'` + +- [ ] **Step 3: Implement the parser** + +In `src/exlab_wizard/sync/manifest.py`: extend the imports (`from datetime import UTC, datetime`), add `"parse_rsync_listing"` to `__all__`, and append: + +```python +# ``rsync --list-only`` line: perms, size (possibly digit-grouped), the +# fixed-format timestamp, then EVERYTHING after the single separating +# space is the path — filenames containing spaces appear literally +# (spec-review blocker, 2026-06-10), so the line must be anchored on the +# timestamp, never whitespace-split. +_RSYNC_LIST_RE = re.compile( + r"^(?P\S+)\s+(?P[\d,.]+)\s+" + r"(?P\d{4}/\d{2}/\d{2})\s+(?P